From 2c86c8c16665abc8c23b490f898f88efede3b006 Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Wed, 8 Apr 2026 13:35:56 +0800 Subject: [PATCH 01/10] feat: AI Agent-driven test infrastructure (Phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundational infrastructure for automated test planning from PR diffs: - pkg/testinfra/types: Domain types (TestPlan, TestTask, DiffSummary, FileChange, priority/status enums) - pkg/testinfra/planner: Unified diff parser, 22 static path→test-category mapping rules, Planner combining diff + mapping into prioritized TestPlan - pkg/testinfra/executor: Executor interface + LocalExecutor wrapping go test/go vet/mo-tester - pkg/testinfra/dedup: SQL fingerprinting (normalize case/whitespace/literals → SHA-256), .test file parser, batch dedup - cmd/mo-testplan: CLI tool for generating TestPlan from git diff - .github/workflows/testplan.yaml: PR trigger → generate TestPlan → upload artifact → post summary comment --- .github/workflows/testplan.yaml | 136 ++++++++++ cmd/mo-testplan/main.go | 107 ++++++++ pkg/testinfra/dedup/dedup.go | 211 ++++++++++++++++ pkg/testinfra/dedup/dedup_test.go | 179 ++++++++++++++ pkg/testinfra/executor/executor.go | 208 ++++++++++++++++ pkg/testinfra/executor/executor_test.go | 116 +++++++++ pkg/testinfra/planner/diff.go | 142 +++++++++++ pkg/testinfra/planner/mapping.go | 315 ++++++++++++++++++++++++ pkg/testinfra/planner/planner.go | 126 ++++++++++ pkg/testinfra/planner/planner_test.go | 313 +++++++++++++++++++++++ pkg/testinfra/types/types.go | 208 ++++++++++++++++ pkg/testinfra/types/types_test.go | 128 ++++++++++ 12 files changed, 2189 insertions(+) create mode 100644 .github/workflows/testplan.yaml create mode 100644 cmd/mo-testplan/main.go create mode 100644 pkg/testinfra/dedup/dedup.go create mode 100644 pkg/testinfra/dedup/dedup_test.go create mode 100644 pkg/testinfra/executor/executor.go create mode 100644 pkg/testinfra/executor/executor_test.go create mode 100644 pkg/testinfra/planner/diff.go create mode 100644 pkg/testinfra/planner/mapping.go create mode 100644 pkg/testinfra/planner/planner.go create mode 100644 pkg/testinfra/planner/planner_test.go create mode 100644 pkg/testinfra/types/types.go create mode 100644 pkg/testinfra/types/types_test.go diff --git a/.github/workflows/testplan.yaml b/.github/workflows/testplan.yaml new file mode 100644 index 0000000000000..490c4382ff149 --- /dev/null +++ b/.github/workflows/testplan.yaml @@ -0,0 +1,136 @@ +name: Generate TestPlan + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - '**.go' + - '**.c' + - '**.h' + - 'test/distributed/**' + +concurrency: + group: testplan-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + generate-testplan: + name: Generate TestPlan + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Build mo-testplan + run: go build -o mo-testplan ./cmd/mo-testplan + + - name: Generate diff + run: | + git diff origin/${{ github.base_ref }}...${{ github.event.pull_request.head.sha }} > /tmp/pr.diff + + - name: Generate TestPlan (JSON) + run: | + ./mo-testplan \ + --pr ${{ github.event.pull_request.number }} \ + --base ${{ github.base_ref }} \ + --head ${{ github.head_ref }} \ + --diff /tmp/pr.diff \ + --format json > /tmp/testplan.json + + - name: Generate TestPlan (Summary) + id: summary + run: | + SUMMARY=$(./mo-testplan \ + --pr ${{ github.event.pull_request.number }} \ + --base ${{ github.base_ref }} \ + --head ${{ github.head_ref }} \ + --diff /tmp/pr.diff \ + --format summary) + echo "testplan<> $GITHUB_OUTPUT + echo "$SUMMARY" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Upload TestPlan artifact + uses: actions/upload-artifact@v4 + with: + name: testplan-pr${{ github.event.pull_request.number }} + path: /tmp/testplan.json + + - name: Comment on PR + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const testplan = JSON.parse(fs.readFileSync('/tmp/testplan.json', 'utf8')); + const taskCount = testplan.tasks.length; + const utTasks = testplan.tasks.filter(t => t.type === 'unit_test').length; + const bvtTasks = testplan.tasks.filter(t => t.type === 'bvt').length; + const scaTasks = testplan.tasks.filter(t => t.type === 'sca').length; + + const body = `## 🤖 AI TestPlan Generated + + **PR #${testplan.pr_number}** (${testplan.head_branch} → ${testplan.base_branch}) + + | Metric | Count | + |--------|-------| + | Files Changed | ${testplan.diff_summary.files.length} | + | Lines Added | ${testplan.diff_summary.total_added} | + | Lines Deleted | ${testplan.diff_summary.total_deleted} | + | **Total Tasks** | **${taskCount}** | + | Unit Tests | ${utTasks} | + | BVT Tests | ${bvtTasks} | + | Static Analysis | ${scaTasks} | + +
+ 📋 Task Details + + \`\`\` + ${process.env.TESTPLAN_SUMMARY || 'See artifact for details'} + \`\`\` + +
+ +
+ 📦 Affected Packages + + ${[...new Set(testplan.diff_summary.files.filter(f => f.package).map(f => f.package))].map(p => '- `' + p + '`').join('\n')} + +
+ + > 💡 Full TestPlan JSON available as workflow artifact. + `; + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const botComment = comments.find(c => c.body.includes('🤖 AI TestPlan Generated')); + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body, + }); + } + env: + TESTPLAN_SUMMARY: ${{ steps.summary.outputs.testplan }} diff --git a/cmd/mo-testplan/main.go b/cmd/mo-testplan/main.go new file mode 100644 index 0000000000000..2583831e24480 --- /dev/null +++ b/cmd/mo-testplan/main.go @@ -0,0 +1,107 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// mo-testplan is a CLI tool that generates a structured TestPlan from a +// git diff. It is designed to be used in CI pipelines (e.g., GitHub Actions) +// to automatically determine which tests to run for a given PR. +// +// Usage: +// +// # From git diff on stdin: +// git diff origin/main...HEAD | mo-testplan --pr 12345 --base main --head feature/x +// +// # From a diff file: +// mo-testplan --pr 12345 --base main --head feature/x --diff changes.patch +// +// # Output format: +// mo-testplan --pr 12345 --base main --head feature/x --format json +// mo-testplan --pr 12345 --base main --head feature/x --format summary +package main + +import ( + "flag" + "fmt" + "io" + "os" + + "github.com/matrixorigin/matrixone/pkg/testinfra/planner" +) + +func main() { + var ( + prNumber int + baseBranch string + headBranch string + diffFile string + format string + ) + + flag.IntVar(&prNumber, "pr", 0, "PR number") + flag.StringVar(&baseBranch, "base", "main", "base branch name") + flag.StringVar(&headBranch, "head", "", "head branch name") + flag.StringVar(&diffFile, "diff", "", "path to diff file (reads stdin if empty)") + flag.StringVar(&format, "format", "json", "output format: json or summary") + flag.Parse() + + // Read diff + var diffBytes []byte + var err error + if diffFile != "" { + diffBytes, err = os.ReadFile(diffFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error reading diff file: %v\n", err) + os.Exit(1) + } + } else { + diffBytes, err = io.ReadAll(os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "Error reading stdin: %v\n", err) + os.Exit(1) + } + } + + if len(diffBytes) == 0 { + fmt.Fprintf(os.Stderr, "No diff provided. Pipe a git diff or use --diff flag.\n") + os.Exit(1) + } + + // Generate plan + p := planner.NewPlanner() + plan := p.GeneratePlanFromDiff(string(diffBytes), prNumber, baseBranch, headBranch) + + // Output + switch format { + case "json": + data, err := plan.ToJSON() + if err != nil { + fmt.Fprintf(os.Stderr, "Error serializing plan: %v\n", err) + os.Exit(1) + } + fmt.Println(string(data)) + case "summary": + fmt.Println(plan.Summary) + fmt.Printf("\nTasks (%d):\n", len(plan.Tasks)) + for _, task := range plan.Tasks { + target := string(task.Package) + if target == "" { + target = string(task.Category) + } + fmt.Printf(" [%s] %s %-8s %s (%s)\n", + task.Priority.String(), task.ID, task.Type, target, task.Reason) + } + default: + fmt.Fprintf(os.Stderr, "Unknown format: %s (use json or summary)\n", format) + os.Exit(1) + } +} diff --git a/pkg/testinfra/dedup/dedup.go b/pkg/testinfra/dedup/dedup.go new file mode 100644 index 0000000000000..3de49b72d9a0d --- /dev/null +++ b/pkg/testinfra/dedup/dedup.go @@ -0,0 +1,211 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package dedup provides SQL test case deduplication based on SQL statement +// fingerprinting. It normalizes SQL statements to produce a canonical +// fingerprint that can be compared for equality, enabling detection of +// duplicate or near-duplicate test cases. +package dedup + +import ( + "crypto/sha256" + "fmt" + "regexp" + "strings" +) + +// Fingerprint represents the normalized hash of a SQL statement. +type Fingerprint string + +// SQLFingerprinter normalizes SQL statements to detect duplicates. +type SQLFingerprinter struct{} + +// NewSQLFingerprinter creates a new fingerprinter. +func NewSQLFingerprinter() *SQLFingerprinter { + return &SQLFingerprinter{} +} + +// Fingerprint normalizes a SQL statement and returns its fingerprint. +func (f *SQLFingerprinter) Fingerprint(sql string) Fingerprint { + normalized := NormalizeSQL(sql) + hash := sha256.Sum256([]byte(normalized)) + return Fingerprint(fmt.Sprintf("%x", hash[:8])) +} + +// NormalizeSQL produces a canonical form of a SQL statement by: +// - converting to lowercase +// - replacing literal numbers with ? +// - replacing quoted strings with ? +// - collapsing whitespace +// - removing trailing semicolons +// - removing comments +func NormalizeSQL(sql string) string { + s := sql + + // Remove single-line comments + s = removeSingleLineComments(s) + + // Remove multi-line comments + s = removeMultiLineComments(s) + + // Lowercase + s = strings.ToLower(s) + + // Replace quoted strings (single and double quotes) + s = replaceQuotedStrings(s) + + // Replace numbers + s = replaceNumbers(s) + + // Collapse whitespace + s = collapseWhitespace(s) + + // Trim + s = strings.TrimSpace(s) + + // Remove trailing semicolons + s = strings.TrimRight(s, ";") + s = strings.TrimSpace(s) + + return s +} + +var singleLineCommentRe = regexp.MustCompile(`--[^\n]*`) + +func removeSingleLineComments(s string) string { + return singleLineCommentRe.ReplaceAllString(s, "") +} + +var multiLineCommentRe = regexp.MustCompile(`/\*.*?\*/`) + +func removeMultiLineComments(s string) string { + return multiLineCommentRe.ReplaceAllString(s, "") +} + +// replaceQuotedStrings replaces 'string' and "string" with ? +func replaceQuotedStrings(s string) string { + var result strings.Builder + i := 0 + for i < len(s) { + if s[i] == '\'' || s[i] == '"' { + quote := s[i] + i++ + for i < len(s) && s[i] != quote { + if s[i] == '\\' { + i++ // skip escaped char + } + i++ + } + if i < len(s) { + i++ // skip closing quote + } + result.WriteByte('?') + } else { + result.WriteByte(s[i]) + i++ + } + } + return result.String() +} + +// numberRe matches standalone numbers (integers and decimals). +var numberRe = regexp.MustCompile(`\b\d+(\.\d+)?\b`) + +func replaceNumbers(s string) string { + return numberRe.ReplaceAllString(s, "?") +} + +var whitespaceRe = regexp.MustCompile(`\s+`) + +func collapseWhitespace(s string) string { + return whitespaceRe.ReplaceAllString(s, " ") +} + +// DedupResult describes the result of deduplication. +type DedupResult struct { + // Unique are SQL statements that have no duplicates. + Unique []string + // Duplicates maps a fingerprint to the group of duplicate SQLs. + Duplicates map[Fingerprint][]string +} + +// Dedup takes a list of SQL statements and identifies duplicates. +func (f *SQLFingerprinter) Dedup(sqls []string) *DedupResult { + groups := make(map[Fingerprint][]string) + for _, sql := range sqls { + fp := f.Fingerprint(sql) + groups[fp] = append(groups[fp], sql) + } + + result := &DedupResult{ + Duplicates: make(map[Fingerprint][]string), + } + for fp, group := range groups { + if len(group) == 1 { + result.Unique = append(result.Unique, group[0]) + } else { + result.Duplicates[fp] = group + } + } + return result +} + +// ExtractSQLStatements parses a mo-tester .test file content and extracts +// the SQL statements from it, ignoring comments and tag lines. +func ExtractSQLStatements(content string) []string { + var statements []string + var current strings.Builder + + lines := strings.Split(content, "\n") + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Skip empty lines + if trimmed == "" { + continue + } + + // Skip mo-tester tag lines (-- @bvt, -- @skip, etc.) + if strings.HasPrefix(trimmed, "-- @") { + continue + } + + // Skip pure comment lines + if strings.HasPrefix(trimmed, "--") { + continue + } + + current.WriteString(trimmed) + current.WriteString(" ") + + // Statement ends with semicolon + if strings.HasSuffix(trimmed, ";") { + stmt := strings.TrimSpace(current.String()) + if stmt != "" { + statements = append(statements, stmt) + } + current.Reset() + } + } + + // Handle statement without trailing semicolon + if current.Len() > 0 { + stmt := strings.TrimSpace(current.String()) + if stmt != "" { + statements = append(statements, stmt) + } + } + + return statements +} diff --git a/pkg/testinfra/dedup/dedup_test.go b/pkg/testinfra/dedup/dedup_test.go new file mode 100644 index 0000000000000..2d2b2db2e2ea8 --- /dev/null +++ b/pkg/testinfra/dedup/dedup_test.go @@ -0,0 +1,179 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dedup + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeSQL(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "basic select", + input: "SELECT * FROM t1 WHERE id = 42;", + want: "select * from t1 where id = ?", + }, + { + name: "string literals", + input: "INSERT INTO t1 VALUES ('hello', 'world');", + want: "insert into t1 values (?, ?)", + }, + { + name: "comments removed", + input: "SELECT * FROM t1; -- this is a comment", + want: "select * from t1", + }, + { + name: "multi-line comment", + input: "SELECT /* inline */ * FROM t1;", + want: "select * from t1", + }, + { + name: "whitespace collapsed", + input: "SELECT * FROM t1\n WHERE id = 1;", + want: "select * from t1 where id = ?", + }, + { + name: "case insensitive", + input: "SELECT * FROM T1 WHERE Name = 'Alice';", + want: "select * from t1 where name = ?", + }, + { + name: "decimal numbers", + input: "SELECT * FROM t1 WHERE val > 3.14;", + want: "select * from t1 where val > ?", + }, + { + name: "pure numbers", + input: "SELECT 42 FROM dual;", + want: "select ? from dual", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NormalizeSQL(tt.input) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestFingerprint(t *testing.T) { + f := NewSQLFingerprinter() + + // Same logical query should produce same fingerprint + fp1 := f.Fingerprint("SELECT * FROM t1 WHERE id = 42;") + fp2 := f.Fingerprint("select * from t1 where id = 100;") + assert.Equal(t, fp1, fp2) + + // Different queries should produce different fingerprints + fp3 := f.Fingerprint("INSERT INTO t1 VALUES (1, 2);") + assert.NotEqual(t, fp1, fp3) +} + +func TestDedup(t *testing.T) { + f := NewSQLFingerprinter() + sqls := []string{ + "SELECT * FROM t1 WHERE id = 1;", + "SELECT * FROM t1 WHERE id = 2;", // duplicate of first + "INSERT INTO t1 VALUES (1, 'a');", + "INSERT INTO t1 VALUES (2, 'b');", // duplicate of third + "DELETE FROM t1 WHERE id = 1;", // unique + } + + result := f.Dedup(sqls) + + assert.Len(t, result.Unique, 1) // only DELETE is unique + assert.Contains(t, result.Unique[0], "DELETE") + assert.Len(t, result.Duplicates, 2) // SELECT group and INSERT group + + // Check that duplicate groups have 2 items each and contain the right statements + for _, group := range result.Duplicates { + assert.Len(t, group, 2) + } + // Verify specific groupings via fingerprint + selectFP := f.Fingerprint(sqls[0]) + insertFP := f.Fingerprint(sqls[2]) + assert.Equal(t, selectFP, f.Fingerprint(sqls[1]), "sqls[0] and sqls[1] should share fingerprint") + assert.Equal(t, insertFP, f.Fingerprint(sqls[3]), "sqls[2] and sqls[3] should share fingerprint") + assert.NotEqual(t, selectFP, insertFP, "SELECT and INSERT fingerprints should differ") +} + +func TestDedupAllUnique(t *testing.T) { + f := NewSQLFingerprinter() + sqls := []string{ + "SELECT * FROM t1;", + "INSERT INTO t1 VALUES (1);", + "DELETE FROM t1 WHERE id = 1;", + } + + result := f.Dedup(sqls) + assert.Len(t, result.Unique, 3) + assert.Empty(t, result.Duplicates) +} + +func TestExtractSQLStatements(t *testing.T) { + content := `-- @bvt:issue#12345 +-- This is a comment +CREATE TABLE t1 (a INT, b VARCHAR(100)); +INSERT INTO t1 VALUES (1, 'hello'); +INSERT INTO t1 VALUES (2, 'world'); + +-- @sortkey:0 +SELECT * FROM t1 +WHERE a > 0 +ORDER BY a; + +DROP TABLE t1; +` + + stmts := ExtractSQLStatements(content) + require.Len(t, stmts, 5) + assert.Contains(t, stmts[0], "CREATE TABLE") + assert.Contains(t, stmts[1], "INSERT INTO") + assert.Contains(t, stmts[2], "INSERT INTO") + assert.Contains(t, stmts[3], "SELECT * FROM t1") + // Multi-line SELECT should be joined + assert.Contains(t, stmts[3], "WHERE a > 0") + assert.Contains(t, stmts[4], "DROP TABLE") +} + +func TestExtractSQLStatementsEmpty(t *testing.T) { + stmts := ExtractSQLStatements("") + assert.Empty(t, stmts) +} + +func TestExtractSQLStatementsTagsOnly(t *testing.T) { + content := `-- @bvt:issue#999 +-- @skip:issue#888 +-- Just comments +` + stmts := ExtractSQLStatements(content) + assert.Empty(t, stmts) +} + +func TestExtractSQLStatementsNoSemicolon(t *testing.T) { + content := `SELECT 1` + stmts := ExtractSQLStatements(content) + require.Len(t, stmts, 1) + assert.Equal(t, "SELECT 1", stmts[0]) +} diff --git a/pkg/testinfra/executor/executor.go b/pkg/testinfra/executor/executor.go new file mode 100644 index 0000000000000..94856b87da9d6 --- /dev/null +++ b/pkg/testinfra/executor/executor.go @@ -0,0 +1,208 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package executor provides interfaces and implementations for executing +// test tasks described by a TestPlan. It wraps the existing optools shell +// scripts and provides a programmatic API for running unit tests, BVT +// tests, and static analysis. +package executor + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// Result holds the outcome of a single test task execution. +type Result struct { + TaskID string `json:"task_id"` + Status types.TaskStatus `json:"status"` + Output string `json:"output"` + Duration time.Duration `json:"duration"` + Error string `json:"error,omitempty"` + StartedAt time.Time `json:"started_at"` + EndedAt time.Time `json:"ended_at"` +} + +// Executor defines the interface for running test tasks. +type Executor interface { + // Execute runs the given task and returns the result. + Execute(ctx context.Context, task types.TestTask) (*Result, error) +} + +// LocalExecutor runs tests on the local machine by invoking Go test +// commands and BVT scripts. +type LocalExecutor struct { + // RepoRoot is the absolute path to the matrixone repository root. + RepoRoot string + // UTTimeout is the timeout for unit test execution. + UTTimeout time.Duration + // Env holds additional environment variables for test execution. + Env []string +} + +// NewLocalExecutor creates a LocalExecutor with sensible defaults. +func NewLocalExecutor(repoRoot string) *LocalExecutor { + return &LocalExecutor{ + RepoRoot: repoRoot, + UTTimeout: 15 * time.Minute, + } +} + +// Execute runs a single test task. +func (e *LocalExecutor) Execute(ctx context.Context, task types.TestTask) (*Result, error) { + result := &Result{ + TaskID: task.ID, + Status: types.TaskStatusRunning, + StartedAt: time.Now(), + } + + var err error + switch task.Type { + case types.TestTypeUT: + err = e.executeUT(ctx, task, result) + case types.TestTypeBVT: + err = e.executeBVT(ctx, task, result) + case types.TestTypeSCA: + err = e.executeSCA(ctx, result) + default: + err = fmt.Errorf("unknown task type: %s", task.Type) + } + + result.EndedAt = time.Now() + result.Duration = result.EndedAt.Sub(result.StartedAt) + + if err != nil { + result.Status = types.TaskStatusFailed + result.Error = err.Error() + return result, nil + } + + result.Status = types.TaskStatusPassed + return result, nil +} + +func (e *LocalExecutor) executeUT(ctx context.Context, task types.TestTask, result *Result) error { + if task.Package == "" { + return fmt.Errorf("UT task requires a package") + } + + args := []string{ + "test", "-short", "-count=1", + "-timeout", e.UTTimeout.String(), + "-tags", "matrixone_test", + fmt.Sprintf("./%s", task.Package), + } + + cmd := exec.CommandContext(ctx, "go", args...) + cmd.Dir = e.RepoRoot + cmd.Env = append(cmd.Environ(), e.Env...) + + out, err := cmd.CombinedOutput() + result.Output = string(out) + return err +} + +func (e *LocalExecutor) executeBVT(ctx context.Context, task types.TestTask, result *Result) error { + if task.Category == "" && task.TestFile == "" { + return fmt.Errorf("BVT task requires a category or test_file") + } + + // For BVT, we document the command that would be run. + // Actual BVT execution requires mo-tester and a running MO instance, + // so in this first phase we record the intent. + var target string + if task.TestFile != "" { + target = task.TestFile + } else { + target = filepath.Join("test/distributed/cases", string(task.Category)) + } + + result.Output = fmt.Sprintf("[BVT] Target: %s\nTo execute: mo-tester -p %s", + target, filepath.Join(e.RepoRoot, target)) + return nil +} + +func (e *LocalExecutor) executeSCA(ctx context.Context, result *Result) error { + args := []string{ + "vet", "-tags", "matrixone_test", + "./pkg/...", + } + + cmd := exec.CommandContext(ctx, "go", args...) + cmd.Dir = e.RepoRoot + cmd.Env = append(cmd.Environ(), e.Env...) + + out, err := cmd.CombinedOutput() + result.Output = string(out) + return err +} + +// ExecutePlan runs all tasks in a TestPlan sequentially and returns results. +func ExecutePlan(ctx context.Context, executor Executor, plan *types.TestPlan) []*Result { + var results []*Result + for i := range plan.Tasks { + task := &plan.Tasks[i] + task.Status = types.TaskStatusRunning + + r, err := executor.Execute(ctx, *task) + if err != nil { + r = &Result{ + TaskID: task.ID, + Status: types.TaskStatusFailed, + Error: err.Error(), + } + } + task.Status = r.Status + results = append(results, r) + } + return results +} + +// FormatResults produces a human-readable summary of execution results. +func FormatResults(results []*Result) string { + var b strings.Builder + passed, failed, skipped := 0, 0, 0 + for _, r := range results { + switch r.Status { + case types.TaskStatusPassed: + passed++ + case types.TaskStatusFailed: + failed++ + case types.TaskStatusSkipped: + skipped++ + } + } + fmt.Fprintf(&b, "Execution Summary: %d passed, %d failed, %d skipped (total: %d)\n", + passed, failed, skipped, len(results)) + + for _, r := range results { + icon := "✅" + if r.Status == types.TaskStatusFailed { + icon = "❌" + } else if r.Status == types.TaskStatusSkipped { + icon = "⏭️" + } + fmt.Fprintf(&b, " %s %s [%s] %s\n", icon, r.TaskID, r.Status, r.Duration) + if r.Error != "" { + fmt.Fprintf(&b, " Error: %s\n", r.Error) + } + } + return b.String() +} diff --git a/pkg/testinfra/executor/executor_test.go b/pkg/testinfra/executor/executor_test.go new file mode 100644 index 0000000000000..be64958d8daad --- /dev/null +++ b/pkg/testinfra/executor/executor_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// mockExecutor is a simple Executor for testing that always succeeds. +type mockExecutor struct{} + +func (m *mockExecutor) Execute(_ context.Context, task types.TestTask) (*Result, error) { + return &Result{ + TaskID: task.ID, + Status: types.TaskStatusPassed, + Output: "mock: ok", + }, nil +} + +func TestExecutePlan(t *testing.T) { + plan := &types.TestPlan{ + Tasks: []types.TestTask{ + {ID: "t1", Type: types.TestTypeUT, Package: "pkg/sql/plan/..."}, + {ID: "t2", Type: types.TestTypeBVT, Category: types.CategoryOptimizer}, + }, + } + results := ExecutePlan(context.Background(), &mockExecutor{}, plan) + + assert.Len(t, results, 2) + for _, r := range results { + assert.Equal(t, types.TaskStatusPassed, r.Status) + } + // Plan tasks should be updated + assert.Equal(t, types.TaskStatusPassed, plan.Tasks[0].Status) + assert.Equal(t, types.TaskStatusPassed, plan.Tasks[1].Status) +} + +func TestFormatResults(t *testing.T) { + results := []*Result{ + {TaskID: "t1", Status: types.TaskStatusPassed}, + {TaskID: "t2", Status: types.TaskStatusFailed, Error: "test failed"}, + {TaskID: "t3", Status: types.TaskStatusSkipped}, + } + output := FormatResults(results) + assert.Contains(t, output, "1 passed") + assert.Contains(t, output, "1 failed") + assert.Contains(t, output, "1 skipped") + assert.Contains(t, output, "t2") + assert.Contains(t, output, "test failed") +} + +func TestLocalExecutorBVT(t *testing.T) { + e := NewLocalExecutor("/tmp/test-repo") + task := types.TestTask{ + ID: "bvt-1", + Type: types.TestTypeBVT, + Category: types.CategoryOptimizer, + } + result, err := e.Execute(context.Background(), task) + assert.NoError(t, err) + assert.Equal(t, types.TaskStatusPassed, result.Status) + assert.Contains(t, result.Output, "optimizer") +} + +func TestLocalExecutorBVTMissingCategory(t *testing.T) { + e := NewLocalExecutor("/tmp/test-repo") + task := types.TestTask{ + ID: "bvt-bad", + Type: types.TestTypeBVT, + } + result, err := e.Execute(context.Background(), task) + assert.NoError(t, err) + assert.Equal(t, types.TaskStatusFailed, result.Status) + assert.Contains(t, result.Error, "requires a category") +} + +func TestLocalExecutorUTMissingPackage(t *testing.T) { + e := NewLocalExecutor("/tmp/test-repo") + task := types.TestTask{ + ID: "ut-bad", + Type: types.TestTypeUT, + } + result, err := e.Execute(context.Background(), task) + assert.NoError(t, err) + assert.Equal(t, types.TaskStatusFailed, result.Status) + assert.Contains(t, result.Error, "requires a package") +} + +func TestLocalExecutorUnknownType(t *testing.T) { + e := NewLocalExecutor("/tmp/test-repo") + task := types.TestTask{ + ID: "unknown", + Type: "foobar", + } + result, err := e.Execute(context.Background(), task) + assert.NoError(t, err) + assert.Equal(t, types.TaskStatusFailed, result.Status) + assert.Contains(t, result.Error, "unknown task type") +} diff --git a/pkg/testinfra/planner/diff.go b/pkg/testinfra/planner/diff.go new file mode 100644 index 0000000000000..735a46e6020b0 --- /dev/null +++ b/pkg/testinfra/planner/diff.go @@ -0,0 +1,142 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package planner + +import ( + "bufio" + "path" + "regexp" + "strings" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// ParseUnifiedDiff parses a unified diff (as produced by `git diff`) and +// returns a DiffSummary describing the changed files. +// +// It extracts: +// - file paths from "diff --git a/... b/..." lines +// - change kind (added, deleted, modified, renamed) +// - Go package paths (derived from directory) +// - changed function names (best-effort, from @@ hunk headers) +// - total added / deleted line counts +func ParseUnifiedDiff(diffText string) *types.DiffSummary { + summary := &types.DiffSummary{} + + scanner := bufio.NewScanner(strings.NewReader(diffText)) + var currentFile *types.FileChange + + for scanner.Scan() { + line := scanner.Text() + + // --- detect new file in diff --- + if strings.HasPrefix(line, "diff --git ") { + if currentFile != nil { + summary.Files = append(summary.Files, *currentFile) + } + currentFile = parseDiffHeader(line) + continue + } + + if currentFile == nil { + continue + } + + // --- detect change kind --- + if strings.HasPrefix(line, "new file mode") { + currentFile.ChangeKind = "added" + continue + } + if strings.HasPrefix(line, "deleted file mode") { + currentFile.ChangeKind = "deleted" + continue + } + if strings.HasPrefix(line, "rename from ") || strings.HasPrefix(line, "rename to ") { + currentFile.ChangeKind = "renamed" + continue + } + + // --- extract function names from hunk headers --- + if strings.HasPrefix(line, "@@") { + if fn := extractFuncFromHunk(line); fn != "" { + currentFile.Functions = appendIfNew(currentFile.Functions, fn) + } + continue + } + + // --- count added/deleted lines --- + if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "+++") { + summary.TotalAdded++ + } + if strings.HasPrefix(line, "-") && !strings.HasPrefix(line, "---") { + summary.TotalDeleted++ + } + } + + if currentFile != nil { + summary.Files = append(summary.Files, *currentFile) + } + + return summary +} + +// parseDiffHeader parses a "diff --git a/path b/path" line. +func parseDiffHeader(line string) *types.FileChange { + // "diff --git a/pkg/sql/plan/build.go b/pkg/sql/plan/build.go" + parts := strings.SplitN(line, " b/", 2) + if len(parts) != 2 { + return &types.FileChange{Path: line, ChangeKind: "modified"} + } + filePath := parts[1] + fc := &types.FileChange{ + Path: filePath, + ChangeKind: "modified", + } + + // derive Go package from directory + dir := path.Dir(filePath) + if isGoPackage(filePath) && dir != "." { + fc.Package = dir + } + + return fc +} + +// hunkFuncRe matches the function name in a Go diff hunk header like: +// @@ -10,5 +10,6 @@ func (p *Planner) Build(... +var hunkFuncRe = regexp.MustCompile(`@@[^@]+@@\s+(?:func\s+(?:\([^)]+\)\s+)?(\w+))`) + +// extractFuncFromHunk tries to extract a Go function name from a hunk header. +func extractFuncFromHunk(line string) string { + matches := hunkFuncRe.FindStringSubmatch(line) + if len(matches) >= 2 { + return matches[1] + } + return "" +} + +// isGoPackage returns true if the file path looks like a Go source file. +func isGoPackage(filePath string) bool { + return strings.HasSuffix(filePath, ".go") +} + +func appendIfNew(slice []string, s string) []string { + for _, v := range slice { + if v == s { + return slice + } + } + return append(slice, s) +} diff --git a/pkg/testinfra/planner/mapping.go b/pkg/testinfra/planner/mapping.go new file mode 100644 index 0000000000000..1c81d1b4b960f --- /dev/null +++ b/pkg/testinfra/planner/mapping.go @@ -0,0 +1,315 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package planner implements the TestPlan generation logic. It analyses PR +// diffs, maps code changes to test categories, and produces a structured +// TestPlan that can be consumed by the executor. +package planner + +import ( + "strings" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// PathMapping defines the static mapping between source code path prefixes +// and the BVT test categories / UT packages they are expected to exercise. +type PathMapping struct { + // PathPrefix is matched against the beginning of a changed file path. + PathPrefix string + // UTPackages lists the Go packages whose unit tests should be run. + UTPackages []string + // BVTCategories lists the BVT categories whose .test files should run. + BVTCategories []types.TestCategory + // Priority is the default priority for tasks generated from this mapping. + Priority types.Priority +} + +// DefaultMappings returns the built-in static mapping table for the +// matrixone repository. This maps source code path prefixes to the test +// categories and UT packages they are likely to affect. +// +// The mapping is intentionally broad – it is better to run a few extra +// tests than to miss a regression. +func DefaultMappings() []PathMapping { + return []PathMapping{ + // ── SQL Plan / Optimizer ── + { + PathPrefix: "pkg/sql/plan/", + UTPackages: []string{"pkg/sql/plan/..."}, + BVTCategories: []types.TestCategory{types.CategoryOptimizer, types.CategoryPlanCache, types.CategoryJoin, types.CategorySubquery, types.CategoryCTE, types.CategoryRecursiveCTE, types.CategoryHint}, + Priority: types.PriorityHigh, + }, + // ── SQL Compile ── + { + PathPrefix: "pkg/sql/compile/", + UTPackages: []string{"pkg/sql/compile/..."}, + BVTCategories: []types.TestCategory{types.CategoryDDL, types.CategoryDML, types.CategoryFunction, types.CategoryExpression}, + Priority: types.PriorityHigh, + }, + // ── Column Executors ── + { + PathPrefix: "pkg/sql/colexec/", + UTPackages: []string{"pkg/sql/colexec/..."}, + BVTCategories: []types.TestCategory{types.CategoryFunction, types.CategoryExpression, types.CategoryJoin, types.CategoryWindow}, + Priority: types.PriorityHigh, + }, + // ── SQL Parsers ── + { + PathPrefix: "pkg/sql/parsers/", + UTPackages: []string{"pkg/sql/parsers/..."}, + BVTCategories: []types.TestCategory{types.CategoryDDL, types.CategoryDML, types.CategoryPrepare}, + Priority: types.PriorityHigh, + }, + // ── Distributed TAE Engine ── + { + PathPrefix: "pkg/vm/engine/disttae/", + UTPackages: []string{"pkg/vm/engine/disttae/..."}, + BVTCategories: []types.TestCategory{types.CategoryDisttae, types.CategoryPessimisticTransaction, types.CategoryOptimistic, types.CategorySnapshot, types.CategoryPITR}, + Priority: types.PriorityCritical, + }, + // ── TAE Engine ── + { + PathPrefix: "pkg/vm/engine/tae/", + UTPackages: []string{"pkg/vm/engine/tae/..."}, + BVTCategories: []types.TestCategory{types.CategoryDisttae, types.CategoryPessimisticTransaction}, + Priority: types.PriorityCritical, + }, + // ── Object IO ── + { + PathPrefix: "pkg/objectio/", + UTPackages: []string{"pkg/objectio/..."}, + BVTCategories: []types.TestCategory{types.CategoryLoadData, types.CategoryDisttae}, + Priority: types.PriorityHigh, + }, + // ── Frontend (session, auth, protocol) ── + { + PathPrefix: "pkg/frontend/", + UTPackages: []string{"pkg/frontend/..."}, + BVTCategories: []types.TestCategory{types.CategorySecurity, types.CategoryTenant, types.CategoryAccessControl, types.CategorySnapshot, types.CategoryPITR, types.CategorySystemVariable, types.CategorySet}, + Priority: types.PriorityHigh, + }, + // ── Container types (vector, batch, bytejson) ── + { + PathPrefix: "pkg/container/", + UTPackages: []string{"pkg/container/..."}, + BVTCategories: []types.TestCategory{types.CategoryDtype, types.CategoryArray, types.CategoryVector}, + Priority: types.PriorityMedium, + }, + // ── Fulltext ── + { + PathPrefix: "pkg/fulltext/", + UTPackages: []string{"pkg/fulltext/...", "pkg/sql/colexec/table_function/..."}, + BVTCategories: []types.TestCategory{types.CategoryFulltext}, + Priority: types.PriorityMedium, + }, + // ── CDC ── + { + PathPrefix: "pkg/cdc/", + UTPackages: []string{"pkg/cdc/..."}, + BVTCategories: []types.TestCategory{types.CategoryCDC}, + Priority: types.PriorityMedium, + }, + // ── UDF ── + { + PathPrefix: "pkg/udf/", + UTPackages: []string{"pkg/udf/..."}, + BVTCategories: []types.TestCategory{types.CategoryUDF}, + Priority: types.PriorityMedium, + }, + // ── Lock service ── + { + PathPrefix: "pkg/lockservice/", + UTPackages: []string{"pkg/lockservice/..."}, + BVTCategories: []types.TestCategory{types.CategoryPessimisticTransaction}, + Priority: types.PriorityHigh, + }, + // ── Transaction ── + { + PathPrefix: "pkg/txn/", + UTPackages: []string{"pkg/txn/..."}, + BVTCategories: []types.TestCategory{types.CategoryPessimisticTransaction, types.CategoryOptimistic}, + Priority: types.PriorityCritical, + }, + // ── Catalog ── + { + PathPrefix: "pkg/catalog/", + UTPackages: []string{"pkg/catalog/..."}, + BVTCategories: []types.TestCategory{types.CategoryDDL, types.CategoryDatabase, types.CategoryTable, types.CategorySystem}, + Priority: types.PriorityHigh, + }, + // ── File service ── + { + PathPrefix: "pkg/fileservice/", + UTPackages: []string{"pkg/fileservice/..."}, + BVTCategories: []types.TestCategory{types.CategoryStage, types.CategoryLoadData}, + Priority: types.PriorityMedium, + }, + // ── Partition ── + { + PathPrefix: "pkg/partition/", + UTPackages: []string{"pkg/partition/...", "pkg/partitionservice/...", "pkg/partitionprune/..."}, + BVTCategories: []types.TestCategory{types.CategoryDDL, types.CategoryDML}, + Priority: types.PriorityMedium, + }, + // ── Proxy ── + { + PathPrefix: "pkg/proxy/", + UTPackages: []string{"pkg/proxy/..."}, + BVTCategories: []types.TestCategory{types.CategoryTenant}, + Priority: types.PriorityLow, + }, + // ── Bootstrap ── + { + PathPrefix: "pkg/bootstrap/", + UTPackages: []string{"pkg/bootstrap/..."}, + BVTCategories: []types.TestCategory{types.CategorySystem}, + Priority: types.PriorityMedium, + }, + // ── Vector index ── + { + PathPrefix: "pkg/vectorindex/", + UTPackages: []string{"pkg/vectorindex/..."}, + BVTCategories: []types.TestCategory{types.CategoryVector}, + Priority: types.PriorityMedium, + }, + // ── NLP/LLM ── + { + PathPrefix: "pkg/monlp/", + UTPackages: []string{"pkg/monlp/..."}, + BVTCategories: []types.TestCategory{types.CategoryFulltext}, + Priority: types.PriorityMedium, + }, + // ── Stage ── + { + PathPrefix: "pkg/stage/", + UTPackages: []string{"pkg/stage/..."}, + BVTCategories: []types.TestCategory{types.CategoryStage}, + Priority: types.PriorityMedium, + }, + // ── BVT test cases themselves ── + { + PathPrefix: "test/distributed/", + UTPackages: nil, + BVTCategories: nil, // handled specially by MatchBVTTestFile + Priority: types.PriorityHigh, + }, + } +} + +// MatchResult holds the test categories and UT packages that a single file +// change maps to. +type MatchResult struct { + UTPackages []string + BVTCategories []types.TestCategory + Priority types.Priority +} + +// Matcher uses PathMappings to resolve which tests a set of file changes +// should trigger. +type Matcher struct { + mappings []PathMapping +} + +// NewMatcher creates a Matcher with the given mappings. +func NewMatcher(mappings []PathMapping) *Matcher { + return &Matcher{mappings: mappings} +} + +// NewDefaultMatcher creates a Matcher using DefaultMappings. +func NewDefaultMatcher() *Matcher { + return NewMatcher(DefaultMappings()) +} + +// Match returns the aggregated MatchResult for a single file path. +func (m *Matcher) Match(filePath string) *MatchResult { + var result MatchResult + result.Priority = types.PriorityLow + + for _, mapping := range m.mappings { + if strings.HasPrefix(filePath, mapping.PathPrefix) { + result.UTPackages = appendUnique(result.UTPackages, mapping.UTPackages) + result.BVTCategories = appendUniqueCategories(result.BVTCategories, mapping.BVTCategories) + if mapping.Priority < result.Priority { + result.Priority = mapping.Priority + } + } + } + + // Special handling for BVT test file changes: infer category from path. + if cat := inferBVTCategory(filePath); cat != "" { + result.BVTCategories = appendUniqueCategories(result.BVTCategories, []types.TestCategory{cat}) + } + + return &result +} + +// MatchAll aggregates match results across multiple file changes. +func (m *Matcher) MatchAll(files []types.FileChange) *MatchResult { + var agg MatchResult + agg.Priority = types.PriorityLow + + for _, f := range files { + r := m.Match(f.Path) + agg.UTPackages = appendUnique(agg.UTPackages, r.UTPackages) + agg.BVTCategories = appendUniqueCategories(agg.BVTCategories, r.BVTCategories) + if r.Priority < agg.Priority { + agg.Priority = r.Priority + } + } + return &agg +} + +// inferBVTCategory attempts to extract a BVT category from a file path +// under test/distributed/cases//... +func inferBVTCategory(path string) types.TestCategory { + const prefix = "test/distributed/cases/" + if !strings.HasPrefix(path, prefix) { + return "" + } + rest := path[len(prefix):] + idx := strings.Index(rest, "/") + if idx <= 0 { + return "" + } + return types.TestCategory(rest[:idx]) +} + +func appendUnique(dst, src []string) []string { + seen := make(map[string]struct{}, len(dst)) + for _, s := range dst { + seen[s] = struct{}{} + } + for _, s := range src { + if _, ok := seen[s]; !ok { + seen[s] = struct{}{} + dst = append(dst, s) + } + } + return dst +} + +func appendUniqueCategories(dst, src []types.TestCategory) []types.TestCategory { + seen := make(map[types.TestCategory]struct{}, len(dst)) + for _, c := range dst { + seen[c] = struct{}{} + } + for _, c := range src { + if _, ok := seen[c]; !ok { + seen[c] = struct{}{} + dst = append(dst, c) + } + } + return dst +} diff --git a/pkg/testinfra/planner/planner.go b/pkg/testinfra/planner/planner.go new file mode 100644 index 0000000000000..f32ed13679d96 --- /dev/null +++ b/pkg/testinfra/planner/planner.go @@ -0,0 +1,126 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package planner + +import ( + "fmt" + "strings" + "time" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// Planner generates a TestPlan from a DiffSummary using the Matcher. +type Planner struct { + matcher *Matcher +} + +// NewPlanner creates a Planner with the default mappings. +func NewPlanner() *Planner { + return &Planner{matcher: NewDefaultMatcher()} +} + +// NewPlannerWithMatcher creates a Planner with a custom Matcher. +func NewPlannerWithMatcher(m *Matcher) *Planner { + return &Planner{matcher: m} +} + +// GeneratePlan produces a TestPlan for the given diff summary. +func (p *Planner) GeneratePlan(diff *types.DiffSummary) *types.TestPlan { + plan := &types.TestPlan{ + ID: fmt.Sprintf("tp-pr%d-%d", diff.PRNumber, time.Now().Unix()), + PRNumber: diff.PRNumber, + BaseBranch: diff.BaseBranch, + HeadBranch: diff.HeadBranch, + CreatedAt: time.Now(), + DiffSummary: *diff, + } + + result := p.matcher.MatchAll(diff.Files) + + taskID := 0 + + // Always run SCA if any Go files changed. + if hasGoFiles(diff.Files) { + taskID++ + plan.Tasks = append(plan.Tasks, types.TestTask{ + ID: fmt.Sprintf("task-%d", taskID), + Type: types.TestTypeSCA, + Priority: types.PriorityCritical, + Status: types.TaskStatusPending, + Reason: "Go source files changed – static analysis required", + }) + } + + // Generate UT tasks for each affected package. + for _, pkg := range result.UTPackages { + taskID++ + plan.Tasks = append(plan.Tasks, types.TestTask{ + ID: fmt.Sprintf("task-%d", taskID), + Type: types.TestTypeUT, + Package: pkg, + Priority: result.Priority, + Status: types.TaskStatusPending, + Reason: fmt.Sprintf("unit tests for affected package %s", pkg), + }) + } + + // Generate BVT tasks for each affected category. + for _, cat := range result.BVTCategories { + taskID++ + plan.Tasks = append(plan.Tasks, types.TestTask{ + ID: fmt.Sprintf("task-%d", taskID), + Type: types.TestTypeBVT, + Category: cat, + Priority: result.Priority, + Status: types.TaskStatusPending, + Reason: fmt.Sprintf("BVT category %s mapped from code changes", string(cat)), + }) + } + + plan.Summary = p.buildSummary(diff, plan) + return plan +} + +// GeneratePlanFromDiff is a convenience function that parses a unified diff +// string and generates a TestPlan in one step. +func (p *Planner) GeneratePlanFromDiff(diffText string, prNumber int, baseBranch, headBranch string) *types.TestPlan { + diff := ParseUnifiedDiff(diffText) + diff.PRNumber = prNumber + diff.BaseBranch = baseBranch + diff.HeadBranch = headBranch + return p.GeneratePlan(diff) +} + +func (p *Planner) buildSummary(diff *types.DiffSummary, plan *types.TestPlan) string { + var b strings.Builder + fmt.Fprintf(&b, "TestPlan for PR #%d (%s → %s)\n", diff.PRNumber, diff.HeadBranch, diff.BaseBranch) + fmt.Fprintf(&b, "Files changed: %d (+%d/-%d)\n", len(diff.Files), diff.TotalAdded, diff.TotalDeleted) + fmt.Fprintf(&b, "Affected packages: %s\n", strings.Join(diff.AffectedPackages(), ", ")) + + byType := plan.TaskCountByType() + fmt.Fprintf(&b, "Tasks: %d total (UT: %d, BVT: %d, SCA: %d)", + len(plan.Tasks), byType[types.TestTypeUT], byType[types.TestTypeBVT], byType[types.TestTypeSCA]) + return b.String() +} + +func hasGoFiles(files []types.FileChange) bool { + for _, f := range files { + if strings.HasSuffix(f.Path, ".go") { + return true + } + } + return false +} diff --git a/pkg/testinfra/planner/planner_test.go b/pkg/testinfra/planner/planner_test.go new file mode 100644 index 0000000000000..e3a1953968191 --- /dev/null +++ b/pkg/testinfra/planner/planner_test.go @@ -0,0 +1,313 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package planner + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// --- Diff parsing tests --- + +const sampleDiff = `diff --git a/pkg/sql/plan/build.go b/pkg/sql/plan/build.go +index abc1234..def5678 100644 +--- a/pkg/sql/plan/build.go ++++ b/pkg/sql/plan/build.go +@@ -10,5 +10,6 @@ func (p *Planner) Build(ctx context.Context) error { ++ // new line +@@ -50,3 +51,4 @@ func (p *Planner) Optimize() { ++ // another line +diff --git a/pkg/vm/engine/disttae/txn.go b/pkg/vm/engine/disttae/txn.go +new file mode 100644 +--- /dev/null ++++ b/pkg/vm/engine/disttae/txn.go +@@ -0,0 +1,20 @@ ++package disttae ++ ++func NewTxn() {} +diff --git a/README.md b/README.md +index aaa..bbb 100644 +--- a/README.md ++++ b/README.md +@@ -1,2 +1,3 @@ ++Updated readme +` + +func TestParseUnifiedDiff(t *testing.T) { + summary := ParseUnifiedDiff(sampleDiff) + require.Len(t, summary.Files, 3) + + // File 1: pkg/sql/plan/build.go + f1 := summary.Files[0] + assert.Equal(t, "pkg/sql/plan/build.go", f1.Path) + assert.Equal(t, "modified", f1.ChangeKind) + assert.Equal(t, "pkg/sql/plan", f1.Package) + assert.Contains(t, f1.Functions, "Build") + assert.Contains(t, f1.Functions, "Optimize") + + // File 2: new file + f2 := summary.Files[1] + assert.Equal(t, "pkg/vm/engine/disttae/txn.go", f2.Path) + assert.Equal(t, "added", f2.ChangeKind) + assert.Equal(t, "pkg/vm/engine/disttae", f2.Package) + + // File 3: non-Go file + f3 := summary.Files[2] + assert.Equal(t, "README.md", f3.Path) + assert.Equal(t, "modified", f3.ChangeKind) + assert.Equal(t, "", f3.Package) + + // Counts + assert.True(t, summary.TotalAdded > 0) +} + +func TestParseUnifiedDiffRenamed(t *testing.T) { + diff := `diff --git a/pkg/old/file.go b/pkg/new/file.go +rename from pkg/old/file.go +rename to pkg/new/file.go +` + summary := ParseUnifiedDiff(diff) + require.Len(t, summary.Files, 1) + assert.Equal(t, "renamed", summary.Files[0].ChangeKind) +} + +func TestParseUnifiedDiffDeleted(t *testing.T) { + diff := `diff --git a/pkg/sql/plan/old.go b/pkg/sql/plan/old.go +deleted file mode 100644 +--- a/pkg/sql/plan/old.go ++++ /dev/null +@@ -1,10 +0,0 @@ +-package plan +` + summary := ParseUnifiedDiff(diff) + require.Len(t, summary.Files, 1) + assert.Equal(t, "deleted", summary.Files[0].ChangeKind) + assert.Equal(t, 1, summary.TotalDeleted) +} + +func TestParseUnifiedDiffEmpty(t *testing.T) { + summary := ParseUnifiedDiff("") + assert.Empty(t, summary.Files) +} + +// --- Matcher tests --- + +func TestMatcherSQLPlan(t *testing.T) { + m := NewDefaultMatcher() + r := m.Match("pkg/sql/plan/build.go") + + assert.Contains(t, r.UTPackages, "pkg/sql/plan/...") + assert.Contains(t, r.BVTCategories, types.CategoryOptimizer) + assert.Contains(t, r.BVTCategories, types.CategoryPlanCache) + assert.Equal(t, types.PriorityHigh, r.Priority) +} + +func TestMatcherDisttae(t *testing.T) { + m := NewDefaultMatcher() + r := m.Match("pkg/vm/engine/disttae/logtail.go") + + assert.Contains(t, r.UTPackages, "pkg/vm/engine/disttae/...") + assert.Contains(t, r.BVTCategories, types.CategoryDisttae) + assert.Contains(t, r.BVTCategories, types.CategoryPessimisticTransaction) + assert.Equal(t, types.PriorityCritical, r.Priority) +} + +func TestMatcherBVTTestFile(t *testing.T) { + m := NewDefaultMatcher() + r := m.Match("test/distributed/cases/optimizer/basic.test") + + assert.Contains(t, r.BVTCategories, types.TestCategory("optimizer")) +} + +func TestMatcherNoMatch(t *testing.T) { + m := NewDefaultMatcher() + r := m.Match("docs/something.md") + + assert.Empty(t, r.UTPackages) + assert.Empty(t, r.BVTCategories) + assert.Equal(t, types.PriorityLow, r.Priority) +} + +func TestMatcherMatchAll(t *testing.T) { + m := NewDefaultMatcher() + files := []types.FileChange{ + {Path: "pkg/sql/plan/build.go"}, + {Path: "pkg/vm/engine/disttae/txn.go"}, + } + r := m.MatchAll(files) + + assert.Contains(t, r.UTPackages, "pkg/sql/plan/...") + assert.Contains(t, r.UTPackages, "pkg/vm/engine/disttae/...") + assert.Contains(t, r.BVTCategories, types.CategoryOptimizer) + assert.Contains(t, r.BVTCategories, types.CategoryDisttae) + // critical wins over high + assert.Equal(t, types.PriorityCritical, r.Priority) +} + +func TestInferBVTCategory(t *testing.T) { + tests := []struct { + path string + want types.TestCategory + }{ + {"test/distributed/cases/optimizer/basic.test", "optimizer"}, + {"test/distributed/cases/ddl/create_table.test", "ddl"}, + {"test/distributed/cases/README.md", ""}, + {"pkg/sql/plan/build.go", ""}, + {"test/distributed/cases/", ""}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, inferBVTCategory(tt.path), "path=%s", tt.path) + } +} + +// --- Planner tests --- + +func TestPlannerGeneratePlan(t *testing.T) { + p := NewPlanner() + diff := &types.DiffSummary{ + PRNumber: 42, + BaseBranch: "main", + HeadBranch: "feature/optimizer", + Files: []types.FileChange{ + {Path: "pkg/sql/plan/build.go", Package: "pkg/sql/plan"}, + {Path: "pkg/sql/plan/optimize.go", Package: "pkg/sql/plan"}, + }, + TotalAdded: 10, + TotalDeleted: 5, + } + + plan := p.GeneratePlan(diff) + + assert.Equal(t, 42, plan.PRNumber) + assert.Equal(t, "main", plan.BaseBranch) + assert.True(t, len(plan.Tasks) > 0) + + // Should have SCA task (Go files changed) + hasSCA := false + for _, task := range plan.Tasks { + if task.Type == types.TestTypeSCA { + hasSCA = true + } + } + assert.True(t, hasSCA, "should have SCA task for Go changes") + + // Should have UT task for pkg/sql/plan + hasUT := false + for _, task := range plan.Tasks { + if task.Type == types.TestTypeUT && task.Package == "pkg/sql/plan/..." { + hasUT = true + } + } + assert.True(t, hasUT, "should have UT task for pkg/sql/plan") + + // Should have BVT optimizer task + hasBVT := false + for _, task := range plan.Tasks { + if task.Type == types.TestTypeBVT && task.Category == types.CategoryOptimizer { + hasBVT = true + } + } + assert.True(t, hasBVT, "should have BVT optimizer task") + + // Summary should be populated + assert.Contains(t, plan.Summary, "PR #42") +} + +func TestPlannerGeneratePlanFromDiff(t *testing.T) { + p := NewPlanner() + plan := p.GeneratePlanFromDiff(sampleDiff, 100, "main", "fix/bug") + + assert.Equal(t, 100, plan.PRNumber) + assert.Equal(t, "main", plan.BaseBranch) + assert.Equal(t, "fix/bug", plan.HeadBranch) + assert.True(t, len(plan.Tasks) > 0) + + // Check tasks include both pkg/sql/plan and pkg/vm/engine/disttae + pkgs := make(map[string]bool) + for _, task := range plan.Tasks { + if task.Type == types.TestTypeUT { + pkgs[task.Package] = true + } + } + assert.True(t, pkgs["pkg/sql/plan/..."]) + assert.True(t, pkgs["pkg/vm/engine/disttae/..."]) +} + +func TestPlannerNoGoFiles(t *testing.T) { + p := NewPlanner() + diff := &types.DiffSummary{ + PRNumber: 99, + BaseBranch: "main", + HeadBranch: "docs/update", + Files: []types.FileChange{ + {Path: "docs/readme.md"}, + {Path: "README.md"}, + }, + } + + plan := p.GeneratePlan(diff) + + // No SCA task for non-Go changes + for _, task := range plan.Tasks { + assert.NotEqual(t, types.TestTypeSCA, task.Type) + } +} + +func TestPlannerCustomMatcher(t *testing.T) { + custom := []PathMapping{ + { + PathPrefix: "custom/", + UTPackages: []string{"custom/..."}, + BVTCategories: []types.TestCategory{"custom_cat"}, + Priority: types.PriorityCritical, + }, + } + m := NewMatcher(custom) + p := NewPlannerWithMatcher(m) + + diff := &types.DiffSummary{ + Files: []types.FileChange{ + {Path: "custom/foo.go"}, + }, + } + + plan := p.GeneratePlan(diff) + found := false + for _, task := range plan.Tasks { + if task.Category == "custom_cat" { + found = true + } + } + assert.True(t, found) +} + +func TestExtractFuncFromHunk(t *testing.T) { + tests := []struct { + line string + want string + }{ + {"@@ -10,5 +10,6 @@ func (p *Planner) Build(ctx context.Context) error {", "Build"}, + {"@@ -10,5 +10,6 @@ func Optimize() {", "Optimize"}, + {"@@ -10,5 +10,6 @@ type Foo struct {", ""}, + {"@@ -10,5 +10,6 @@", ""}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, extractFuncFromHunk(tt.line), "line=%s", tt.line) + } +} diff --git a/pkg/testinfra/types/types.go b/pkg/testinfra/types/types.go new file mode 100644 index 0000000000000..0d47586716ff0 --- /dev/null +++ b/pkg/testinfra/types/types.go @@ -0,0 +1,208 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "encoding/json" + "time" +) + +// Priority represents the execution priority of a test task. +type Priority int + +const ( + PriorityCritical Priority = iota + PriorityHigh + PriorityMedium + PriorityLow +) + +func (p Priority) String() string { + switch p { + case PriorityCritical: + return "critical" + case PriorityHigh: + return "high" + case PriorityMedium: + return "medium" + case PriorityLow: + return "low" + default: + return "unknown" + } +} + +// TestType represents the kind of test to run. +type TestType string + +const ( + TestTypeUT TestType = "unit_test" + TestTypeBVT TestType = "bvt" + TestTypeSCA TestType = "sca" +) + +// TaskStatus represents the execution status of a test task. +type TaskStatus string + +const ( + TaskStatusPending TaskStatus = "pending" + TaskStatusRunning TaskStatus = "running" + TaskStatusPassed TaskStatus = "passed" + TaskStatusFailed TaskStatus = "failed" + TaskStatusSkipped TaskStatus = "skipped" + TaskStatusCancelled TaskStatus = "cancelled" +) + +// TestCategory represents a category of BVT test cases mapped from +// test/distributed/cases/ subdirectories. +type TestCategory string + +// Well-known BVT test categories corresponding to directories under +// test/distributed/cases/. +const ( + CategoryDDL TestCategory = "ddl" + CategoryDML TestCategory = "dml" + CategoryFunction TestCategory = "function" + CategoryExpression TestCategory = "expression" + CategoryJoin TestCategory = "join" + CategorySubquery TestCategory = "subquery" + CategoryOptimizer TestCategory = "optimizer" + CategoryPlanCache TestCategory = "plan_cache" + CategoryDisttae TestCategory = "disttae" + CategoryPessimisticTransaction TestCategory = "pessimistic_transaction" + CategoryOptimistic TestCategory = "optimistic" + CategoryLoadData TestCategory = "load_data" + CategoryDtype TestCategory = "dtype" + CategoryView TestCategory = "view" + CategoryCTE TestCategory = "cte" + CategoryRecursiveCTE TestCategory = "recursive_cte" + CategoryWindow TestCategory = "window" + CategoryUnion TestCategory = "union" + CategoryTable TestCategory = "table" + CategoryDatabase TestCategory = "database" + CategoryForeignKey TestCategory = "foreign_key" + CategorySnapshot TestCategory = "snapshot" + CategoryPITR TestCategory = "pitr" + CategorySequence TestCategory = "sequence" + CategoryProcedure TestCategory = "procedure" + CategoryPrepare TestCategory = "prepare" + CategorySecurity TestCategory = "security" + CategorySystem TestCategory = "system" + CategoryFulltext TestCategory = "fulltext" + CategoryUDF TestCategory = "udf" + CategoryVector TestCategory = "vector" + CategoryArray TestCategory = "array" + CategoryStage TestCategory = "stage" + CategoryHint TestCategory = "hint" + CategoryAutoIncrement TestCategory = "auto_increment" + CategoryCharsetCollation TestCategory = "charset_collation" + CategoryTenant TestCategory = "tenant" + CategoryPlugin TestCategory = "plugin" + CategoryAccessControl TestCategory = "zz_accesscontrol" + CategoryCDC TestCategory = "cdc" + CategorySet TestCategory = "set" + CategorySystemVariable TestCategory = "system_variable" +) + +// FileChange represents a single file changed in a PR diff. +type FileChange struct { + Path string `json:"path"` + ChangeKind string `json:"change_kind"` // added, modified, deleted, renamed + Package string `json:"package"` // Go package path, e.g. "pkg/sql/plan" + Functions []string `json:"functions"` // changed function names (best effort) +} + +// DiffSummary contains the parsed result of a PR's code changes. +type DiffSummary struct { + PRNumber int `json:"pr_number"` + BaseBranch string `json:"base_branch"` + HeadBranch string `json:"head_branch"` + Files []FileChange `json:"files"` + TotalAdded int `json:"total_added"` + TotalDeleted int `json:"total_deleted"` +} + +// AffectedPackages returns the deduplicated set of Go packages affected. +func (d *DiffSummary) AffectedPackages() []string { + seen := make(map[string]struct{}) + var pkgs []string + for _, f := range d.Files { + if f.Package != "" { + if _, ok := seen[f.Package]; !ok { + seen[f.Package] = struct{}{} + pkgs = append(pkgs, f.Package) + } + } + } + return pkgs +} + +// TestTask represents a single executable test task within a TestPlan. +type TestTask struct { + ID string `json:"id"` + Type TestType `json:"type"` + Category TestCategory `json:"category,omitempty"` + Package string `json:"package,omitempty"` // Go package for UT + TestFile string `json:"test_file,omitempty"` // BVT .test file path + Priority Priority `json:"priority"` + EstDuration string `json:"est_duration,omitempty"` + Status TaskStatus `json:"status"` + Reason string `json:"reason"` // why this task is included +} + +// TestPlan is the structured output produced by the planner. It describes +// which tests should be run for a given PR. +type TestPlan struct { + ID string `json:"id"` + PRNumber int `json:"pr_number"` + BaseBranch string `json:"base_branch"` + HeadBranch string `json:"head_branch"` + CreatedAt time.Time `json:"created_at"` + Summary string `json:"summary"` + DiffSummary DiffSummary `json:"diff_summary"` + Tasks []TestTask `json:"tasks"` +} + +// ToJSON serializes the TestPlan to indented JSON. +func (tp *TestPlan) ToJSON() ([]byte, error) { + return json.MarshalIndent(tp, "", " ") +} + +// FromJSON deserializes a TestPlan from JSON bytes. +func FromJSON(data []byte) (*TestPlan, error) { + var tp TestPlan + if err := json.Unmarshal(data, &tp); err != nil { + return nil, err + } + return &tp, nil +} + +// TaskCountByStatus returns a map of status → count for quick summaries. +func (tp *TestPlan) TaskCountByStatus() map[TaskStatus]int { + m := make(map[TaskStatus]int) + for _, t := range tp.Tasks { + m[t.Status]++ + } + return m +} + +// TaskCountByType returns a map of type → count. +func (tp *TestPlan) TaskCountByType() map[TestType]int { + m := make(map[TestType]int) + for _, t := range tp.Tasks { + m[t.Type]++ + } + return m +} diff --git a/pkg/testinfra/types/types_test.go b/pkg/testinfra/types/types_test.go new file mode 100644 index 0000000000000..83f2ce163ae2f --- /dev/null +++ b/pkg/testinfra/types/types_test.go @@ -0,0 +1,128 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPriorityString(t *testing.T) { + tests := []struct { + p Priority + want string + }{ + {PriorityCritical, "critical"}, + {PriorityHigh, "high"}, + {PriorityMedium, "medium"}, + {PriorityLow, "low"}, + {Priority(99), "unknown"}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, tt.p.String()) + } +} + +func TestDiffSummaryAffectedPackages(t *testing.T) { + ds := DiffSummary{ + Files: []FileChange{ + {Path: "pkg/sql/plan/build.go", Package: "pkg/sql/plan"}, + {Path: "pkg/sql/plan/optimize.go", Package: "pkg/sql/plan"}, + {Path: "pkg/vm/engine/disttae/txn.go", Package: "pkg/vm/engine/disttae"}, + {Path: "README.md", Package: ""}, + }, + } + pkgs := ds.AffectedPackages() + assert.Equal(t, 2, len(pkgs)) + assert.Contains(t, pkgs, "pkg/sql/plan") + assert.Contains(t, pkgs, "pkg/vm/engine/disttae") +} + +func TestTestPlanJSON(t *testing.T) { + plan := &TestPlan{ + ID: "tp-001", + PRNumber: 12345, + BaseBranch: "main", + HeadBranch: "feature/test", + CreatedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + Summary: "Test plan for PR #12345", + Tasks: []TestTask{ + { + ID: "task-1", + Type: TestTypeUT, + Package: "pkg/sql/plan", + Priority: PriorityHigh, + Status: TaskStatusPending, + Reason: "pkg/sql/plan modified", + }, + { + ID: "task-2", + Type: TestTypeBVT, + Category: CategoryOptimizer, + Priority: PriorityMedium, + Status: TaskStatusPending, + Reason: "optimizer category mapped from pkg/sql/plan", + }, + }, + } + + data, err := plan.ToJSON() + require.NoError(t, err) + + var parsed map[string]interface{} + err = json.Unmarshal(data, &parsed) + require.NoError(t, err) + assert.Equal(t, "tp-001", parsed["id"]) + assert.Equal(t, float64(12345), parsed["pr_number"]) + + restored, err := FromJSON(data) + require.NoError(t, err) + assert.Equal(t, plan.ID, restored.ID) + assert.Equal(t, plan.PRNumber, restored.PRNumber) + assert.Equal(t, len(plan.Tasks), len(restored.Tasks)) + assert.Equal(t, plan.Tasks[0].Package, restored.Tasks[0].Package) +} + +func TestTestPlanTaskCounts(t *testing.T) { + plan := &TestPlan{ + Tasks: []TestTask{ + {Status: TaskStatusPending, Type: TestTypeUT}, + {Status: TaskStatusPending, Type: TestTypeBVT}, + {Status: TaskStatusRunning, Type: TestTypeUT}, + {Status: TaskStatusPassed, Type: TestTypeBVT}, + {Status: TaskStatusFailed, Type: TestTypeSCA}, + }, + } + + byStatus := plan.TaskCountByStatus() + assert.Equal(t, 2, byStatus[TaskStatusPending]) + assert.Equal(t, 1, byStatus[TaskStatusRunning]) + assert.Equal(t, 1, byStatus[TaskStatusPassed]) + assert.Equal(t, 1, byStatus[TaskStatusFailed]) + + byType := plan.TaskCountByType() + assert.Equal(t, 2, byType[TestTypeUT]) + assert.Equal(t, 2, byType[TestTypeBVT]) + assert.Equal(t, 1, byType[TestTypeSCA]) +} + +func TestFromJSONInvalid(t *testing.T) { + _, err := FromJSON([]byte("not json")) + assert.Error(t, err) +} From 3dcb6374644e978bdc7c25b4543a39cac97f006e Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Wed, 8 Apr 2026 14:01:53 +0800 Subject: [PATCH 02/10] fix: add issues write permission to testplan workflow --- .github/workflows/testplan.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testplan.yaml b/.github/workflows/testplan.yaml index 490c4382ff149..b2ea40a54b938 100644 --- a/.github/workflows/testplan.yaml +++ b/.github/workflows/testplan.yaml @@ -19,6 +19,7 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write + issues: write steps: - name: Checkout code uses: actions/checkout@v4 From b995f2459d6c52205eb4f4d50af1f6345791aa7f Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Wed, 8 Apr 2026 14:05:34 +0800 Subject: [PATCH 03/10] fix: use pull_request_target for fork PR write permissions GitHub restricts GITHUB_TOKEN to read-only for pull_request events from forks. Switch to pull_request_target which runs in the base repo context and has write access. Checkout only the base branch (safe) and fetch the diff via GitHub API instead of git. --- .github/workflows/testplan.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testplan.yaml b/.github/workflows/testplan.yaml index b2ea40a54b938..d132d06565d85 100644 --- a/.github/workflows/testplan.yaml +++ b/.github/workflows/testplan.yaml @@ -1,7 +1,7 @@ name: Generate TestPlan on: - pull_request: + pull_request_target: types: [opened, synchronize, reopened] paths: - '**.go' @@ -21,9 +21,11 @@ jobs: pull-requests: write issues: write steps: - - name: Checkout code + # Checkout the BASE branch (safe — not fork code) + - name: Checkout base branch uses: actions/checkout@v4 with: + ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 0 - name: Set up Go @@ -34,9 +36,12 @@ jobs: - name: Build mo-testplan run: go build -o mo-testplan ./cmd/mo-testplan - - name: Generate diff + - name: Generate diff via GitHub API + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - git diff origin/${{ github.base_ref }}...${{ github.event.pull_request.head.sha }} > /tmp/pr.diff + gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }} \ + -H "Accept: application/vnd.github.v3.diff" > /tmp/pr.diff - name: Generate TestPlan (JSON) run: | From 182c4de84b087155a9898593a48cfda08e956755 Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Wed, 8 Apr 2026 14:13:45 +0800 Subject: [PATCH 04/10] fix: replace fmt.Errorf with moerr to pass static-check --- pkg/testinfra/executor/executor.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/testinfra/executor/executor.go b/pkg/testinfra/executor/executor.go index 94856b87da9d6..4460f14b1333d 100644 --- a/pkg/testinfra/executor/executor.go +++ b/pkg/testinfra/executor/executor.go @@ -26,6 +26,7 @@ import ( "strings" "time" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/testinfra/types" ) @@ -82,7 +83,7 @@ func (e *LocalExecutor) Execute(ctx context.Context, task types.TestTask) (*Resu case types.TestTypeSCA: err = e.executeSCA(ctx, result) default: - err = fmt.Errorf("unknown task type: %s", task.Type) + err = moerr.NewInternalErrorNoCtxf("unknown task type: %s", task.Type) } result.EndedAt = time.Now() @@ -100,7 +101,7 @@ func (e *LocalExecutor) Execute(ctx context.Context, task types.TestTask) (*Resu func (e *LocalExecutor) executeUT(ctx context.Context, task types.TestTask, result *Result) error { if task.Package == "" { - return fmt.Errorf("UT task requires a package") + return moerr.NewInternalErrorNoCtx("UT task requires a package") } args := []string{ @@ -121,7 +122,7 @@ func (e *LocalExecutor) executeUT(ctx context.Context, task types.TestTask, resu func (e *LocalExecutor) executeBVT(ctx context.Context, task types.TestTask, result *Result) error { if task.Category == "" && task.TestFile == "" { - return fmt.Errorf("BVT task requires a category or test_file") + return moerr.NewInternalErrorNoCtx("BVT task requires a category or test_file") } // For BVT, we document the command that would be run. From de7041f1e454347707f3a73771c0a67f98c4a49c Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Wed, 8 Apr 2026 14:35:38 +0800 Subject: [PATCH 05/10] fix: replace fmt.Print with os.Stdout.Write to pass molint --- cmd/mo-testplan/main.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/mo-testplan/main.go b/cmd/mo-testplan/main.go index 2583831e24480..c1a5b55cd487f 100644 --- a/cmd/mo-testplan/main.go +++ b/cmd/mo-testplan/main.go @@ -88,17 +88,18 @@ func main() { fmt.Fprintf(os.Stderr, "Error serializing plan: %v\n", err) os.Exit(1) } - fmt.Println(string(data)) + os.Stdout.Write(data) + os.Stdout.WriteString("\n") case "summary": - fmt.Println(plan.Summary) - fmt.Printf("\nTasks (%d):\n", len(plan.Tasks)) + os.Stdout.WriteString(plan.Summary + "\n") + os.Stdout.WriteString(fmt.Sprintf("\nTasks (%d):\n", len(plan.Tasks))) for _, task := range plan.Tasks { target := string(task.Package) if target == "" { target = string(task.Category) } - fmt.Printf(" [%s] %s %-8s %s (%s)\n", - task.Priority.String(), task.ID, task.Type, target, task.Reason) + os.Stdout.WriteString(fmt.Sprintf(" [%s] %s %-8s %s (%s)\n", + task.Priority.String(), task.ID, task.Type, target, task.Reason)) } default: fmt.Fprintf(os.Stderr, "Unknown format: %s (use json or summary)\n", format) From 002a9e716a615afe990f89bf335271945805aef6 Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Wed, 8 Apr 2026 15:25:50 +0800 Subject: [PATCH 06/10] fix: gofmt alignment and prealloc lint issues --- pkg/testinfra/dedup/dedup_test.go | 6 +++--- pkg/testinfra/executor/executor.go | 2 +- pkg/testinfra/types/types.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/testinfra/dedup/dedup_test.go b/pkg/testinfra/dedup/dedup_test.go index 2d2b2db2e2ea8..b3da18d027e3f 100644 --- a/pkg/testinfra/dedup/dedup_test.go +++ b/pkg/testinfra/dedup/dedup_test.go @@ -94,10 +94,10 @@ func TestDedup(t *testing.T) { f := NewSQLFingerprinter() sqls := []string{ "SELECT * FROM t1 WHERE id = 1;", - "SELECT * FROM t1 WHERE id = 2;", // duplicate of first + "SELECT * FROM t1 WHERE id = 2;", // duplicate of first "INSERT INTO t1 VALUES (1, 'a');", - "INSERT INTO t1 VALUES (2, 'b');", // duplicate of third - "DELETE FROM t1 WHERE id = 1;", // unique + "INSERT INTO t1 VALUES (2, 'b');", // duplicate of third + "DELETE FROM t1 WHERE id = 1;", // unique } result := f.Dedup(sqls) diff --git a/pkg/testinfra/executor/executor.go b/pkg/testinfra/executor/executor.go index 4460f14b1333d..0021a50ba031f 100644 --- a/pkg/testinfra/executor/executor.go +++ b/pkg/testinfra/executor/executor.go @@ -157,7 +157,7 @@ func (e *LocalExecutor) executeSCA(ctx context.Context, result *Result) error { // ExecutePlan runs all tasks in a TestPlan sequentially and returns results. func ExecutePlan(ctx context.Context, executor Executor, plan *types.TestPlan) []*Result { - var results []*Result + results := make([]*Result, 0, len(plan.Tasks)) for i := range plan.Tasks { task := &plan.Tasks[i] task.Status = types.TaskStatusRunning diff --git a/pkg/testinfra/types/types.go b/pkg/testinfra/types/types.go index 0d47586716ff0..8e656f82b36c1 100644 --- a/pkg/testinfra/types/types.go +++ b/pkg/testinfra/types/types.go @@ -154,8 +154,8 @@ type TestTask struct { ID string `json:"id"` Type TestType `json:"type"` Category TestCategory `json:"category,omitempty"` - Package string `json:"package,omitempty"` // Go package for UT - TestFile string `json:"test_file,omitempty"` // BVT .test file path + Package string `json:"package,omitempty"` // Go package for UT + TestFile string `json:"test_file,omitempty"` // BVT .test file path Priority Priority `json:"priority"` EstDuration string `json:"est_duration,omitempty"` Status TaskStatus `json:"status"` From 7e8a1df9f958bd00510ef669b0d95295939736b5 Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Wed, 8 Apr 2026 19:10:39 +0800 Subject: [PATCH 07/10] docs: add V2 design for AI Agent-driven test infrastructure --- pkg/testinfra/DESIGN_V2.md | 355 +++++++++++++++++++++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 pkg/testinfra/DESIGN_V2.md diff --git a/pkg/testinfra/DESIGN_V2.md b/pkg/testinfra/DESIGN_V2.md new file mode 100644 index 0000000000000..1cc27f173230d --- /dev/null +++ b/pkg/testinfra/DESIGN_V2.md @@ -0,0 +1,355 @@ +# AI Agent 驱动的测试基础设施方案（V2） + +## 一、背景与目标 + +### 背景 +MO 的测试体系分布在多个仓库、多种测试类型中。当开发者提交 PR 后，需要人工判断该变更可能影响哪些功能，以及现有的测试是否充分覆盖了这些变更。这个过程依赖经验，容易遗漏。 + +### 核心目标 +**手动选择一个 PR → AI 分析 diff + 阅读 skill 文档（MO 底层实现知识库）→ 判断 6 类测试中缺少哪些 case → 自动提 PR 补充到对应仓库。** + +### 与 V1 方案的区别 + +| | V1（已废弃） | V2（当前） | +|---|------------|---------| +| 触发方式 | PR 自动触发 | **手动选择 PR** | +| 分析依据 | 22 条硬编码路径映射规则 | **AI + skill 文档** | +| 分析工具 | Go 代码 `mo-testplan` | **Copilot Agent** | +| 测试范围 | UT + BVT + SCA | **6 类测试**（BVT/稳定性/chaos/大数据/PITR/snapshot） | +| 输出 | TestPlan JSON | **自动提 PR 补充 case** | +| 目标仓库 | 只有 matrixone | **matrixone + mo-nightly-regression** | + +--- + +## 二、整体架构 + +``` + ┌────────────────────┐ + │ 手动选择 PR 编号 │ + └────────┬───────────┘ + │ + ▼ + ┌────────────────────┐ + │ 获取 diff (vs main)│ + └────────┬───────────┘ + │ + ┌────────────┼────────────┐ + ▼ ▼ ▼ + ┌──────────┐ ┌──────────┐ ┌──────────────┐ + │ diff 内容 │ │skill 文档│ │ 已有 case 库 │ + │ │ │(MO 知识库)│ │ (6类测试) │ + └────┬─────┘ └────┬─────┘ └──────┬───────┘ + │ │ │ + └─────────────┼───────────────┘ + │ + ▼ + ┌──────────────────────┐ + │ Copilot Agent │ + │ │ + │ 1. 理解 diff 改了什么 │ + │ 2. 对照 skill 文档 │ + │ 3. 判断影响哪些功能域 │ + │ 4. 检查 6 类测试覆盖 │ + │ 5. 生成缺失的 case │ + │ 6. 去重 │ + └──────────┬───────────┘ + │ + ┌────────────┼────────────┐ + ▼ ▼ ▼ + ┌──────────┐ ┌──────────────┐ ┌────────────┐ + │ PR → MO │ │ PR → nightly │ │ PR → nightly│ + │ (BVT) │ │ main 分支 │ │ big_data │ + └──────────┘ └──────────────┘ └────────────┘ +``` + +--- + +## 三、6 类测试详解 + +### 3.1 BVT 测试 +- **仓库：** matrixone +- **路径：** `test/distributed/cases/` +- **类别（70+）：** + +``` +analyze, array, auto_increment, benchmark, charset_collation, comment, +cte, database, ddl, distinct, disttae, dml, dtype, expression, fake_pk, +feature_limit, foreign_key, fulltext, function, hint, join, keyword, +load_data, log, metadata, mo_cloud, operator, optimistic, optimizer, +pessimistic_transaction, pg_cast, pitr, plan_cache, plugin, prepare, +procedure, query_result, recursive_cte, replace_statement, result_count, +sample, save_query_result, security, sequence, set, snapshot, sql_inject, +sql_source_type, stage, statement_query_type, subquery, system, +system_variable, table, temporary, tenant, time_window, udf, union, +util, vector, view, window, zz_accesscontrol, ... +``` + +- **case 格式：** `.test` 文件（SQL 语句 + mo-tester 标签） +- **补充方式：** 在对应类别目录下新增/修改 `.test` 文件 → 提 PR 到 matrixone + +### 3.2 稳定性测试 +- **仓库：** mo-nightly-regression (main 分支) +- **Job：** [stability workflow](https://github.com/matrixorigin/mo-nightly-regression/actions/runs/23894864579/workflow) +- **测试项：** + - TPCH — 标准分析型基准测试 + - TPCC — 标准 OLTP 基准测试 + - Sysbench — 高并发压测 + - Fulltext-vector — 全文+向量检索 + - Vector IVF + DML concurrency — 向量索引并发写入 +- **目的：** 测试 MO 在长时间运行下的稳定程度 +- **补充方式：** 提 PR 到 mo-nightly-regression 修改 workflow 或配置 + +### 3.3 Chaos 测试 +- **仓库：** mo-nightly-regression (main 分支) +- **Job：** [chaos workflow](https://github.com/matrixorigin/mo-nightly-regression/actions/runs/24126717463/workflow) +- **配置目录：** `mo-chaos-config/` +- **测试项：** sysbench、tpcc、fulltext +- **故障注入类型：** + - 杀 CN（计算节点） + - 杀 DN/TN（数据节点） + - 杀 LogService + - 其他（网络分区、磁盘故障等） +- **补充方式：** 提 PR 到 mo-nightly-regression 修改 `mo-chaos-config/` 或 workflow + +### 3.4 大数据量测试 +- **仓库：** mo-nightly-regression (**big_data 分支**) +- **路径：** `tools/mo-regression-test/cases/big_data_test/` +- **特点：** 从云上 load 大规模数据，验证大数据量下的正确性和性能 +- **补充方式：** 提 PR 到 mo-nightly-regression 的 big_data 分支 + +### 3.5 PITR 测试 +- **仓库：** mo-nightly-regression (main 分支) +- **Workflow：** [pitr-backup-restore-regression-main.yml](https://github.com/matrixorigin/mo-nightly-regression/actions/workflows/pitr-backup-restore-regression-main.yml) +- **测试内容：** Point-In-Time Recovery 备份恢复功能 +- **补充方式：** 提 PR 增加 PITR 场景 case + +### 3.6 Snapshot 测试 +- **仓库：** mo-nightly-regression (main 分支) +- **Workflow：** [snapshot_backup_restore_main.yml](https://github.com/matrixorigin/mo-nightly-regression/actions/workflows/snapshot_backup_restore_main.yml) +- **测试内容：** Snapshot 备份恢复功能 +- **补充方式：** 提 PR 增加 snapshot 场景 case + +--- + +## 四、Skill 文档（MO 知识库） + +### 4.1 定位 +Skill 文档是 **AI 理解 MO 底层实现的知识来源**。AI 不是靠路径映射规则，而是靠阅读这个文档来理解"改了某段代码意味着什么、可能影响什么"。 + +### 4.2 内容结构（建议） + +``` +docs/ai-skills/ +├── architecture.md # MO 整体架构（CN/TN/LogService/Proxy） +├── storage-engine.md # 存储引擎（TAE、DisttaE、对象存储） +├── transaction.md # 事务模型（乐观/悲观、MVCC、锁服务） +├── sql-engine.md # SQL 引擎（parser → plan → compile → execute） +├── backup-restore.md # 备份恢复（PITR、Snapshot 实现原理） +├── cdc.md # CDC 实现 +├── fulltext-vector.md # 全文检索 + 向量索引 +├── multi-cn.md # 多 CN 架构 + 负载均衡 +├── fileservice.md # FileService 对象存储抽象层 +├── testing-guide.md # 测试体系总览（6 类测试怎么跑） +├── test-env-setup.md # 测试环境配置（docker 启动、多 CN、flush/gc 加速） +└── module-test-mapping.md # 各模块与测试类型的关联 +``` + +### 4.3 关键文档示例：`module-test-mapping.md` + +```markdown +# 模块 → 测试关联 + +## 事务模块 (pkg/txn/, pkg/lockservice/) +- **BVT:** pessimistic_transaction, optimistic +- **稳定性:** tpcc (长事务场景), sysbench (高并发事务) +- **Chaos:** 杀 TN 后事务恢复, 杀 CN 后事务回滚 +- **PITR:** 事务一致性点恢复 + +## 存储引擎 (pkg/vm/engine/tae/, pkg/vm/engine/disttae/) +- **BVT:** disttae +- **稳定性:** tpch (大量扫描), tpcc (频繁写入) +- **Chaos:** 杀 TN 后数据一致性 +- **大数据:** 大规模 load + 查询 +- **Snapshot:** 存储层 snapshot 一致性 + +## SQL 计划 (pkg/sql/plan/) +- **BVT:** optimizer, join, subquery, cte, window, hint, plan_cache +- **稳定性:** tpch (复杂查询计划) +- **大数据:** 大数据量下查询计划选择 + +## CDC (pkg/cdc/) +- **BVT:** cdc +- **稳定性:** 长时间 CDC 同步 +- **Chaos:** 杀 CN/TN 后 CDC 恢复 + +## 备份恢复 (pkg/frontend/ snapshot/pitr 相关) +- **BVT:** pitr, snapshot +- **PITR:** 全量 PITR 测试 +- **Snapshot:** 全量 snapshot 测试 +``` + +### 4.4 维护方式 +- **初始版本：** 根据 MO 架构和代码编写 +- **持续补充：** 随着对 MO 理解加深，不断在 skill 文档中补充新知识 +- **内容几乎固定：** 底层架构不会频繁变动，文档更新频率低 + +--- + +## 五、AI Agent 工作流程 + +### 5.1 输入 +``` +用户指定: PR #24088 +``` + +### 5.2 步骤 + +``` +Step 1: 获取 diff + gh pr diff 24088 --repo matrixorigin/matrixone + +Step 2: 解析变更 + 变更文件: + - pkg/sql/plan/function/func_compare_fix.go (修改了 getAsFloat64Slice, isNumericType) + - pkg/sql/plan/opt_misc.go (修改了 remapHavingClause, remapWindowClause) + - pkg/sql/plan/query_builder.go (修改了 remapAllColRefs) + +Step 3: 阅读 skill 文档 + → 读 sql-engine.md → 了解 plan 模块在 SQL 执行流程中的位置 + → 读 module-test-mapping.md → 了解 pkg/sql/plan/ 关联的测试类型 + → 读 testing-guide.md → 了解各类测试怎么跑、case 格式是什么 + +Step 4: 分析影响 + → 这个改动涉及：类型比较(decimal vs integer) + 窗口函数列引用 remap + → 功能域：SQL 类型系统、窗口函数、HAVING 子句 + +Step 5: 检查已有 case 覆盖情况 + → BVT optimizer/ 目录: 有窗口函数基础 case，但没有 decimal vs integer 混合比较 + → BVT window/ 目录: 有基础窗口函数 case，没有子查询 filter 下推场景 + → 稳定性 tpch: tpch 查询不涉及 decimal/integer 混合比较，不需要补 + → 大数据/chaos/PITR/snapshot: 不直接相关 + +Step 6: 生成缺失 case + → 需要补充 BVT case: + 1. window/ 目录: decimal 列窗口函数 + integer filter + 2. optimizer/ 目录: HAVING 子句列 remap 场景 + → 其他 5 类测试: 不需要补充 + +Step 7: 去重 + → 与 window/ 目录已有 .test 文件比对，确认不重复 + +Step 8: 自动提 PR + → 提 PR 到 matrixone，在 test/distributed/cases/window/ 下新增 case +``` + +### 5.3 输出 + +#### 分析报告（评论到 PR 或输出到终端） +``` +## PR #24088 测试覆盖分析 + +### 变更摘要 +修复 valueDec128Compare 在 decimal 与 integer 类型比较时的 panic。 +涉及模块: SQL 类型比较、窗口函数列引用 remap、HAVING 子句。 + +### 6 类测试覆盖情况 + +| 测试类型 | 覆盖状态 | 说明 | +|---------|---------|------| +| BVT | ⚠️ 需补充 | window/ 缺少 decimal vs integer 混合比较场景 | +| 稳定性 | ✅ 已覆盖 | tpch/tpcc 间接覆盖类型比较 | +| Chaos | ➖ 不相关 | 非故障注入相关改动 | +| 大数据 | ➖ 不相关 | 非数据量相关改动 | +| PITR | ➖ 不相关 | 非备份恢复相关改动 | +| Snapshot | ➖ 不相关 | 非快照相关改动 | + +### 自动补充 +已提 PR #XXXXX 到 matrixone，补充 2 个 BVT case: +- test/distributed/cases/window/decimal_filter.test +- test/distributed/cases/optimizer/having_remap.test +``` + +--- + +## 六、实现路径 + +### Phase 1: Skill 文档编写 +- [ ] 编写 `architecture.md` — MO 整体架构概览 +- [ ] 编写 `sql-engine.md` — SQL 引擎模块详解 +- [ ] 编写 `storage-engine.md` — 存储引擎详解 +- [ ] 编写 `transaction.md` — 事务模型 +- [ ] 编写 `backup-restore.md` — 备份恢复原理 +- [ ] 编写 `testing-guide.md` — 6 类测试操作指南 +- [ ] 编写 `module-test-mapping.md` — 模块→测试关联表 +- [ ] 编写 `test-env-setup.md` — 测试环境配置（docker/多CN/flush/gc） + +### Phase 2: Agent 基础能力 +- [ ] 实现 diff 获取（`gh pr diff`） +- [ ] 实现已有 case 扫描（遍历 `test/distributed/cases/` + mo-nightly-regression） +- [ ] 实现 AI 调用（Copilot API / GitHub Models） +- [ ] 实现 prompt 工程（diff + skill 文档 + 已有 case → 分析 + 生成） + +### Phase 3: 自动提 PR +- [ ] 实现 BVT case 生成 → 提 PR 到 matrixone +- [ ] 实现 nightly case 生成 → 提 PR 到 mo-nightly-regression +- [ ] SQL 去重（复用 V1 的 dedup 模块） + +### Phase 4: 迭代优化 +- [ ] 持续补充 skill 文档 +- [ ] 根据实际效果调优 prompt +- [ ] 支持更多测试类型的 case 生成 + +--- + +## 七、Copilot 集成方式 + +### 方案 A: Copilot Agent（推荐） +在仓库中配置 `.github/copilot-instructions.md` + skill 文件，通过 Copilot Chat 直接交互： + +``` +用户: @workspace 分析 PR #24088 的测试覆盖情况，参考 docs/ai-skills/ 下的文档 + +Copilot: + 1. 获取 diff... + 2. 阅读 skill 文档... + 3. 分析结果: BVT 缺少 window/decimal 场景... + 4. 生成 case... +``` + +### 方案 B: GitHub Actions + LLM API +在 workflow 中调用 Copilot API，全自动执行： + +```yaml +- name: Analyze test coverage + run: | + DIFF=$(gh pr diff $PR_NUMBER) + SKILLS=$(cat docs/ai-skills/*.md) + # 调用 Copilot API + curl -X POST $COPILOT_API \ + -d "{\"messages\": [{\"role\":\"system\", \"content\":\"$SKILLS\"}, + {\"role\":\"user\", \"content\":\"分析以下diff的测试覆盖: $DIFF\"}]}" +``` + +### 方案 C: 本地 CLI + LLM +扩展 `mo-testplan` CLI，加入 LLM 调用： + +```bash +mo-testplan analyze --pr 24088 --skills docs/ai-skills/ --model copilot +``` + +**建议先走方案 A**（Copilot Agent），最轻量，不需要写多少代码，主要工作量在 skill 文档上。验证效果后再考虑 B 或 C 做自动化。 + +--- + +## 八、与 V1 代码的关系 + +| V1 模块 | V2 中的去留 | +|--------|-----------| +| `pkg/testinfra/types/` | ⚠️ 可能需要扩展（增加 6 类测试类型） | +| `pkg/testinfra/planner/diff.go` | ❌ 不需要（直接用 `gh pr diff` 原文喂给 AI） | +| `pkg/testinfra/planner/mapping.go` | ❌ 废弃（用 skill 文档替代静态映射） | +| `pkg/testinfra/planner/planner.go` | ❌ 废弃（AI 直接生成分析结果） | +| `pkg/testinfra/dedup/` | ✅ 保留（生成 case 后仍需去重） | +| `pkg/testinfra/executor/` | ❌ 暂不需要（V2 只管生成 case，不管执行） | +| `cmd/mo-testplan/` | ⚠️ 如果走方案 C 则需要重写 | +| `.github/workflows/testplan.yaml` | ⚠️ 如果走方案 B 则需要重写 | From a890953528d8425e181397c8096db900d9e633fa Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Thu, 9 Apr 2026 11:39:34 +0800 Subject: [PATCH 08/10] docs: add AI skill documents for MO test infrastructure --- docs/ai-skills/architecture.md | 47 +++++++++ docs/ai-skills/backup-restore.md | 58 +++++++++++ docs/ai-skills/cdc.md | 47 +++++++++ docs/ai-skills/fileservice.md | 43 ++++++++ docs/ai-skills/fulltext-vector.md | 44 ++++++++ docs/ai-skills/module-test-mapping.md | 141 ++++++++++++++++++++++++++ docs/ai-skills/multi-cn.md | 46 +++++++++ docs/ai-skills/sql-engine.md | 55 ++++++++++ docs/ai-skills/storage-engine.md | 54 ++++++++++ docs/ai-skills/test-env-setup.md | 76 ++++++++++++++ docs/ai-skills/testing-guide.md | 81 +++++++++++++++ docs/ai-skills/transaction.md | 49 +++++++++ 12 files changed, 741 insertions(+) create mode 100644 docs/ai-skills/architecture.md create mode 100644 docs/ai-skills/backup-restore.md create mode 100644 docs/ai-skills/cdc.md create mode 100644 docs/ai-skills/fileservice.md create mode 100644 docs/ai-skills/fulltext-vector.md create mode 100644 docs/ai-skills/module-test-mapping.md create mode 100644 docs/ai-skills/multi-cn.md create mode 100644 docs/ai-skills/sql-engine.md create mode 100644 docs/ai-skills/storage-engine.md create mode 100644 docs/ai-skills/test-env-setup.md create mode 100644 docs/ai-skills/testing-guide.md create mode 100644 docs/ai-skills/transaction.md diff --git a/docs/ai-skills/architecture.md b/docs/ai-skills/architecture.md new file mode 100644 index 0000000000000..abbffd7090959 --- /dev/null +++ b/docs/ai-skills/architecture.md @@ -0,0 +1,47 @@ +# MO 整体架构 + +## 核心组件 + +MO 采用存算分离架构，由 4 个核心服务组成： + +| 组件 | 包路径 | 职责 | +|------|--------|------| +| **CN (Compute Node)** | `pkg/cnservice/` | SQL 解析、计划生成、编译执行。无状态，可水平扩展 | +| **TN (Transaction Node)** | `pkg/tnservice/` | 事务处理、数据持久化、WAL 管理 | +| **LogService** | `pkg/logservice/` | 分布式日志（基于 Dragonboat/Raft），提供数据一致性保证 | +| **Proxy** | `pkg/proxy/` | 连接路由、负载均衡，将客户端请求分发到不同 CN | + +## 辅助组件 + +| 组件 | 包路径 | 职责 | +|------|--------|------| +| **HAKeeper** | `pkg/hakeeper/` | 集群编排与健康检查，基于 Raft 状态机 | +| **ClusterService** | `pkg/clusterservice/` | 集群拓扑管理、节点发现 | +| **Gossip** | `pkg/gossip/` | 节点间元数据传播（基于 memberlist） | +| **TaskService** | `pkg/taskservice/` | 后台任务调度（CDC、自增 ID 等） | +| **Bootstrap** | `pkg/bootstrap/` | 集群初始化、版本管理 | + +## 启动流程 + +入口：`cmd/mo-service/main.go` → `launch.go` + +``` +启动 → 读配置 → 启动 LogService → 启动 TN → 启动 CN → 启动 Proxy +``` + +配置文件位于 `etc/launch/`： +- `launch.toml` — 集群协调配置 +- `cn.toml` — CN 配置（端口、引擎类型） +- `tn.toml` — TN 配置 +- `log.toml` — LogService 配置 + +## 数据流 + +``` +客户端 → Proxy → CN (SQL解析/计划/编译/执行) + ↕ + DisttaE (分布式引擎) ←→ TN (事务/存储) + ↕ ↕ + FileService LogService + (对象存储) (WAL/Raft) +``` diff --git a/docs/ai-skills/backup-restore.md b/docs/ai-skills/backup-restore.md new file mode 100644 index 0000000000000..df2bfdc8c93b7 --- /dev/null +++ b/docs/ai-skills/backup-restore.md @@ -0,0 +1,58 @@ +# 备份恢复（PITR / Snapshot） + +## 概述 + +MO 支持两种备份恢复机制： +- **PITR (Point-In-Time Recovery)** — 恢复到任意时间点 +- **Snapshot** — 恢复到指定快照 + +## PITR + +### 原理 +- 基于 Logtail 的增量日志回放 +- 记录事务提交时间戳，恢复时回放到指定时间点 +- 依赖 TN 的 checkpoint + WAL + +### 关键路径 +- `pkg/backup/` — 备份配置与元数据 +- `pkg/frontend/` — 前端 PITR SQL 命令处理 +- `pkg/vm/engine/tae/` — 存储层 checkpoint 管理 +- `pkg/logservice/` — WAL 日志持久化 + +### SQL 语法 +```sql +CREATE PITR pitr_name FOR ACCOUNT account_name RANGE value unit; +ALTER PITR pitr_name ...; +DROP PITR pitr_name; +RESTORE ACCOUNT account_name FROM PITR pitr_name TIMESTAMP '2024-01-01 00:00:00'; +``` + +## Snapshot + +### 原理 +- 创建数据库/表的一致性快照 +- 基于 MVCC 时间戳实现 +- 快照元数据存储在系统表中 + +### SQL 语法 +```sql +CREATE SNAPSHOT snapshot_name FOR ACCOUNT account_name; +RESTORE ACCOUNT account_name FROM SNAPSHOT snapshot_name; +DROP SNAPSHOT snapshot_name; +``` + +## Backup 包（`pkg/backup/`） + +- `BackupType` — 备份类型 +- `BackupTs` — 备份时间戳 +- `BackupObject` (objectio) — 备份对象元数据 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| PITR 核心逻辑 | PITR 测试; BVT: pitr | +| Snapshot 核心逻辑 | Snapshot 测试; BVT: snapshot | +| Logtail/Checkpoint | PITR + Snapshot 都受影响 | +| 备份元数据 | BVT: pitr, snapshot | +| 多租户备份 | BVT: tenant; PITR/Snapshot 多租户场景 | diff --git a/docs/ai-skills/cdc.md b/docs/ai-skills/cdc.md new file mode 100644 index 0000000000000..e170622d919ab --- /dev/null +++ b/docs/ai-skills/cdc.md @@ -0,0 +1,47 @@ +# CDC（Change Data Capture） + +## 概述 + +MO 的 CDC 模块基于 Logtail 捕获数据变更，将变更同步到下游系统。 + +## 核心组件（`pkg/cdc/`） + +| 组件 | 职责 | +|------|------| +| `CDCStateManager` | 每个 publication 的状态跟踪 | +| `CDCStatementBuilder` | 为下游生成复制 SQL | +| `WatermarkUpdater` | 同步进度（水位线）管理 | +| Table Scanner | 扫描表变更 | + +## 工作流程 + +``` +表数据变更 → Logtail 捕获 → CDC 模块消费 → 生成下游 SQL → 同步到下游 + ↓ + WatermarkUpdater 更新水位线 +``` + +## 后台任务类型 + +- `JT_CDC_GetOrAddCommittedWM` — 获取/添加已提交水位线 +- `JT_CDC_CommittingWM` — 提交中的水位线 +- `JT_CDC_UpdateWMErrMsg` — 更新水位线错误信息 +- `JT_CDC_RemoveCachedWM` — 清除缓存水位线 + +## SQL 语法 + +```sql +CREATE CDC cdc_name ...; +ALTER CDC cdc_name ...; +DROP CDC cdc_name; +SHOW CDC; +``` + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| CDC 状态管理 | BVT: cdc | +| 水位线逻辑 | BVT: cdc; 稳定性: 长时间 CDC 同步 | +| Logtail 消费 | CDC + PITR + Snapshot | +| 杀节点后 CDC 恢复 | Chaos: 杀 CN/TN 后 CDC 恢复 | diff --git a/docs/ai-skills/fileservice.md b/docs/ai-skills/fileservice.md new file mode 100644 index 0000000000000..a49e3578ef6a7 --- /dev/null +++ b/docs/ai-skills/fileservice.md @@ -0,0 +1,43 @@ +# FileService + +## 概述 + +FileService 是 MO 的对象存储抽象层，屏蔽底层存储差异（本地磁盘 / S3 / MinIO）。 + +## 核心接口（`pkg/fileservice/`） + +| 接口 | 职责 | +|------|------| +| `FileService` | 基础读写操作 | +| `MutableFileService` | 追加/删除操作 | +| `ReaderWriterFileService` | 流式 I/O | +| `CacheDataAllocator` | 内存管理 | +| `FileCache` | 本地缓存层 | +| `ETLFileService` | ETL 专用操作 | + +## 存储后端 + +- **本地文件系统** — 开发/测试环境 +- **S3 / MinIO** — 生产环境对象存储 +- **HTTP** — 远程文件访问 + +## 配置 + +```toml +# etc/launch/tn.toml +[fileservice.s3] +bucket = "my-bucket" +key-prefix = "mo-data" +endpoint = "http://minio:9000" +``` + +本地存储模式：数据存储在 `mo-data/` 目录下。 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| 读写逻辑 | BVT: load_data, stage | +| 缓存层 | 稳定性: tpch（大量读取）| +| S3 对接 | 大数据量测试（云端 load）| +| ETL 功能 | BVT: load_data | diff --git a/docs/ai-skills/fulltext-vector.md b/docs/ai-skills/fulltext-vector.md new file mode 100644 index 0000000000000..8385e16a95264 --- /dev/null +++ b/docs/ai-skills/fulltext-vector.md @@ -0,0 +1,44 @@ +# 全文检索 + 向量索引 + +## 全文检索（`pkg/fulltext/`） + +- **核心类型：** + - `FullTextScoreAlgo` — 相关性评分算法 + - `FullTextParserParam` — 解析配置 + - `FullTextBooleanOperator` — 布尔查询算子（AND/OR/NOT） + +- **SQL 语法：** +```sql +CREATE FULLTEXT INDEX idx ON t(col); +SELECT * FROM t WHERE MATCH(col) AGAINST('keyword' IN BOOLEAN MODE); +``` + +## 向量索引（`pkg/vectorindex/`） + +- **核心类型：** + - `IndexTableConfig` — 索引配置 + - `IvfflatIndexConfig` — IVF-Flat 算法配置 + - `VectorIndexCdc[T]` — 向量索引的 CDC 更新 + - `SearchResultIf` — 搜索结果接口 + +- **支持算法：** IVF-Flat, HNSW + +- **SQL 语法：** +```sql +CREATE INDEX idx USING IVFFLAT ON t(embedding_col) LISTS = 100; +SELECT * FROM t ORDER BY l2_distance(embedding_col, '[1,2,3]') LIMIT 10; +``` + +## 向量化层（`pkg/vectorize/`） + +- 表达式向量化执行 +- 批量向量运算优化 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| 全文检索 | BVT: fulltext; 稳定性: fulltext-vector | +| 向量索引 | BVT: vector; 稳定性: vector IVF+DML concurrency | +| 向量索引 CDC | Chaos: fulltext 故障场景 | +| 评分算法 | BVT: fulltext | diff --git a/docs/ai-skills/module-test-mapping.md b/docs/ai-skills/module-test-mapping.md new file mode 100644 index 0000000000000..be046570419b3 --- /dev/null +++ b/docs/ai-skills/module-test-mapping.md @@ -0,0 +1,141 @@ +# 模块 → 测试关联 + +本文档描述 MO 各代码模块与 6 类测试的关联关系。AI 分析 diff 时，根据变更文件所属模块查找需要关注的测试类型。 + +## SQL 引擎 + +### pkg/sql/plan/ +- **BVT:** optimizer, plan_cache, join, subquery, cte, recursive_cte, hint, window +- **稳定性:** tpch（复杂查询计划） +- **大数据:** 大数据量下查询计划选择 + +### pkg/sql/compile/ +- **BVT:** ddl, dml, function, expression +- **稳定性:** tpch, tpcc + +### pkg/sql/colexec/ +- **BVT:** function, expression, join, window + +### pkg/sql/parsers/ +- **BVT:** ddl, dml, prepare + +## 存储引擎 + +### pkg/vm/engine/disttae/ +- **BVT:** disttae, pessimistic_transaction, optimistic +- **稳定性:** tpcc, sysbench +- **Chaos:** 杀 TN 后数据一致性 +- **Snapshot:** 存储层快照一致性 +- **PITR:** 存储层恢复 + +### pkg/vm/engine/tae/ +- **BVT:** disttae, pessimistic_transaction +- **稳定性:** tpcc（频繁写入 flush） +- **Chaos:** 杀 TN 后恢复 +- **PITR:** checkpoint 恢复 + +### pkg/objectio/ +- **BVT:** load_data, disttae +- **大数据:** 大规模数据读写 + +## 事务 + +### pkg/txn/ +- **BVT:** pessimistic_transaction, optimistic +- **稳定性:** tpcc, sysbench（高并发事务） +- **Chaos:** 杀 TN 后事务恢复 + +### pkg/lockservice/ +- **BVT:** pessimistic_transaction +- **Chaos:** 死锁场景、杀节点后锁恢复 + +## 前端 + +### pkg/frontend/ +- **BVT:** security, tenant, zz_accesscontrol, snapshot, pitr, system_variable, set +- **PITR:** 前端 PITR 命令处理 +- **Snapshot:** 前端 snapshot 命令处理 + +## 数据类型 + +### pkg/container/ +- **BVT:** dtype, array, vector + +## 全文 / 向量 + +### pkg/fulltext/ +- **BVT:** fulltext +- **稳定性:** fulltext-vector + +### pkg/vectorindex/ +- **BVT:** vector +- **稳定性:** vector IVF+DML concurrency +- **Chaos:** fulltext 故障场景 + +## CDC + +### pkg/cdc/ +- **BVT:** cdc +- **稳定性:** 长时间 CDC 同步 +- **Chaos:** 杀 CN/TN 后 CDC 恢复 + +## 备份恢复 + +### pkg/backup/ +- **PITR:** 备份元数据 +- **Snapshot:** 快照管理 + +## 基础设施 + +### pkg/fileservice/ +- **BVT:** stage, load_data +- **大数据:** 云端数据 load + +### pkg/partition/ +- **BVT:** ddl, dml（分区表） + +### pkg/proxy/ +- **BVT:** tenant（路由） +- **Chaos:** 杀 CN 后连接恢复 + +### pkg/bootstrap/ +- **BVT:** system + +### pkg/catalog/ +- **BVT:** ddl, database, table, system + +### pkg/udf/ +- **BVT:** udf + +### pkg/stage/ +- **BVT:** stage + +### pkg/logservice/ +- **Chaos:** 杀 LogService 场景 +- **PITR:** WAL 日志完整性 + +## BVT 测试目录索引 + +``` +test/distributed/cases/ +├── ddl/ # CREATE/ALTER/DROP 语句 +├── dml/ # INSERT/UPDATE/DELETE +├── optimizer/ # 查询优化器 +├── join/ # JOIN 语法和优化 +├── subquery/ # 子查询 +├── window/ # 窗口函数 +├── function/ # 内置函数 +├── expression/ # 表达式计算 +├── pessimistic_transaction/ # 悲观事务 +├── optimistic/ # 乐观事务 +├── disttae/ # 分布式存储引擎 +├── fulltext/ # 全文检索 +├── vector/ # 向量索引 +├── cdc/ # CDC +├── pitr/ # PITR +├── snapshot/ # Snapshot +├── load_data/ # 数据加载 +├── tenant/ # 多租户 +├── security/ # 安全/权限 +└── ... # 其他 50+ 类别 +``` diff --git a/docs/ai-skills/multi-cn.md b/docs/ai-skills/multi-cn.md new file mode 100644 index 0000000000000..7ef810da79f8f --- /dev/null +++ b/docs/ai-skills/multi-cn.md @@ -0,0 +1,46 @@ +# 多 CN 架构 + +## 概述 + +MO 的 CN 是无状态的，可以水平扩展。Proxy 负责将客户端连接路由到合适的 CN。 + +## Proxy（`pkg/proxy/`） + +- `Router` 接口 — CN 选择策略 +- `RefreshableRouter` — 动态刷新集群拓扑 +- 连接路由 + 负载均衡 +- 支持基于标签（label）的路由 + +## ClusterService（`pkg/clusterservice/`） + +- `MOCluster` 接口 — 集群拓扑和元数据 +- `ClusterClient` 接口 — CN 间通信 +- `labelSupportedClient` — 基于标签路由的客户端 + +## QueryService（`pkg/queryservice/`） + +- `QueryService` 接口 — 分布式查询处理 +- `Session` 接口 — 会话管理 +- 节点间消息路由 + +## Gossip（`pkg/gossip/`） + +- 基于 memberlist 的节点发现 +- 节点心跳 + 元数据交换 + +## 多 CN 部署配置 + +参考 `etc/launch-multi-cn/`，每个 CN 独立配置： +- 不同端口 +- 相同的 TN 和 LogService 地址 +- 可配置不同标签用于路由 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| Proxy 路由逻辑 | BVT: tenant（多租户路由）| +| 连接均衡 | 稳定性: sysbench（高并发连接）| +| 杀 CN 恢复 | Chaos: 杀 CN 场景 | +| 节点发现 | Chaos: 网络分区场景 | +| 跨 CN 查询 | 大数据量测试（分布式执行）| diff --git a/docs/ai-skills/sql-engine.md b/docs/ai-skills/sql-engine.md new file mode 100644 index 0000000000000..c5a6d151bac60 --- /dev/null +++ b/docs/ai-skills/sql-engine.md @@ -0,0 +1,55 @@ +# SQL 引擎 + +## 执行流水线 + +``` +SQL 文本 → Parser → AST → Planner → Plan Tree → Compiler → Scope/Pipeline → Executor → 结果 +``` + +## 各阶段详解 + +### Parser（`pkg/sql/parsers/`） +- SQL → AST 转换（基于 bison/flex） +- MySQL 方言兼容 +- 输出：语法树节点 + +### Planner（`pkg/sql/plan/`） +- 逻辑计划生成 +- 类型推导与验证 +- 查询优化（代价估算、谓词下推、连接重排） +- 分区裁剪集成 +- 关键子模块： + - `function/` — 函数注册与类型匹配 + - `opt_misc.go` — 各种优化 pass（HAVING remap、窗口函数 remap 等） + +### Compiler（`pkg/sql/compile/`） +- Plan Tree → 可执行 Pipeline +- 核心类型：`Compile` 结构体 +- `Scope` — 执行单元（本地/远程/Load/表函数） +- 负责跨 CN 的远程执行规划 +- DDL/DML/TCL 翻译 + +### Executor（`pkg/sql/colexec/`） +- 列式执行引擎 +- `ExpressionExecutor` 接口 +- 实现：`FixedVectorExpressionExecutor`, `FunctionExpressionExecutor`, `ColumnExpressionExecutor` +- 批量向量化处理 + +## VM Pipeline（`pkg/vm/`） + +``` +Scope → Operator → Operator → ... → Output +``` + +- 每个 SQL 算子对应一个 Operator（scan, filter, join, agg, sort, limit...） +- Pipeline 模式执行（pull-based） + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| Parser | BVT: ddl, dml, prepare | +| Plan 优化 | BVT: optimizer, hint, plan_cache, join, subquery, cte, window | +| Compile/Scope | BVT: expression, function; 稳定性: tpch | +| 类型系统 | BVT: dtype, pg_cast, array, vector | +| 执行引擎 | 大数据量测试（大规模扫描/聚合） | diff --git a/docs/ai-skills/storage-engine.md b/docs/ai-skills/storage-engine.md new file mode 100644 index 0000000000000..a2d5457353c43 --- /dev/null +++ b/docs/ai-skills/storage-engine.md @@ -0,0 +1,54 @@ +# 存储引擎 + +## 引擎分层 + +``` +CN 侧: DisttaE (分布式事务引擎) + ↕ +TN 侧: TAE (本地存储引擎) + ↕ +底层: FileService (对象存储抽象) +``` + +## DisttaE（分布式 TAE） + +- **路径：** `pkg/vm/engine/disttae/` +- **角色：** CN 侧使用的分布式引擎，负责跨节点事务读写 +- **核心类型：** + - `Engine` — 分布式引擎实例 + - `txnDatabase` — 事务级数据库视图 + - `txnTable` / `txnTableDelegate` — 事务级表操作 +- **特性：** MVCC 快照隔离，支持乐观/悲观事务 + +## TAE（Transaction Aware Engine） + +- **路径：** `pkg/vm/engine/tae/` +- **角色：** TN 侧的本地存储引擎 +- **特性：** + - 块状列存格式 + - Checkpoint / Flush 管理 + - Logtail 增量变更捕获 + +## ObjectIO（对象存储模型） + +- **路径：** `pkg/objectio/` +- **核心类型：** + - `BlockInfo` — 存储块元数据 + - `ObjectLocation` — 对象存储位置引用 + - `BackupObject` — 备份对象元数据 +- **说明：** 通过 FileService 执行实际的对象 I/O + +## Engine 接口 + +- **路径：** `pkg/vm/engine/types.go` +- **抽象：** Engine → Database → Relation(Table) → Reader +- **可插拔：** 支持 DisttaE、MemoryEngine (测试用) 等实现 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| DisttaE 事务逻辑 | BVT: disttae, pessimistic_transaction, optimistic | +| TAE flush/checkpoint | 稳定性: tpcc/sysbench 长时间写入 | +| ObjectIO 读写 | BVT: load_data; 大数据量测试 | +| Logtail 变更捕获 | PITR, Snapshot, CDC | diff --git a/docs/ai-skills/test-env-setup.md b/docs/ai-skills/test-env-setup.md new file mode 100644 index 0000000000000..9b26b8f9d2098 --- /dev/null +++ b/docs/ai-skills/test-env-setup.md @@ -0,0 +1,76 @@ +# 测试环境配置 + +## 本地单机部署 + +```bash +# 编译 +make build + +# 启动（单机集群：LogService + TN + CN） +./mo-service -launch etc/launch/launch.toml +``` + +默认端口：`6001`（MySQL 协议） + +## Docker 部署 + +```bash +# 使用 docker compose +cd etc/launch-tae-compose/ +docker compose up -d +``` + +## 本地多 CN 部署 + +配置目录：`etc/launch-multi-cn/` + +```bash +# 启动集群（1 LogService + 1 TN + 多 CN） +./mo-service -launch etc/launch-multi-cn/launch.toml +``` + +每个 CN 使用不同端口基数，共享同一个 TN 和 LogService。 + +## 加速 Flush / GC（测试调参） + +在 TN 配置中调整： + +```toml +# tn.toml - 加快 flush +[tn.Txn.Storage] +# 减小 checkpoint 间隔 +checkpoint-flush-interval = "1s" +# 减小 GC 间隔 +gc-check-interval = "1s" +``` + +**目的：** 测试时加快数据落盘和垃圾回收，缩短测试等待时间。 + +## BVT 测试运行 + +```bash +# 需要 mo-tester 工具 +cd mo-tester/ +./run.sh -p /path/to/matrixone/test/distributed/cases/optimizer/ + +# 跑单个 .test 文件 +./run.sh -p /path/to/test/distributed/cases/window/window_basic.test +``` + +## 稳定性/Chaos/大数据测试 + +这些测试通过 mo-nightly-regression 仓库的 GitHub Actions Workflow 运行： + +```bash +# 触发方式：手动或定时（nightly） +# 仓库：matrixorigin/mo-nightly-regression +# 分支：main (稳定性/chaos/PITR/snapshot), big_data (大数据量) +``` + +## 环境变量 + +| 变量 | 说明 | +|------|------| +| `MO_WORKSPACE` | MO 工作目录 | +| `MO_LOG_LEVEL` | 日志级别 (debug/info/warn/error) | +| `CGO_CFLAGS` | CGO 编译标志（含 thirdparties 路径）| diff --git a/docs/ai-skills/testing-guide.md b/docs/ai-skills/testing-guide.md new file mode 100644 index 0000000000000..79f079f19d648 --- /dev/null +++ b/docs/ai-skills/testing-guide.md @@ -0,0 +1,81 @@ +# 测试体系总览 + +## 6 类测试 + +### 1. BVT 测试（轻量级回归） +- **仓库：** matrixone +- **路径：** `test/distributed/cases/` +- **工具：** mo-tester +- **Case 格式：** `.test` 文件（SQL + 标签） +- **运行时机：** 每个 PR 的 CI + +### 2. 稳定性测试（长时间运行） +- **仓库：** mo-nightly-regression (main) +- **测试项：** TPCH, TPCC, Sysbench, Fulltext-vector, Vector IVF+DML +- **目的：** 验证长时间运行下的稳定性 +- **运行时机：** Nightly + +### 3. Chaos 测试（故障注入） +- **仓库：** mo-nightly-regression (main) +- **配置：** `mo-chaos-config/` +- **测试项：** Sysbench, TPCC, Fulltext（叠加故障注入） +- **故障类型：** 杀 CN、杀 TN、杀 LogService、网络分区 +- **运行时机：** Nightly + +### 4. 大数据量测试 +- **仓库：** mo-nightly-regression (**big_data 分支**) +- **路径：** `tools/mo-regression-test/cases/big_data_test/` +- **特点：** 从云端 load 大规模数据后执行测试 +- **运行时机：** 定期 + +### 5. PITR 测试 +- **仓库：** mo-nightly-regression (main) +- **Workflow：** `pitr-backup-restore-regression-main.yml` +- **内容：** Point-In-Time Recovery 完整流程验证 +- **运行时机：** Nightly + +### 6. Snapshot 测试 +- **仓库：** mo-nightly-regression (main) +- **Workflow：** `snapshot_backup_restore_main.yml` +- **内容：** Snapshot 备份恢复完整流程验证 +- **运行时机：** Nightly + +## BVT Case 格式 + +`.test` 文件示例： +```sql +-- @bvt:issue#12345 +SELECT 1; +-- @bvt:issue#12345 + +-- @session:id=1 +BEGIN; +INSERT INTO t1 VALUES (1); +-- @session + +-- @sortkey:0,1 +SELECT * FROM t1 ORDER BY id; + +-- @ignore:1 +SELECT NOW(), COUNT(*) FROM t1; +``` + +**常用标签：** +| 标签 | 说明 | +|------|------| +| `@bvt:issue#N` | 跳过指定区块 | +| `@skip:issue#N` | 跳过整个文件 | +| `@session:id=N` | 并发会话 | +| `@wait:session:commit/rollback` | 等待会话提交/回滚 | +| `@sortkey:cols` | 结果排序（消除不确定顺序）| +| `@ignore:cols` | 忽略指定列（如时间戳）| + +## BVT 类别（70+） + +核心 SQL: `dml`, `ddl`, `database`, `table`, `view`, `sequence` +查询: `expression`, `function`, `subquery`, `cte`, `recursive_cte`, `window`, `join` +事务: `pessimistic_transaction`, `optimistic`, `snapshot` +高级功能: `fulltext`, `vector`, `udf`, `procedure` +特性: `partition`, `charset_collation`, `foreign_key`, `temporary`, `tenant` +数据操作: `load_data`, `replace_statement`, `prepare`, `hint` +基础设施: `pitr`, `disttae`, `log`, `metadata`, `stage` diff --git a/docs/ai-skills/transaction.md b/docs/ai-skills/transaction.md new file mode 100644 index 0000000000000..fe893d51bb29e --- /dev/null +++ b/docs/ai-skills/transaction.md @@ -0,0 +1,49 @@ +# 事务模型 + +## 两种事务模式 + +| 模式 | 说明 | 核心包 | +|------|------|--------| +| **悲观事务**（默认） | 写前加锁，冲突立即报错 | `pkg/lockservice/` | +| **乐观事务** | 提交时检测冲突，冲突则回滚 | `pkg/txn/` | + +## 核心组件 + +### TxnClient（`pkg/txn/client/`） +- `TxnClient` 接口 — 创建和管理事务 +- `TxnOperator` 接口 — 单个事务的读写操作 +- `TxnTimestampAware` — MVCC 时间戳管理 + +### LockService（`pkg/lockservice/`） +- `LockService` 接口 — 分布式锁协调 +- `LockTableAllocator` — 按表分配锁表实例 +- `lockTable` 接口 — 行级/表级锁 +- 死锁检测：`lockNode`, `deadlockTxn` + +## MVCC 机制 + +``` +写入 → 分配时间戳 → 写入新版本 → 提交 +读取 → 使用快照时间戳 → 读取可见版本 +``` + +- 快照时间戳由时钟服务提供 +- 行版本存储在 TAE 中 +- 增量变更通过 Logtail 系统传播 + +## 事务状态 + +``` +Active → Committing → Committed + ↘ Aborting → Aborted +``` + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| 悲观事务/锁服务 | BVT: pessimistic_transaction | +| 乐观事务/冲突检测 | BVT: optimistic | +| MVCC/快照隔离 | BVT: snapshot; Snapshot 测试 | +| 事务恢复 | Chaos: 杀 TN 后事务恢复; PITR | +| 高并发事务 | 稳定性: tpcc, sysbench | From d752740d493e5018716c3e50f7e84d8c7f502eb3 Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Thu, 9 Apr 2026 14:29:03 +0800 Subject: [PATCH 09/10] chore: remove V1 testinfra implementation --- .github/workflows/testplan.yaml | 142 ---------- cmd/mo-testplan/main.go | 108 ------- pkg/testinfra/DESIGN_V2.md | 355 ------------------------ pkg/testinfra/dedup/dedup.go | 211 -------------- pkg/testinfra/dedup/dedup_test.go | 179 ------------ pkg/testinfra/executor/executor.go | 209 -------------- pkg/testinfra/executor/executor_test.go | 116 -------- pkg/testinfra/planner/diff.go | 142 ---------- pkg/testinfra/planner/mapping.go | 315 --------------------- pkg/testinfra/planner/planner.go | 126 --------- pkg/testinfra/planner/planner_test.go | 313 --------------------- pkg/testinfra/types/types.go | 208 -------------- pkg/testinfra/types/types_test.go | 128 --------- 13 files changed, 2552 deletions(-) delete mode 100644 .github/workflows/testplan.yaml delete mode 100644 cmd/mo-testplan/main.go delete mode 100644 pkg/testinfra/DESIGN_V2.md delete mode 100644 pkg/testinfra/dedup/dedup.go delete mode 100644 pkg/testinfra/dedup/dedup_test.go delete mode 100644 pkg/testinfra/executor/executor.go delete mode 100644 pkg/testinfra/executor/executor_test.go delete mode 100644 pkg/testinfra/planner/diff.go delete mode 100644 pkg/testinfra/planner/mapping.go delete mode 100644 pkg/testinfra/planner/planner.go delete mode 100644 pkg/testinfra/planner/planner_test.go delete mode 100644 pkg/testinfra/types/types.go delete mode 100644 pkg/testinfra/types/types_test.go diff --git a/.github/workflows/testplan.yaml b/.github/workflows/testplan.yaml deleted file mode 100644 index d132d06565d85..0000000000000 --- a/.github/workflows/testplan.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: Generate TestPlan - -on: - pull_request_target: - types: [opened, synchronize, reopened] - paths: - - '**.go' - - '**.c' - - '**.h' - - 'test/distributed/**' - -concurrency: - group: testplan-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - generate-testplan: - name: Generate TestPlan - runs-on: ubuntu-latest - permissions: - pull-requests: write - issues: write - steps: - # Checkout the BASE branch (safe — not fork code) - - name: Checkout base branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.base.ref }} - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Build mo-testplan - run: go build -o mo-testplan ./cmd/mo-testplan - - - name: Generate diff via GitHub API - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }} \ - -H "Accept: application/vnd.github.v3.diff" > /tmp/pr.diff - - - name: Generate TestPlan (JSON) - run: | - ./mo-testplan \ - --pr ${{ github.event.pull_request.number }} \ - --base ${{ github.base_ref }} \ - --head ${{ github.head_ref }} \ - --diff /tmp/pr.diff \ - --format json > /tmp/testplan.json - - - name: Generate TestPlan (Summary) - id: summary - run: | - SUMMARY=$(./mo-testplan \ - --pr ${{ github.event.pull_request.number }} \ - --base ${{ github.base_ref }} \ - --head ${{ github.head_ref }} \ - --diff /tmp/pr.diff \ - --format summary) - echo "testplan<> $GITHUB_OUTPUT - echo "$SUMMARY" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Upload TestPlan artifact - uses: actions/upload-artifact@v4 - with: - name: testplan-pr${{ github.event.pull_request.number }} - path: /tmp/testplan.json - - - name: Comment on PR - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const testplan = JSON.parse(fs.readFileSync('/tmp/testplan.json', 'utf8')); - const taskCount = testplan.tasks.length; - const utTasks = testplan.tasks.filter(t => t.type === 'unit_test').length; - const bvtTasks = testplan.tasks.filter(t => t.type === 'bvt').length; - const scaTasks = testplan.tasks.filter(t => t.type === 'sca').length; - - const body = `## 🤖 AI TestPlan Generated - - **PR #${testplan.pr_number}** (${testplan.head_branch} → ${testplan.base_branch}) - - | Metric | Count | - |--------|-------| - | Files Changed | ${testplan.diff_summary.files.length} | - | Lines Added | ${testplan.diff_summary.total_added} | - | Lines Deleted | ${testplan.diff_summary.total_deleted} | - | **Total Tasks** | **${taskCount}** | - | Unit Tests | ${utTasks} | - | BVT Tests | ${bvtTasks} | - | Static Analysis | ${scaTasks} | - -
- 📋 Task Details - - \`\`\` - ${process.env.TESTPLAN_SUMMARY || 'See artifact for details'} - \`\`\` - -
- -
- 📦 Affected Packages - - ${[...new Set(testplan.diff_summary.files.filter(f => f.package).map(f => f.package))].map(p => '- `' + p + '`').join('\n')} - -
- - > 💡 Full TestPlan JSON available as workflow artifact. - `; - - // Find existing comment - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - const botComment = comments.find(c => c.body.includes('🤖 AI TestPlan Generated')); - - if (botComment) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: body, - }); - } - env: - TESTPLAN_SUMMARY: ${{ steps.summary.outputs.testplan }} diff --git a/cmd/mo-testplan/main.go b/cmd/mo-testplan/main.go deleted file mode 100644 index c1a5b55cd487f..0000000000000 --- a/cmd/mo-testplan/main.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// mo-testplan is a CLI tool that generates a structured TestPlan from a -// git diff. It is designed to be used in CI pipelines (e.g., GitHub Actions) -// to automatically determine which tests to run for a given PR. -// -// Usage: -// -// # From git diff on stdin: -// git diff origin/main...HEAD | mo-testplan --pr 12345 --base main --head feature/x -// -// # From a diff file: -// mo-testplan --pr 12345 --base main --head feature/x --diff changes.patch -// -// # Output format: -// mo-testplan --pr 12345 --base main --head feature/x --format json -// mo-testplan --pr 12345 --base main --head feature/x --format summary -package main - -import ( - "flag" - "fmt" - "io" - "os" - - "github.com/matrixorigin/matrixone/pkg/testinfra/planner" -) - -func main() { - var ( - prNumber int - baseBranch string - headBranch string - diffFile string - format string - ) - - flag.IntVar(&prNumber, "pr", 0, "PR number") - flag.StringVar(&baseBranch, "base", "main", "base branch name") - flag.StringVar(&headBranch, "head", "", "head branch name") - flag.StringVar(&diffFile, "diff", "", "path to diff file (reads stdin if empty)") - flag.StringVar(&format, "format", "json", "output format: json or summary") - flag.Parse() - - // Read diff - var diffBytes []byte - var err error - if diffFile != "" { - diffBytes, err = os.ReadFile(diffFile) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading diff file: %v\n", err) - os.Exit(1) - } - } else { - diffBytes, err = io.ReadAll(os.Stdin) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading stdin: %v\n", err) - os.Exit(1) - } - } - - if len(diffBytes) == 0 { - fmt.Fprintf(os.Stderr, "No diff provided. Pipe a git diff or use --diff flag.\n") - os.Exit(1) - } - - // Generate plan - p := planner.NewPlanner() - plan := p.GeneratePlanFromDiff(string(diffBytes), prNumber, baseBranch, headBranch) - - // Output - switch format { - case "json": - data, err := plan.ToJSON() - if err != nil { - fmt.Fprintf(os.Stderr, "Error serializing plan: %v\n", err) - os.Exit(1) - } - os.Stdout.Write(data) - os.Stdout.WriteString("\n") - case "summary": - os.Stdout.WriteString(plan.Summary + "\n") - os.Stdout.WriteString(fmt.Sprintf("\nTasks (%d):\n", len(plan.Tasks))) - for _, task := range plan.Tasks { - target := string(task.Package) - if target == "" { - target = string(task.Category) - } - os.Stdout.WriteString(fmt.Sprintf(" [%s] %s %-8s %s (%s)\n", - task.Priority.String(), task.ID, task.Type, target, task.Reason)) - } - default: - fmt.Fprintf(os.Stderr, "Unknown format: %s (use json or summary)\n", format) - os.Exit(1) - } -} diff --git a/pkg/testinfra/DESIGN_V2.md b/pkg/testinfra/DESIGN_V2.md deleted file mode 100644 index 1cc27f173230d..0000000000000 --- a/pkg/testinfra/DESIGN_V2.md +++ /dev/null @@ -1,355 +0,0 @@ -# AI Agent 驱动的测试基础设施方案（V2） - -## 一、背景与目标 - -### 背景 -MO 的测试体系分布在多个仓库、多种测试类型中。当开发者提交 PR 后，需要人工判断该变更可能影响哪些功能，以及现有的测试是否充分覆盖了这些变更。这个过程依赖经验，容易遗漏。 - -### 核心目标 -**手动选择一个 PR → AI 分析 diff + 阅读 skill 文档（MO 底层实现知识库）→ 判断 6 类测试中缺少哪些 case → 自动提 PR 补充到对应仓库。** - -### 与 V1 方案的区别 - -| | V1（已废弃） | V2（当前） | -|---|------------|---------| -| 触发方式 | PR 自动触发 | **手动选择 PR** | -| 分析依据 | 22 条硬编码路径映射规则 | **AI + skill 文档** | -| 分析工具 | Go 代码 `mo-testplan` | **Copilot Agent** | -| 测试范围 | UT + BVT + SCA | **6 类测试**（BVT/稳定性/chaos/大数据/PITR/snapshot） | -| 输出 | TestPlan JSON | **自动提 PR 补充 case** | -| 目标仓库 | 只有 matrixone | **matrixone + mo-nightly-regression** | - ---- - -## 二、整体架构 - -``` - ┌────────────────────┐ - │ 手动选择 PR 编号 │ - └────────┬───────────┘ - │ - ▼ - ┌────────────────────┐ - │ 获取 diff (vs main)│ - └────────┬───────────┘ - │ - ┌────────────┼────────────┐ - ▼ ▼ ▼ - ┌──────────┐ ┌──────────┐ ┌──────────────┐ - │ diff 内容 │ │skill 文档│ │ 已有 case 库 │ - │ │ │(MO 知识库)│ │ (6类测试) │ - └────┬─────┘ └────┬─────┘ └──────┬───────┘ - │ │ │ - └─────────────┼───────────────┘ - │ - ▼ - ┌──────────────────────┐ - │ Copilot Agent │ - │ │ - │ 1. 理解 diff 改了什么 │ - │ 2. 对照 skill 文档 │ - │ 3. 判断影响哪些功能域 │ - │ 4. 检查 6 类测试覆盖 │ - │ 5. 生成缺失的 case │ - │ 6. 去重 │ - └──────────┬───────────┘ - │ - ┌────────────┼────────────┐ - ▼ ▼ ▼ - ┌──────────┐ ┌──────────────┐ ┌────────────┐ - │ PR → MO │ │ PR → nightly │ │ PR → nightly│ - │ (BVT) │ │ main 分支 │ │ big_data │ - └──────────┘ └──────────────┘ └────────────┘ -``` - ---- - -## 三、6 类测试详解 - -### 3.1 BVT 测试 -- **仓库：** matrixone -- **路径：** `test/distributed/cases/` -- **类别（70+）：** - -``` -analyze, array, auto_increment, benchmark, charset_collation, comment, -cte, database, ddl, distinct, disttae, dml, dtype, expression, fake_pk, -feature_limit, foreign_key, fulltext, function, hint, join, keyword, -load_data, log, metadata, mo_cloud, operator, optimistic, optimizer, -pessimistic_transaction, pg_cast, pitr, plan_cache, plugin, prepare, -procedure, query_result, recursive_cte, replace_statement, result_count, -sample, save_query_result, security, sequence, set, snapshot, sql_inject, -sql_source_type, stage, statement_query_type, subquery, system, -system_variable, table, temporary, tenant, time_window, udf, union, -util, vector, view, window, zz_accesscontrol, ... -``` - -- **case 格式：** `.test` 文件（SQL 语句 + mo-tester 标签） -- **补充方式：** 在对应类别目录下新增/修改 `.test` 文件 → 提 PR 到 matrixone - -### 3.2 稳定性测试 -- **仓库：** mo-nightly-regression (main 分支) -- **Job：** [stability workflow](https://github.com/matrixorigin/mo-nightly-regression/actions/runs/23894864579/workflow) -- **测试项：** - - TPCH — 标准分析型基准测试 - - TPCC — 标准 OLTP 基准测试 - - Sysbench — 高并发压测 - - Fulltext-vector — 全文+向量检索 - - Vector IVF + DML concurrency — 向量索引并发写入 -- **目的：** 测试 MO 在长时间运行下的稳定程度 -- **补充方式：** 提 PR 到 mo-nightly-regression 修改 workflow 或配置 - -### 3.3 Chaos 测试 -- **仓库：** mo-nightly-regression (main 分支) -- **Job：** [chaos workflow](https://github.com/matrixorigin/mo-nightly-regression/actions/runs/24126717463/workflow) -- **配置目录：** `mo-chaos-config/` -- **测试项：** sysbench、tpcc、fulltext -- **故障注入类型：** - - 杀 CN（计算节点） - - 杀 DN/TN（数据节点） - - 杀 LogService - - 其他（网络分区、磁盘故障等） -- **补充方式：** 提 PR 到 mo-nightly-regression 修改 `mo-chaos-config/` 或 workflow - -### 3.4 大数据量测试 -- **仓库：** mo-nightly-regression (**big_data 分支**) -- **路径：** `tools/mo-regression-test/cases/big_data_test/` -- **特点：** 从云上 load 大规模数据，验证大数据量下的正确性和性能 -- **补充方式：** 提 PR 到 mo-nightly-regression 的 big_data 分支 - -### 3.5 PITR 测试 -- **仓库：** mo-nightly-regression (main 分支) -- **Workflow：** [pitr-backup-restore-regression-main.yml](https://github.com/matrixorigin/mo-nightly-regression/actions/workflows/pitr-backup-restore-regression-main.yml) -- **测试内容：** Point-In-Time Recovery 备份恢复功能 -- **补充方式：** 提 PR 增加 PITR 场景 case - -### 3.6 Snapshot 测试 -- **仓库：** mo-nightly-regression (main 分支) -- **Workflow：** [snapshot_backup_restore_main.yml](https://github.com/matrixorigin/mo-nightly-regression/actions/workflows/snapshot_backup_restore_main.yml) -- **测试内容：** Snapshot 备份恢复功能 -- **补充方式：** 提 PR 增加 snapshot 场景 case - ---- - -## 四、Skill 文档（MO 知识库） - -### 4.1 定位 -Skill 文档是 **AI 理解 MO 底层实现的知识来源**。AI 不是靠路径映射规则，而是靠阅读这个文档来理解"改了某段代码意味着什么、可能影响什么"。 - -### 4.2 内容结构（建议） - -``` -docs/ai-skills/ -├── architecture.md # MO 整体架构（CN/TN/LogService/Proxy） -├── storage-engine.md # 存储引擎（TAE、DisttaE、对象存储） -├── transaction.md # 事务模型（乐观/悲观、MVCC、锁服务） -├── sql-engine.md # SQL 引擎（parser → plan → compile → execute） -├── backup-restore.md # 备份恢复（PITR、Snapshot 实现原理） -├── cdc.md # CDC 实现 -├── fulltext-vector.md # 全文检索 + 向量索引 -├── multi-cn.md # 多 CN 架构 + 负载均衡 -├── fileservice.md # FileService 对象存储抽象层 -├── testing-guide.md # 测试体系总览（6 类测试怎么跑） -├── test-env-setup.md # 测试环境配置（docker 启动、多 CN、flush/gc 加速） -└── module-test-mapping.md # 各模块与测试类型的关联 -``` - -### 4.3 关键文档示例：`module-test-mapping.md` - -```markdown -# 模块 → 测试关联 - -## 事务模块 (pkg/txn/, pkg/lockservice/) -- **BVT:** pessimistic_transaction, optimistic -- **稳定性:** tpcc (长事务场景), sysbench (高并发事务) -- **Chaos:** 杀 TN 后事务恢复, 杀 CN 后事务回滚 -- **PITR:** 事务一致性点恢复 - -## 存储引擎 (pkg/vm/engine/tae/, pkg/vm/engine/disttae/) -- **BVT:** disttae -- **稳定性:** tpch (大量扫描), tpcc (频繁写入) -- **Chaos:** 杀 TN 后数据一致性 -- **大数据:** 大规模 load + 查询 -- **Snapshot:** 存储层 snapshot 一致性 - -## SQL 计划 (pkg/sql/plan/) -- **BVT:** optimizer, join, subquery, cte, window, hint, plan_cache -- **稳定性:** tpch (复杂查询计划) -- **大数据:** 大数据量下查询计划选择 - -## CDC (pkg/cdc/) -- **BVT:** cdc -- **稳定性:** 长时间 CDC 同步 -- **Chaos:** 杀 CN/TN 后 CDC 恢复 - -## 备份恢复 (pkg/frontend/ snapshot/pitr 相关) -- **BVT:** pitr, snapshot -- **PITR:** 全量 PITR 测试 -- **Snapshot:** 全量 snapshot 测试 -``` - -### 4.4 维护方式 -- **初始版本：** 根据 MO 架构和代码编写 -- **持续补充：** 随着对 MO 理解加深，不断在 skill 文档中补充新知识 -- **内容几乎固定：** 底层架构不会频繁变动，文档更新频率低 - ---- - -## 五、AI Agent 工作流程 - -### 5.1 输入 -``` -用户指定: PR #24088 -``` - -### 5.2 步骤 - -``` -Step 1: 获取 diff - gh pr diff 24088 --repo matrixorigin/matrixone - -Step 2: 解析变更 - 变更文件: - - pkg/sql/plan/function/func_compare_fix.go (修改了 getAsFloat64Slice, isNumericType) - - pkg/sql/plan/opt_misc.go (修改了 remapHavingClause, remapWindowClause) - - pkg/sql/plan/query_builder.go (修改了 remapAllColRefs) - -Step 3: 阅读 skill 文档 - → 读 sql-engine.md → 了解 plan 模块在 SQL 执行流程中的位置 - → 读 module-test-mapping.md → 了解 pkg/sql/plan/ 关联的测试类型 - → 读 testing-guide.md → 了解各类测试怎么跑、case 格式是什么 - -Step 4: 分析影响 - → 这个改动涉及：类型比较(decimal vs integer) + 窗口函数列引用 remap - → 功能域：SQL 类型系统、窗口函数、HAVING 子句 - -Step 5: 检查已有 case 覆盖情况 - → BVT optimizer/ 目录: 有窗口函数基础 case，但没有 decimal vs integer 混合比较 - → BVT window/ 目录: 有基础窗口函数 case，没有子查询 filter 下推场景 - → 稳定性 tpch: tpch 查询不涉及 decimal/integer 混合比较，不需要补 - → 大数据/chaos/PITR/snapshot: 不直接相关 - -Step 6: 生成缺失 case - → 需要补充 BVT case: - 1. window/ 目录: decimal 列窗口函数 + integer filter - 2. optimizer/ 目录: HAVING 子句列 remap 场景 - → 其他 5 类测试: 不需要补充 - -Step 7: 去重 - → 与 window/ 目录已有 .test 文件比对，确认不重复 - -Step 8: 自动提 PR - → 提 PR 到 matrixone，在 test/distributed/cases/window/ 下新增 case -``` - -### 5.3 输出 - -#### 分析报告（评论到 PR 或输出到终端） -``` -## PR #24088 测试覆盖分析 - -### 变更摘要 -修复 valueDec128Compare 在 decimal 与 integer 类型比较时的 panic。 -涉及模块: SQL 类型比较、窗口函数列引用 remap、HAVING 子句。 - -### 6 类测试覆盖情况 - -| 测试类型 | 覆盖状态 | 说明 | -|---------|---------|------| -| BVT | ⚠️ 需补充 | window/ 缺少 decimal vs integer 混合比较场景 | -| 稳定性 | ✅ 已覆盖 | tpch/tpcc 间接覆盖类型比较 | -| Chaos | ➖ 不相关 | 非故障注入相关改动 | -| 大数据 | ➖ 不相关 | 非数据量相关改动 | -| PITR | ➖ 不相关 | 非备份恢复相关改动 | -| Snapshot | ➖ 不相关 | 非快照相关改动 | - -### 自动补充 -已提 PR #XXXXX 到 matrixone，补充 2 个 BVT case: -- test/distributed/cases/window/decimal_filter.test -- test/distributed/cases/optimizer/having_remap.test -``` - ---- - -## 六、实现路径 - -### Phase 1: Skill 文档编写 -- [ ] 编写 `architecture.md` — MO 整体架构概览 -- [ ] 编写 `sql-engine.md` — SQL 引擎模块详解 -- [ ] 编写 `storage-engine.md` — 存储引擎详解 -- [ ] 编写 `transaction.md` — 事务模型 -- [ ] 编写 `backup-restore.md` — 备份恢复原理 -- [ ] 编写 `testing-guide.md` — 6 类测试操作指南 -- [ ] 编写 `module-test-mapping.md` — 模块→测试关联表 -- [ ] 编写 `test-env-setup.md` — 测试环境配置（docker/多CN/flush/gc） - -### Phase 2: Agent 基础能力 -- [ ] 实现 diff 获取（`gh pr diff`） -- [ ] 实现已有 case 扫描（遍历 `test/distributed/cases/` + mo-nightly-regression） -- [ ] 实现 AI 调用（Copilot API / GitHub Models） -- [ ] 实现 prompt 工程（diff + skill 文档 + 已有 case → 分析 + 生成） - -### Phase 3: 自动提 PR -- [ ] 实现 BVT case 生成 → 提 PR 到 matrixone -- [ ] 实现 nightly case 生成 → 提 PR 到 mo-nightly-regression -- [ ] SQL 去重（复用 V1 的 dedup 模块） - -### Phase 4: 迭代优化 -- [ ] 持续补充 skill 文档 -- [ ] 根据实际效果调优 prompt -- [ ] 支持更多测试类型的 case 生成 - ---- - -## 七、Copilot 集成方式 - -### 方案 A: Copilot Agent（推荐） -在仓库中配置 `.github/copilot-instructions.md` + skill 文件，通过 Copilot Chat 直接交互： - -``` -用户: @workspace 分析 PR #24088 的测试覆盖情况，参考 docs/ai-skills/ 下的文档 - -Copilot: - 1. 获取 diff... - 2. 阅读 skill 文档... - 3. 分析结果: BVT 缺少 window/decimal 场景... - 4. 生成 case... -``` - -### 方案 B: GitHub Actions + LLM API -在 workflow 中调用 Copilot API，全自动执行： - -```yaml -- name: Analyze test coverage - run: | - DIFF=$(gh pr diff $PR_NUMBER) - SKILLS=$(cat docs/ai-skills/*.md) - # 调用 Copilot API - curl -X POST $COPILOT_API \ - -d "{\"messages\": [{\"role\":\"system\", \"content\":\"$SKILLS\"}, - {\"role\":\"user\", \"content\":\"分析以下diff的测试覆盖: $DIFF\"}]}" -``` - -### 方案 C: 本地 CLI + LLM -扩展 `mo-testplan` CLI，加入 LLM 调用： - -```bash -mo-testplan analyze --pr 24088 --skills docs/ai-skills/ --model copilot -``` - -**建议先走方案 A**（Copilot Agent），最轻量，不需要写多少代码，主要工作量在 skill 文档上。验证效果后再考虑 B 或 C 做自动化。 - ---- - -## 八、与 V1 代码的关系 - -| V1 模块 | V2 中的去留 | -|--------|-----------| -| `pkg/testinfra/types/` | ⚠️ 可能需要扩展（增加 6 类测试类型） | -| `pkg/testinfra/planner/diff.go` | ❌ 不需要（直接用 `gh pr diff` 原文喂给 AI） | -| `pkg/testinfra/planner/mapping.go` | ❌ 废弃（用 skill 文档替代静态映射） | -| `pkg/testinfra/planner/planner.go` | ❌ 废弃（AI 直接生成分析结果） | -| `pkg/testinfra/dedup/` | ✅ 保留（生成 case 后仍需去重） | -| `pkg/testinfra/executor/` | ❌ 暂不需要（V2 只管生成 case，不管执行） | -| `cmd/mo-testplan/` | ⚠️ 如果走方案 C 则需要重写 | -| `.github/workflows/testplan.yaml` | ⚠️ 如果走方案 B 则需要重写 | diff --git a/pkg/testinfra/dedup/dedup.go b/pkg/testinfra/dedup/dedup.go deleted file mode 100644 index 3de49b72d9a0d..0000000000000 --- a/pkg/testinfra/dedup/dedup.go +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package dedup provides SQL test case deduplication based on SQL statement -// fingerprinting. It normalizes SQL statements to produce a canonical -// fingerprint that can be compared for equality, enabling detection of -// duplicate or near-duplicate test cases. -package dedup - -import ( - "crypto/sha256" - "fmt" - "regexp" - "strings" -) - -// Fingerprint represents the normalized hash of a SQL statement. -type Fingerprint string - -// SQLFingerprinter normalizes SQL statements to detect duplicates. -type SQLFingerprinter struct{} - -// NewSQLFingerprinter creates a new fingerprinter. -func NewSQLFingerprinter() *SQLFingerprinter { - return &SQLFingerprinter{} -} - -// Fingerprint normalizes a SQL statement and returns its fingerprint. -func (f *SQLFingerprinter) Fingerprint(sql string) Fingerprint { - normalized := NormalizeSQL(sql) - hash := sha256.Sum256([]byte(normalized)) - return Fingerprint(fmt.Sprintf("%x", hash[:8])) -} - -// NormalizeSQL produces a canonical form of a SQL statement by: -// - converting to lowercase -// - replacing literal numbers with ? -// - replacing quoted strings with ? -// - collapsing whitespace -// - removing trailing semicolons -// - removing comments -func NormalizeSQL(sql string) string { - s := sql - - // Remove single-line comments - s = removeSingleLineComments(s) - - // Remove multi-line comments - s = removeMultiLineComments(s) - - // Lowercase - s = strings.ToLower(s) - - // Replace quoted strings (single and double quotes) - s = replaceQuotedStrings(s) - - // Replace numbers - s = replaceNumbers(s) - - // Collapse whitespace - s = collapseWhitespace(s) - - // Trim - s = strings.TrimSpace(s) - - // Remove trailing semicolons - s = strings.TrimRight(s, ";") - s = strings.TrimSpace(s) - - return s -} - -var singleLineCommentRe = regexp.MustCompile(`--[^\n]*`) - -func removeSingleLineComments(s string) string { - return singleLineCommentRe.ReplaceAllString(s, "") -} - -var multiLineCommentRe = regexp.MustCompile(`/\*.*?\*/`) - -func removeMultiLineComments(s string) string { - return multiLineCommentRe.ReplaceAllString(s, "") -} - -// replaceQuotedStrings replaces 'string' and "string" with ? -func replaceQuotedStrings(s string) string { - var result strings.Builder - i := 0 - for i < len(s) { - if s[i] == '\'' || s[i] == '"' { - quote := s[i] - i++ - for i < len(s) && s[i] != quote { - if s[i] == '\\' { - i++ // skip escaped char - } - i++ - } - if i < len(s) { - i++ // skip closing quote - } - result.WriteByte('?') - } else { - result.WriteByte(s[i]) - i++ - } - } - return result.String() -} - -// numberRe matches standalone numbers (integers and decimals). -var numberRe = regexp.MustCompile(`\b\d+(\.\d+)?\b`) - -func replaceNumbers(s string) string { - return numberRe.ReplaceAllString(s, "?") -} - -var whitespaceRe = regexp.MustCompile(`\s+`) - -func collapseWhitespace(s string) string { - return whitespaceRe.ReplaceAllString(s, " ") -} - -// DedupResult describes the result of deduplication. -type DedupResult struct { - // Unique are SQL statements that have no duplicates. - Unique []string - // Duplicates maps a fingerprint to the group of duplicate SQLs. - Duplicates map[Fingerprint][]string -} - -// Dedup takes a list of SQL statements and identifies duplicates. -func (f *SQLFingerprinter) Dedup(sqls []string) *DedupResult { - groups := make(map[Fingerprint][]string) - for _, sql := range sqls { - fp := f.Fingerprint(sql) - groups[fp] = append(groups[fp], sql) - } - - result := &DedupResult{ - Duplicates: make(map[Fingerprint][]string), - } - for fp, group := range groups { - if len(group) == 1 { - result.Unique = append(result.Unique, group[0]) - } else { - result.Duplicates[fp] = group - } - } - return result -} - -// ExtractSQLStatements parses a mo-tester .test file content and extracts -// the SQL statements from it, ignoring comments and tag lines. -func ExtractSQLStatements(content string) []string { - var statements []string - var current strings.Builder - - lines := strings.Split(content, "\n") - for _, line := range lines { - trimmed := strings.TrimSpace(line) - - // Skip empty lines - if trimmed == "" { - continue - } - - // Skip mo-tester tag lines (-- @bvt, -- @skip, etc.) - if strings.HasPrefix(trimmed, "-- @") { - continue - } - - // Skip pure comment lines - if strings.HasPrefix(trimmed, "--") { - continue - } - - current.WriteString(trimmed) - current.WriteString(" ") - - // Statement ends with semicolon - if strings.HasSuffix(trimmed, ";") { - stmt := strings.TrimSpace(current.String()) - if stmt != "" { - statements = append(statements, stmt) - } - current.Reset() - } - } - - // Handle statement without trailing semicolon - if current.Len() > 0 { - stmt := strings.TrimSpace(current.String()) - if stmt != "" { - statements = append(statements, stmt) - } - } - - return statements -} diff --git a/pkg/testinfra/dedup/dedup_test.go b/pkg/testinfra/dedup/dedup_test.go deleted file mode 100644 index b3da18d027e3f..0000000000000 --- a/pkg/testinfra/dedup/dedup_test.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dedup - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNormalizeSQL(t *testing.T) { - tests := []struct { - name string - input string - want string - }{ - { - name: "basic select", - input: "SELECT * FROM t1 WHERE id = 42;", - want: "select * from t1 where id = ?", - }, - { - name: "string literals", - input: "INSERT INTO t1 VALUES ('hello', 'world');", - want: "insert into t1 values (?, ?)", - }, - { - name: "comments removed", - input: "SELECT * FROM t1; -- this is a comment", - want: "select * from t1", - }, - { - name: "multi-line comment", - input: "SELECT /* inline */ * FROM t1;", - want: "select * from t1", - }, - { - name: "whitespace collapsed", - input: "SELECT * FROM t1\n WHERE id = 1;", - want: "select * from t1 where id = ?", - }, - { - name: "case insensitive", - input: "SELECT * FROM T1 WHERE Name = 'Alice';", - want: "select * from t1 where name = ?", - }, - { - name: "decimal numbers", - input: "SELECT * FROM t1 WHERE val > 3.14;", - want: "select * from t1 where val > ?", - }, - { - name: "pure numbers", - input: "SELECT 42 FROM dual;", - want: "select ? from dual", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := NormalizeSQL(tt.input) - assert.Equal(t, tt.want, got) - }) - } -} - -func TestFingerprint(t *testing.T) { - f := NewSQLFingerprinter() - - // Same logical query should produce same fingerprint - fp1 := f.Fingerprint("SELECT * FROM t1 WHERE id = 42;") - fp2 := f.Fingerprint("select * from t1 where id = 100;") - assert.Equal(t, fp1, fp2) - - // Different queries should produce different fingerprints - fp3 := f.Fingerprint("INSERT INTO t1 VALUES (1, 2);") - assert.NotEqual(t, fp1, fp3) -} - -func TestDedup(t *testing.T) { - f := NewSQLFingerprinter() - sqls := []string{ - "SELECT * FROM t1 WHERE id = 1;", - "SELECT * FROM t1 WHERE id = 2;", // duplicate of first - "INSERT INTO t1 VALUES (1, 'a');", - "INSERT INTO t1 VALUES (2, 'b');", // duplicate of third - "DELETE FROM t1 WHERE id = 1;", // unique - } - - result := f.Dedup(sqls) - - assert.Len(t, result.Unique, 1) // only DELETE is unique - assert.Contains(t, result.Unique[0], "DELETE") - assert.Len(t, result.Duplicates, 2) // SELECT group and INSERT group - - // Check that duplicate groups have 2 items each and contain the right statements - for _, group := range result.Duplicates { - assert.Len(t, group, 2) - } - // Verify specific groupings via fingerprint - selectFP := f.Fingerprint(sqls[0]) - insertFP := f.Fingerprint(sqls[2]) - assert.Equal(t, selectFP, f.Fingerprint(sqls[1]), "sqls[0] and sqls[1] should share fingerprint") - assert.Equal(t, insertFP, f.Fingerprint(sqls[3]), "sqls[2] and sqls[3] should share fingerprint") - assert.NotEqual(t, selectFP, insertFP, "SELECT and INSERT fingerprints should differ") -} - -func TestDedupAllUnique(t *testing.T) { - f := NewSQLFingerprinter() - sqls := []string{ - "SELECT * FROM t1;", - "INSERT INTO t1 VALUES (1);", - "DELETE FROM t1 WHERE id = 1;", - } - - result := f.Dedup(sqls) - assert.Len(t, result.Unique, 3) - assert.Empty(t, result.Duplicates) -} - -func TestExtractSQLStatements(t *testing.T) { - content := `-- @bvt:issue#12345 --- This is a comment -CREATE TABLE t1 (a INT, b VARCHAR(100)); -INSERT INTO t1 VALUES (1, 'hello'); -INSERT INTO t1 VALUES (2, 'world'); - --- @sortkey:0 -SELECT * FROM t1 -WHERE a > 0 -ORDER BY a; - -DROP TABLE t1; -` - - stmts := ExtractSQLStatements(content) - require.Len(t, stmts, 5) - assert.Contains(t, stmts[0], "CREATE TABLE") - assert.Contains(t, stmts[1], "INSERT INTO") - assert.Contains(t, stmts[2], "INSERT INTO") - assert.Contains(t, stmts[3], "SELECT * FROM t1") - // Multi-line SELECT should be joined - assert.Contains(t, stmts[3], "WHERE a > 0") - assert.Contains(t, stmts[4], "DROP TABLE") -} - -func TestExtractSQLStatementsEmpty(t *testing.T) { - stmts := ExtractSQLStatements("") - assert.Empty(t, stmts) -} - -func TestExtractSQLStatementsTagsOnly(t *testing.T) { - content := `-- @bvt:issue#999 --- @skip:issue#888 --- Just comments -` - stmts := ExtractSQLStatements(content) - assert.Empty(t, stmts) -} - -func TestExtractSQLStatementsNoSemicolon(t *testing.T) { - content := `SELECT 1` - stmts := ExtractSQLStatements(content) - require.Len(t, stmts, 1) - assert.Equal(t, "SELECT 1", stmts[0]) -} diff --git a/pkg/testinfra/executor/executor.go b/pkg/testinfra/executor/executor.go deleted file mode 100644 index 0021a50ba031f..0000000000000 --- a/pkg/testinfra/executor/executor.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package executor provides interfaces and implementations for executing -// test tasks described by a TestPlan. It wraps the existing optools shell -// scripts and provides a programmatic API for running unit tests, BVT -// tests, and static analysis. -package executor - -import ( - "context" - "fmt" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/matrixorigin/matrixone/pkg/common/moerr" - "github.com/matrixorigin/matrixone/pkg/testinfra/types" -) - -// Result holds the outcome of a single test task execution. -type Result struct { - TaskID string `json:"task_id"` - Status types.TaskStatus `json:"status"` - Output string `json:"output"` - Duration time.Duration `json:"duration"` - Error string `json:"error,omitempty"` - StartedAt time.Time `json:"started_at"` - EndedAt time.Time `json:"ended_at"` -} - -// Executor defines the interface for running test tasks. -type Executor interface { - // Execute runs the given task and returns the result. - Execute(ctx context.Context, task types.TestTask) (*Result, error) -} - -// LocalExecutor runs tests on the local machine by invoking Go test -// commands and BVT scripts. -type LocalExecutor struct { - // RepoRoot is the absolute path to the matrixone repository root. - RepoRoot string - // UTTimeout is the timeout for unit test execution. - UTTimeout time.Duration - // Env holds additional environment variables for test execution. - Env []string -} - -// NewLocalExecutor creates a LocalExecutor with sensible defaults. -func NewLocalExecutor(repoRoot string) *LocalExecutor { - return &LocalExecutor{ - RepoRoot: repoRoot, - UTTimeout: 15 * time.Minute, - } -} - -// Execute runs a single test task. -func (e *LocalExecutor) Execute(ctx context.Context, task types.TestTask) (*Result, error) { - result := &Result{ - TaskID: task.ID, - Status: types.TaskStatusRunning, - StartedAt: time.Now(), - } - - var err error - switch task.Type { - case types.TestTypeUT: - err = e.executeUT(ctx, task, result) - case types.TestTypeBVT: - err = e.executeBVT(ctx, task, result) - case types.TestTypeSCA: - err = e.executeSCA(ctx, result) - default: - err = moerr.NewInternalErrorNoCtxf("unknown task type: %s", task.Type) - } - - result.EndedAt = time.Now() - result.Duration = result.EndedAt.Sub(result.StartedAt) - - if err != nil { - result.Status = types.TaskStatusFailed - result.Error = err.Error() - return result, nil - } - - result.Status = types.TaskStatusPassed - return result, nil -} - -func (e *LocalExecutor) executeUT(ctx context.Context, task types.TestTask, result *Result) error { - if task.Package == "" { - return moerr.NewInternalErrorNoCtx("UT task requires a package") - } - - args := []string{ - "test", "-short", "-count=1", - "-timeout", e.UTTimeout.String(), - "-tags", "matrixone_test", - fmt.Sprintf("./%s", task.Package), - } - - cmd := exec.CommandContext(ctx, "go", args...) - cmd.Dir = e.RepoRoot - cmd.Env = append(cmd.Environ(), e.Env...) - - out, err := cmd.CombinedOutput() - result.Output = string(out) - return err -} - -func (e *LocalExecutor) executeBVT(ctx context.Context, task types.TestTask, result *Result) error { - if task.Category == "" && task.TestFile == "" { - return moerr.NewInternalErrorNoCtx("BVT task requires a category or test_file") - } - - // For BVT, we document the command that would be run. - // Actual BVT execution requires mo-tester and a running MO instance, - // so in this first phase we record the intent. - var target string - if task.TestFile != "" { - target = task.TestFile - } else { - target = filepath.Join("test/distributed/cases", string(task.Category)) - } - - result.Output = fmt.Sprintf("[BVT] Target: %s\nTo execute: mo-tester -p %s", - target, filepath.Join(e.RepoRoot, target)) - return nil -} - -func (e *LocalExecutor) executeSCA(ctx context.Context, result *Result) error { - args := []string{ - "vet", "-tags", "matrixone_test", - "./pkg/...", - } - - cmd := exec.CommandContext(ctx, "go", args...) - cmd.Dir = e.RepoRoot - cmd.Env = append(cmd.Environ(), e.Env...) - - out, err := cmd.CombinedOutput() - result.Output = string(out) - return err -} - -// ExecutePlan runs all tasks in a TestPlan sequentially and returns results. -func ExecutePlan(ctx context.Context, executor Executor, plan *types.TestPlan) []*Result { - results := make([]*Result, 0, len(plan.Tasks)) - for i := range plan.Tasks { - task := &plan.Tasks[i] - task.Status = types.TaskStatusRunning - - r, err := executor.Execute(ctx, *task) - if err != nil { - r = &Result{ - TaskID: task.ID, - Status: types.TaskStatusFailed, - Error: err.Error(), - } - } - task.Status = r.Status - results = append(results, r) - } - return results -} - -// FormatResults produces a human-readable summary of execution results. -func FormatResults(results []*Result) string { - var b strings.Builder - passed, failed, skipped := 0, 0, 0 - for _, r := range results { - switch r.Status { - case types.TaskStatusPassed: - passed++ - case types.TaskStatusFailed: - failed++ - case types.TaskStatusSkipped: - skipped++ - } - } - fmt.Fprintf(&b, "Execution Summary: %d passed, %d failed, %d skipped (total: %d)\n", - passed, failed, skipped, len(results)) - - for _, r := range results { - icon := "✅" - if r.Status == types.TaskStatusFailed { - icon = "❌" - } else if r.Status == types.TaskStatusSkipped { - icon = "⏭️" - } - fmt.Fprintf(&b, " %s %s [%s] %s\n", icon, r.TaskID, r.Status, r.Duration) - if r.Error != "" { - fmt.Fprintf(&b, " Error: %s\n", r.Error) - } - } - return b.String() -} diff --git a/pkg/testinfra/executor/executor_test.go b/pkg/testinfra/executor/executor_test.go deleted file mode 100644 index be64958d8daad..0000000000000 --- a/pkg/testinfra/executor/executor_test.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package executor - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/matrixorigin/matrixone/pkg/testinfra/types" -) - -// mockExecutor is a simple Executor for testing that always succeeds. -type mockExecutor struct{} - -func (m *mockExecutor) Execute(_ context.Context, task types.TestTask) (*Result, error) { - return &Result{ - TaskID: task.ID, - Status: types.TaskStatusPassed, - Output: "mock: ok", - }, nil -} - -func TestExecutePlan(t *testing.T) { - plan := &types.TestPlan{ - Tasks: []types.TestTask{ - {ID: "t1", Type: types.TestTypeUT, Package: "pkg/sql/plan/..."}, - {ID: "t2", Type: types.TestTypeBVT, Category: types.CategoryOptimizer}, - }, - } - results := ExecutePlan(context.Background(), &mockExecutor{}, plan) - - assert.Len(t, results, 2) - for _, r := range results { - assert.Equal(t, types.TaskStatusPassed, r.Status) - } - // Plan tasks should be updated - assert.Equal(t, types.TaskStatusPassed, plan.Tasks[0].Status) - assert.Equal(t, types.TaskStatusPassed, plan.Tasks[1].Status) -} - -func TestFormatResults(t *testing.T) { - results := []*Result{ - {TaskID: "t1", Status: types.TaskStatusPassed}, - {TaskID: "t2", Status: types.TaskStatusFailed, Error: "test failed"}, - {TaskID: "t3", Status: types.TaskStatusSkipped}, - } - output := FormatResults(results) - assert.Contains(t, output, "1 passed") - assert.Contains(t, output, "1 failed") - assert.Contains(t, output, "1 skipped") - assert.Contains(t, output, "t2") - assert.Contains(t, output, "test failed") -} - -func TestLocalExecutorBVT(t *testing.T) { - e := NewLocalExecutor("/tmp/test-repo") - task := types.TestTask{ - ID: "bvt-1", - Type: types.TestTypeBVT, - Category: types.CategoryOptimizer, - } - result, err := e.Execute(context.Background(), task) - assert.NoError(t, err) - assert.Equal(t, types.TaskStatusPassed, result.Status) - assert.Contains(t, result.Output, "optimizer") -} - -func TestLocalExecutorBVTMissingCategory(t *testing.T) { - e := NewLocalExecutor("/tmp/test-repo") - task := types.TestTask{ - ID: "bvt-bad", - Type: types.TestTypeBVT, - } - result, err := e.Execute(context.Background(), task) - assert.NoError(t, err) - assert.Equal(t, types.TaskStatusFailed, result.Status) - assert.Contains(t, result.Error, "requires a category") -} - -func TestLocalExecutorUTMissingPackage(t *testing.T) { - e := NewLocalExecutor("/tmp/test-repo") - task := types.TestTask{ - ID: "ut-bad", - Type: types.TestTypeUT, - } - result, err := e.Execute(context.Background(), task) - assert.NoError(t, err) - assert.Equal(t, types.TaskStatusFailed, result.Status) - assert.Contains(t, result.Error, "requires a package") -} - -func TestLocalExecutorUnknownType(t *testing.T) { - e := NewLocalExecutor("/tmp/test-repo") - task := types.TestTask{ - ID: "unknown", - Type: "foobar", - } - result, err := e.Execute(context.Background(), task) - assert.NoError(t, err) - assert.Equal(t, types.TaskStatusFailed, result.Status) - assert.Contains(t, result.Error, "unknown task type") -} diff --git a/pkg/testinfra/planner/diff.go b/pkg/testinfra/planner/diff.go deleted file mode 100644 index 735a46e6020b0..0000000000000 --- a/pkg/testinfra/planner/diff.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package planner - -import ( - "bufio" - "path" - "regexp" - "strings" - - "github.com/matrixorigin/matrixone/pkg/testinfra/types" -) - -// ParseUnifiedDiff parses a unified diff (as produced by `git diff`) and -// returns a DiffSummary describing the changed files. -// -// It extracts: -// - file paths from "diff --git a/... b/..." lines -// - change kind (added, deleted, modified, renamed) -// - Go package paths (derived from directory) -// - changed function names (best-effort, from @@ hunk headers) -// - total added / deleted line counts -func ParseUnifiedDiff(diffText string) *types.DiffSummary { - summary := &types.DiffSummary{} - - scanner := bufio.NewScanner(strings.NewReader(diffText)) - var currentFile *types.FileChange - - for scanner.Scan() { - line := scanner.Text() - - // --- detect new file in diff --- - if strings.HasPrefix(line, "diff --git ") { - if currentFile != nil { - summary.Files = append(summary.Files, *currentFile) - } - currentFile = parseDiffHeader(line) - continue - } - - if currentFile == nil { - continue - } - - // --- detect change kind --- - if strings.HasPrefix(line, "new file mode") { - currentFile.ChangeKind = "added" - continue - } - if strings.HasPrefix(line, "deleted file mode") { - currentFile.ChangeKind = "deleted" - continue - } - if strings.HasPrefix(line, "rename from ") || strings.HasPrefix(line, "rename to ") { - currentFile.ChangeKind = "renamed" - continue - } - - // --- extract function names from hunk headers --- - if strings.HasPrefix(line, "@@") { - if fn := extractFuncFromHunk(line); fn != "" { - currentFile.Functions = appendIfNew(currentFile.Functions, fn) - } - continue - } - - // --- count added/deleted lines --- - if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "+++") { - summary.TotalAdded++ - } - if strings.HasPrefix(line, "-") && !strings.HasPrefix(line, "---") { - summary.TotalDeleted++ - } - } - - if currentFile != nil { - summary.Files = append(summary.Files, *currentFile) - } - - return summary -} - -// parseDiffHeader parses a "diff --git a/path b/path" line. -func parseDiffHeader(line string) *types.FileChange { - // "diff --git a/pkg/sql/plan/build.go b/pkg/sql/plan/build.go" - parts := strings.SplitN(line, " b/", 2) - if len(parts) != 2 { - return &types.FileChange{Path: line, ChangeKind: "modified"} - } - filePath := parts[1] - fc := &types.FileChange{ - Path: filePath, - ChangeKind: "modified", - } - - // derive Go package from directory - dir := path.Dir(filePath) - if isGoPackage(filePath) && dir != "." { - fc.Package = dir - } - - return fc -} - -// hunkFuncRe matches the function name in a Go diff hunk header like: -// @@ -10,5 +10,6 @@ func (p *Planner) Build(... -var hunkFuncRe = regexp.MustCompile(`@@[^@]+@@\s+(?:func\s+(?:\([^)]+\)\s+)?(\w+))`) - -// extractFuncFromHunk tries to extract a Go function name from a hunk header. -func extractFuncFromHunk(line string) string { - matches := hunkFuncRe.FindStringSubmatch(line) - if len(matches) >= 2 { - return matches[1] - } - return "" -} - -// isGoPackage returns true if the file path looks like a Go source file. -func isGoPackage(filePath string) bool { - return strings.HasSuffix(filePath, ".go") -} - -func appendIfNew(slice []string, s string) []string { - for _, v := range slice { - if v == s { - return slice - } - } - return append(slice, s) -} diff --git a/pkg/testinfra/planner/mapping.go b/pkg/testinfra/planner/mapping.go deleted file mode 100644 index 1c81d1b4b960f..0000000000000 --- a/pkg/testinfra/planner/mapping.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package planner implements the TestPlan generation logic. It analyses PR -// diffs, maps code changes to test categories, and produces a structured -// TestPlan that can be consumed by the executor. -package planner - -import ( - "strings" - - "github.com/matrixorigin/matrixone/pkg/testinfra/types" -) - -// PathMapping defines the static mapping between source code path prefixes -// and the BVT test categories / UT packages they are expected to exercise. -type PathMapping struct { - // PathPrefix is matched against the beginning of a changed file path. - PathPrefix string - // UTPackages lists the Go packages whose unit tests should be run. - UTPackages []string - // BVTCategories lists the BVT categories whose .test files should run. - BVTCategories []types.TestCategory - // Priority is the default priority for tasks generated from this mapping. - Priority types.Priority -} - -// DefaultMappings returns the built-in static mapping table for the -// matrixone repository. This maps source code path prefixes to the test -// categories and UT packages they are likely to affect. -// -// The mapping is intentionally broad – it is better to run a few extra -// tests than to miss a regression. -func DefaultMappings() []PathMapping { - return []PathMapping{ - // ── SQL Plan / Optimizer ── - { - PathPrefix: "pkg/sql/plan/", - UTPackages: []string{"pkg/sql/plan/..."}, - BVTCategories: []types.TestCategory{types.CategoryOptimizer, types.CategoryPlanCache, types.CategoryJoin, types.CategorySubquery, types.CategoryCTE, types.CategoryRecursiveCTE, types.CategoryHint}, - Priority: types.PriorityHigh, - }, - // ── SQL Compile ── - { - PathPrefix: "pkg/sql/compile/", - UTPackages: []string{"pkg/sql/compile/..."}, - BVTCategories: []types.TestCategory{types.CategoryDDL, types.CategoryDML, types.CategoryFunction, types.CategoryExpression}, - Priority: types.PriorityHigh, - }, - // ── Column Executors ── - { - PathPrefix: "pkg/sql/colexec/", - UTPackages: []string{"pkg/sql/colexec/..."}, - BVTCategories: []types.TestCategory{types.CategoryFunction, types.CategoryExpression, types.CategoryJoin, types.CategoryWindow}, - Priority: types.PriorityHigh, - }, - // ── SQL Parsers ── - { - PathPrefix: "pkg/sql/parsers/", - UTPackages: []string{"pkg/sql/parsers/..."}, - BVTCategories: []types.TestCategory{types.CategoryDDL, types.CategoryDML, types.CategoryPrepare}, - Priority: types.PriorityHigh, - }, - // ── Distributed TAE Engine ── - { - PathPrefix: "pkg/vm/engine/disttae/", - UTPackages: []string{"pkg/vm/engine/disttae/..."}, - BVTCategories: []types.TestCategory{types.CategoryDisttae, types.CategoryPessimisticTransaction, types.CategoryOptimistic, types.CategorySnapshot, types.CategoryPITR}, - Priority: types.PriorityCritical, - }, - // ── TAE Engine ── - { - PathPrefix: "pkg/vm/engine/tae/", - UTPackages: []string{"pkg/vm/engine/tae/..."}, - BVTCategories: []types.TestCategory{types.CategoryDisttae, types.CategoryPessimisticTransaction}, - Priority: types.PriorityCritical, - }, - // ── Object IO ── - { - PathPrefix: "pkg/objectio/", - UTPackages: []string{"pkg/objectio/..."}, - BVTCategories: []types.TestCategory{types.CategoryLoadData, types.CategoryDisttae}, - Priority: types.PriorityHigh, - }, - // ── Frontend (session, auth, protocol) ── - { - PathPrefix: "pkg/frontend/", - UTPackages: []string{"pkg/frontend/..."}, - BVTCategories: []types.TestCategory{types.CategorySecurity, types.CategoryTenant, types.CategoryAccessControl, types.CategorySnapshot, types.CategoryPITR, types.CategorySystemVariable, types.CategorySet}, - Priority: types.PriorityHigh, - }, - // ── Container types (vector, batch, bytejson) ── - { - PathPrefix: "pkg/container/", - UTPackages: []string{"pkg/container/..."}, - BVTCategories: []types.TestCategory{types.CategoryDtype, types.CategoryArray, types.CategoryVector}, - Priority: types.PriorityMedium, - }, - // ── Fulltext ── - { - PathPrefix: "pkg/fulltext/", - UTPackages: []string{"pkg/fulltext/...", "pkg/sql/colexec/table_function/..."}, - BVTCategories: []types.TestCategory{types.CategoryFulltext}, - Priority: types.PriorityMedium, - }, - // ── CDC ── - { - PathPrefix: "pkg/cdc/", - UTPackages: []string{"pkg/cdc/..."}, - BVTCategories: []types.TestCategory{types.CategoryCDC}, - Priority: types.PriorityMedium, - }, - // ── UDF ── - { - PathPrefix: "pkg/udf/", - UTPackages: []string{"pkg/udf/..."}, - BVTCategories: []types.TestCategory{types.CategoryUDF}, - Priority: types.PriorityMedium, - }, - // ── Lock service ── - { - PathPrefix: "pkg/lockservice/", - UTPackages: []string{"pkg/lockservice/..."}, - BVTCategories: []types.TestCategory{types.CategoryPessimisticTransaction}, - Priority: types.PriorityHigh, - }, - // ── Transaction ── - { - PathPrefix: "pkg/txn/", - UTPackages: []string{"pkg/txn/..."}, - BVTCategories: []types.TestCategory{types.CategoryPessimisticTransaction, types.CategoryOptimistic}, - Priority: types.PriorityCritical, - }, - // ── Catalog ── - { - PathPrefix: "pkg/catalog/", - UTPackages: []string{"pkg/catalog/..."}, - BVTCategories: []types.TestCategory{types.CategoryDDL, types.CategoryDatabase, types.CategoryTable, types.CategorySystem}, - Priority: types.PriorityHigh, - }, - // ── File service ── - { - PathPrefix: "pkg/fileservice/", - UTPackages: []string{"pkg/fileservice/..."}, - BVTCategories: []types.TestCategory{types.CategoryStage, types.CategoryLoadData}, - Priority: types.PriorityMedium, - }, - // ── Partition ── - { - PathPrefix: "pkg/partition/", - UTPackages: []string{"pkg/partition/...", "pkg/partitionservice/...", "pkg/partitionprune/..."}, - BVTCategories: []types.TestCategory{types.CategoryDDL, types.CategoryDML}, - Priority: types.PriorityMedium, - }, - // ── Proxy ── - { - PathPrefix: "pkg/proxy/", - UTPackages: []string{"pkg/proxy/..."}, - BVTCategories: []types.TestCategory{types.CategoryTenant}, - Priority: types.PriorityLow, - }, - // ── Bootstrap ── - { - PathPrefix: "pkg/bootstrap/", - UTPackages: []string{"pkg/bootstrap/..."}, - BVTCategories: []types.TestCategory{types.CategorySystem}, - Priority: types.PriorityMedium, - }, - // ── Vector index ── - { - PathPrefix: "pkg/vectorindex/", - UTPackages: []string{"pkg/vectorindex/..."}, - BVTCategories: []types.TestCategory{types.CategoryVector}, - Priority: types.PriorityMedium, - }, - // ── NLP/LLM ── - { - PathPrefix: "pkg/monlp/", - UTPackages: []string{"pkg/monlp/..."}, - BVTCategories: []types.TestCategory{types.CategoryFulltext}, - Priority: types.PriorityMedium, - }, - // ── Stage ── - { - PathPrefix: "pkg/stage/", - UTPackages: []string{"pkg/stage/..."}, - BVTCategories: []types.TestCategory{types.CategoryStage}, - Priority: types.PriorityMedium, - }, - // ── BVT test cases themselves ── - { - PathPrefix: "test/distributed/", - UTPackages: nil, - BVTCategories: nil, // handled specially by MatchBVTTestFile - Priority: types.PriorityHigh, - }, - } -} - -// MatchResult holds the test categories and UT packages that a single file -// change maps to. -type MatchResult struct { - UTPackages []string - BVTCategories []types.TestCategory - Priority types.Priority -} - -// Matcher uses PathMappings to resolve which tests a set of file changes -// should trigger. -type Matcher struct { - mappings []PathMapping -} - -// NewMatcher creates a Matcher with the given mappings. -func NewMatcher(mappings []PathMapping) *Matcher { - return &Matcher{mappings: mappings} -} - -// NewDefaultMatcher creates a Matcher using DefaultMappings. -func NewDefaultMatcher() *Matcher { - return NewMatcher(DefaultMappings()) -} - -// Match returns the aggregated MatchResult for a single file path. -func (m *Matcher) Match(filePath string) *MatchResult { - var result MatchResult - result.Priority = types.PriorityLow - - for _, mapping := range m.mappings { - if strings.HasPrefix(filePath, mapping.PathPrefix) { - result.UTPackages = appendUnique(result.UTPackages, mapping.UTPackages) - result.BVTCategories = appendUniqueCategories(result.BVTCategories, mapping.BVTCategories) - if mapping.Priority < result.Priority { - result.Priority = mapping.Priority - } - } - } - - // Special handling for BVT test file changes: infer category from path. - if cat := inferBVTCategory(filePath); cat != "" { - result.BVTCategories = appendUniqueCategories(result.BVTCategories, []types.TestCategory{cat}) - } - - return &result -} - -// MatchAll aggregates match results across multiple file changes. -func (m *Matcher) MatchAll(files []types.FileChange) *MatchResult { - var agg MatchResult - agg.Priority = types.PriorityLow - - for _, f := range files { - r := m.Match(f.Path) - agg.UTPackages = appendUnique(agg.UTPackages, r.UTPackages) - agg.BVTCategories = appendUniqueCategories(agg.BVTCategories, r.BVTCategories) - if r.Priority < agg.Priority { - agg.Priority = r.Priority - } - } - return &agg -} - -// inferBVTCategory attempts to extract a BVT category from a file path -// under test/distributed/cases//... -func inferBVTCategory(path string) types.TestCategory { - const prefix = "test/distributed/cases/" - if !strings.HasPrefix(path, prefix) { - return "" - } - rest := path[len(prefix):] - idx := strings.Index(rest, "/") - if idx <= 0 { - return "" - } - return types.TestCategory(rest[:idx]) -} - -func appendUnique(dst, src []string) []string { - seen := make(map[string]struct{}, len(dst)) - for _, s := range dst { - seen[s] = struct{}{} - } - for _, s := range src { - if _, ok := seen[s]; !ok { - seen[s] = struct{}{} - dst = append(dst, s) - } - } - return dst -} - -func appendUniqueCategories(dst, src []types.TestCategory) []types.TestCategory { - seen := make(map[types.TestCategory]struct{}, len(dst)) - for _, c := range dst { - seen[c] = struct{}{} - } - for _, c := range src { - if _, ok := seen[c]; !ok { - seen[c] = struct{}{} - dst = append(dst, c) - } - } - return dst -} diff --git a/pkg/testinfra/planner/planner.go b/pkg/testinfra/planner/planner.go deleted file mode 100644 index f32ed13679d96..0000000000000 --- a/pkg/testinfra/planner/planner.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package planner - -import ( - "fmt" - "strings" - "time" - - "github.com/matrixorigin/matrixone/pkg/testinfra/types" -) - -// Planner generates a TestPlan from a DiffSummary using the Matcher. -type Planner struct { - matcher *Matcher -} - -// NewPlanner creates a Planner with the default mappings. -func NewPlanner() *Planner { - return &Planner{matcher: NewDefaultMatcher()} -} - -// NewPlannerWithMatcher creates a Planner with a custom Matcher. -func NewPlannerWithMatcher(m *Matcher) *Planner { - return &Planner{matcher: m} -} - -// GeneratePlan produces a TestPlan for the given diff summary. -func (p *Planner) GeneratePlan(diff *types.DiffSummary) *types.TestPlan { - plan := &types.TestPlan{ - ID: fmt.Sprintf("tp-pr%d-%d", diff.PRNumber, time.Now().Unix()), - PRNumber: diff.PRNumber, - BaseBranch: diff.BaseBranch, - HeadBranch: diff.HeadBranch, - CreatedAt: time.Now(), - DiffSummary: *diff, - } - - result := p.matcher.MatchAll(diff.Files) - - taskID := 0 - - // Always run SCA if any Go files changed. - if hasGoFiles(diff.Files) { - taskID++ - plan.Tasks = append(plan.Tasks, types.TestTask{ - ID: fmt.Sprintf("task-%d", taskID), - Type: types.TestTypeSCA, - Priority: types.PriorityCritical, - Status: types.TaskStatusPending, - Reason: "Go source files changed – static analysis required", - }) - } - - // Generate UT tasks for each affected package. - for _, pkg := range result.UTPackages { - taskID++ - plan.Tasks = append(plan.Tasks, types.TestTask{ - ID: fmt.Sprintf("task-%d", taskID), - Type: types.TestTypeUT, - Package: pkg, - Priority: result.Priority, - Status: types.TaskStatusPending, - Reason: fmt.Sprintf("unit tests for affected package %s", pkg), - }) - } - - // Generate BVT tasks for each affected category. - for _, cat := range result.BVTCategories { - taskID++ - plan.Tasks = append(plan.Tasks, types.TestTask{ - ID: fmt.Sprintf("task-%d", taskID), - Type: types.TestTypeBVT, - Category: cat, - Priority: result.Priority, - Status: types.TaskStatusPending, - Reason: fmt.Sprintf("BVT category %s mapped from code changes", string(cat)), - }) - } - - plan.Summary = p.buildSummary(diff, plan) - return plan -} - -// GeneratePlanFromDiff is a convenience function that parses a unified diff -// string and generates a TestPlan in one step. -func (p *Planner) GeneratePlanFromDiff(diffText string, prNumber int, baseBranch, headBranch string) *types.TestPlan { - diff := ParseUnifiedDiff(diffText) - diff.PRNumber = prNumber - diff.BaseBranch = baseBranch - diff.HeadBranch = headBranch - return p.GeneratePlan(diff) -} - -func (p *Planner) buildSummary(diff *types.DiffSummary, plan *types.TestPlan) string { - var b strings.Builder - fmt.Fprintf(&b, "TestPlan for PR #%d (%s → %s)\n", diff.PRNumber, diff.HeadBranch, diff.BaseBranch) - fmt.Fprintf(&b, "Files changed: %d (+%d/-%d)\n", len(diff.Files), diff.TotalAdded, diff.TotalDeleted) - fmt.Fprintf(&b, "Affected packages: %s\n", strings.Join(diff.AffectedPackages(), ", ")) - - byType := plan.TaskCountByType() - fmt.Fprintf(&b, "Tasks: %d total (UT: %d, BVT: %d, SCA: %d)", - len(plan.Tasks), byType[types.TestTypeUT], byType[types.TestTypeBVT], byType[types.TestTypeSCA]) - return b.String() -} - -func hasGoFiles(files []types.FileChange) bool { - for _, f := range files { - if strings.HasSuffix(f.Path, ".go") { - return true - } - } - return false -} diff --git a/pkg/testinfra/planner/planner_test.go b/pkg/testinfra/planner/planner_test.go deleted file mode 100644 index e3a1953968191..0000000000000 --- a/pkg/testinfra/planner/planner_test.go +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package planner - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/matrixorigin/matrixone/pkg/testinfra/types" -) - -// --- Diff parsing tests --- - -const sampleDiff = `diff --git a/pkg/sql/plan/build.go b/pkg/sql/plan/build.go -index abc1234..def5678 100644 ---- a/pkg/sql/plan/build.go -+++ b/pkg/sql/plan/build.go -@@ -10,5 +10,6 @@ func (p *Planner) Build(ctx context.Context) error { -+ // new line -@@ -50,3 +51,4 @@ func (p *Planner) Optimize() { -+ // another line -diff --git a/pkg/vm/engine/disttae/txn.go b/pkg/vm/engine/disttae/txn.go -new file mode 100644 ---- /dev/null -+++ b/pkg/vm/engine/disttae/txn.go -@@ -0,0 +1,20 @@ -+package disttae -+ -+func NewTxn() {} -diff --git a/README.md b/README.md -index aaa..bbb 100644 ---- a/README.md -+++ b/README.md -@@ -1,2 +1,3 @@ -+Updated readme -` - -func TestParseUnifiedDiff(t *testing.T) { - summary := ParseUnifiedDiff(sampleDiff) - require.Len(t, summary.Files, 3) - - // File 1: pkg/sql/plan/build.go - f1 := summary.Files[0] - assert.Equal(t, "pkg/sql/plan/build.go", f1.Path) - assert.Equal(t, "modified", f1.ChangeKind) - assert.Equal(t, "pkg/sql/plan", f1.Package) - assert.Contains(t, f1.Functions, "Build") - assert.Contains(t, f1.Functions, "Optimize") - - // File 2: new file - f2 := summary.Files[1] - assert.Equal(t, "pkg/vm/engine/disttae/txn.go", f2.Path) - assert.Equal(t, "added", f2.ChangeKind) - assert.Equal(t, "pkg/vm/engine/disttae", f2.Package) - - // File 3: non-Go file - f3 := summary.Files[2] - assert.Equal(t, "README.md", f3.Path) - assert.Equal(t, "modified", f3.ChangeKind) - assert.Equal(t, "", f3.Package) - - // Counts - assert.True(t, summary.TotalAdded > 0) -} - -func TestParseUnifiedDiffRenamed(t *testing.T) { - diff := `diff --git a/pkg/old/file.go b/pkg/new/file.go -rename from pkg/old/file.go -rename to pkg/new/file.go -` - summary := ParseUnifiedDiff(diff) - require.Len(t, summary.Files, 1) - assert.Equal(t, "renamed", summary.Files[0].ChangeKind) -} - -func TestParseUnifiedDiffDeleted(t *testing.T) { - diff := `diff --git a/pkg/sql/plan/old.go b/pkg/sql/plan/old.go -deleted file mode 100644 ---- a/pkg/sql/plan/old.go -+++ /dev/null -@@ -1,10 +0,0 @@ --package plan -` - summary := ParseUnifiedDiff(diff) - require.Len(t, summary.Files, 1) - assert.Equal(t, "deleted", summary.Files[0].ChangeKind) - assert.Equal(t, 1, summary.TotalDeleted) -} - -func TestParseUnifiedDiffEmpty(t *testing.T) { - summary := ParseUnifiedDiff("") - assert.Empty(t, summary.Files) -} - -// --- Matcher tests --- - -func TestMatcherSQLPlan(t *testing.T) { - m := NewDefaultMatcher() - r := m.Match("pkg/sql/plan/build.go") - - assert.Contains(t, r.UTPackages, "pkg/sql/plan/...") - assert.Contains(t, r.BVTCategories, types.CategoryOptimizer) - assert.Contains(t, r.BVTCategories, types.CategoryPlanCache) - assert.Equal(t, types.PriorityHigh, r.Priority) -} - -func TestMatcherDisttae(t *testing.T) { - m := NewDefaultMatcher() - r := m.Match("pkg/vm/engine/disttae/logtail.go") - - assert.Contains(t, r.UTPackages, "pkg/vm/engine/disttae/...") - assert.Contains(t, r.BVTCategories, types.CategoryDisttae) - assert.Contains(t, r.BVTCategories, types.CategoryPessimisticTransaction) - assert.Equal(t, types.PriorityCritical, r.Priority) -} - -func TestMatcherBVTTestFile(t *testing.T) { - m := NewDefaultMatcher() - r := m.Match("test/distributed/cases/optimizer/basic.test") - - assert.Contains(t, r.BVTCategories, types.TestCategory("optimizer")) -} - -func TestMatcherNoMatch(t *testing.T) { - m := NewDefaultMatcher() - r := m.Match("docs/something.md") - - assert.Empty(t, r.UTPackages) - assert.Empty(t, r.BVTCategories) - assert.Equal(t, types.PriorityLow, r.Priority) -} - -func TestMatcherMatchAll(t *testing.T) { - m := NewDefaultMatcher() - files := []types.FileChange{ - {Path: "pkg/sql/plan/build.go"}, - {Path: "pkg/vm/engine/disttae/txn.go"}, - } - r := m.MatchAll(files) - - assert.Contains(t, r.UTPackages, "pkg/sql/plan/...") - assert.Contains(t, r.UTPackages, "pkg/vm/engine/disttae/...") - assert.Contains(t, r.BVTCategories, types.CategoryOptimizer) - assert.Contains(t, r.BVTCategories, types.CategoryDisttae) - // critical wins over high - assert.Equal(t, types.PriorityCritical, r.Priority) -} - -func TestInferBVTCategory(t *testing.T) { - tests := []struct { - path string - want types.TestCategory - }{ - {"test/distributed/cases/optimizer/basic.test", "optimizer"}, - {"test/distributed/cases/ddl/create_table.test", "ddl"}, - {"test/distributed/cases/README.md", ""}, - {"pkg/sql/plan/build.go", ""}, - {"test/distributed/cases/", ""}, - } - for _, tt := range tests { - assert.Equal(t, tt.want, inferBVTCategory(tt.path), "path=%s", tt.path) - } -} - -// --- Planner tests --- - -func TestPlannerGeneratePlan(t *testing.T) { - p := NewPlanner() - diff := &types.DiffSummary{ - PRNumber: 42, - BaseBranch: "main", - HeadBranch: "feature/optimizer", - Files: []types.FileChange{ - {Path: "pkg/sql/plan/build.go", Package: "pkg/sql/plan"}, - {Path: "pkg/sql/plan/optimize.go", Package: "pkg/sql/plan"}, - }, - TotalAdded: 10, - TotalDeleted: 5, - } - - plan := p.GeneratePlan(diff) - - assert.Equal(t, 42, plan.PRNumber) - assert.Equal(t, "main", plan.BaseBranch) - assert.True(t, len(plan.Tasks) > 0) - - // Should have SCA task (Go files changed) - hasSCA := false - for _, task := range plan.Tasks { - if task.Type == types.TestTypeSCA { - hasSCA = true - } - } - assert.True(t, hasSCA, "should have SCA task for Go changes") - - // Should have UT task for pkg/sql/plan - hasUT := false - for _, task := range plan.Tasks { - if task.Type == types.TestTypeUT && task.Package == "pkg/sql/plan/..." { - hasUT = true - } - } - assert.True(t, hasUT, "should have UT task for pkg/sql/plan") - - // Should have BVT optimizer task - hasBVT := false - for _, task := range plan.Tasks { - if task.Type == types.TestTypeBVT && task.Category == types.CategoryOptimizer { - hasBVT = true - } - } - assert.True(t, hasBVT, "should have BVT optimizer task") - - // Summary should be populated - assert.Contains(t, plan.Summary, "PR #42") -} - -func TestPlannerGeneratePlanFromDiff(t *testing.T) { - p := NewPlanner() - plan := p.GeneratePlanFromDiff(sampleDiff, 100, "main", "fix/bug") - - assert.Equal(t, 100, plan.PRNumber) - assert.Equal(t, "main", plan.BaseBranch) - assert.Equal(t, "fix/bug", plan.HeadBranch) - assert.True(t, len(plan.Tasks) > 0) - - // Check tasks include both pkg/sql/plan and pkg/vm/engine/disttae - pkgs := make(map[string]bool) - for _, task := range plan.Tasks { - if task.Type == types.TestTypeUT { - pkgs[task.Package] = true - } - } - assert.True(t, pkgs["pkg/sql/plan/..."]) - assert.True(t, pkgs["pkg/vm/engine/disttae/..."]) -} - -func TestPlannerNoGoFiles(t *testing.T) { - p := NewPlanner() - diff := &types.DiffSummary{ - PRNumber: 99, - BaseBranch: "main", - HeadBranch: "docs/update", - Files: []types.FileChange{ - {Path: "docs/readme.md"}, - {Path: "README.md"}, - }, - } - - plan := p.GeneratePlan(diff) - - // No SCA task for non-Go changes - for _, task := range plan.Tasks { - assert.NotEqual(t, types.TestTypeSCA, task.Type) - } -} - -func TestPlannerCustomMatcher(t *testing.T) { - custom := []PathMapping{ - { - PathPrefix: "custom/", - UTPackages: []string{"custom/..."}, - BVTCategories: []types.TestCategory{"custom_cat"}, - Priority: types.PriorityCritical, - }, - } - m := NewMatcher(custom) - p := NewPlannerWithMatcher(m) - - diff := &types.DiffSummary{ - Files: []types.FileChange{ - {Path: "custom/foo.go"}, - }, - } - - plan := p.GeneratePlan(diff) - found := false - for _, task := range plan.Tasks { - if task.Category == "custom_cat" { - found = true - } - } - assert.True(t, found) -} - -func TestExtractFuncFromHunk(t *testing.T) { - tests := []struct { - line string - want string - }{ - {"@@ -10,5 +10,6 @@ func (p *Planner) Build(ctx context.Context) error {", "Build"}, - {"@@ -10,5 +10,6 @@ func Optimize() {", "Optimize"}, - {"@@ -10,5 +10,6 @@ type Foo struct {", ""}, - {"@@ -10,5 +10,6 @@", ""}, - } - for _, tt := range tests { - assert.Equal(t, tt.want, extractFuncFromHunk(tt.line), "line=%s", tt.line) - } -} diff --git a/pkg/testinfra/types/types.go b/pkg/testinfra/types/types.go deleted file mode 100644 index 8e656f82b36c1..0000000000000 --- a/pkg/testinfra/types/types.go +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "encoding/json" - "time" -) - -// Priority represents the execution priority of a test task. -type Priority int - -const ( - PriorityCritical Priority = iota - PriorityHigh - PriorityMedium - PriorityLow -) - -func (p Priority) String() string { - switch p { - case PriorityCritical: - return "critical" - case PriorityHigh: - return "high" - case PriorityMedium: - return "medium" - case PriorityLow: - return "low" - default: - return "unknown" - } -} - -// TestType represents the kind of test to run. -type TestType string - -const ( - TestTypeUT TestType = "unit_test" - TestTypeBVT TestType = "bvt" - TestTypeSCA TestType = "sca" -) - -// TaskStatus represents the execution status of a test task. -type TaskStatus string - -const ( - TaskStatusPending TaskStatus = "pending" - TaskStatusRunning TaskStatus = "running" - TaskStatusPassed TaskStatus = "passed" - TaskStatusFailed TaskStatus = "failed" - TaskStatusSkipped TaskStatus = "skipped" - TaskStatusCancelled TaskStatus = "cancelled" -) - -// TestCategory represents a category of BVT test cases mapped from -// test/distributed/cases/ subdirectories. -type TestCategory string - -// Well-known BVT test categories corresponding to directories under -// test/distributed/cases/. -const ( - CategoryDDL TestCategory = "ddl" - CategoryDML TestCategory = "dml" - CategoryFunction TestCategory = "function" - CategoryExpression TestCategory = "expression" - CategoryJoin TestCategory = "join" - CategorySubquery TestCategory = "subquery" - CategoryOptimizer TestCategory = "optimizer" - CategoryPlanCache TestCategory = "plan_cache" - CategoryDisttae TestCategory = "disttae" - CategoryPessimisticTransaction TestCategory = "pessimistic_transaction" - CategoryOptimistic TestCategory = "optimistic" - CategoryLoadData TestCategory = "load_data" - CategoryDtype TestCategory = "dtype" - CategoryView TestCategory = "view" - CategoryCTE TestCategory = "cte" - CategoryRecursiveCTE TestCategory = "recursive_cte" - CategoryWindow TestCategory = "window" - CategoryUnion TestCategory = "union" - CategoryTable TestCategory = "table" - CategoryDatabase TestCategory = "database" - CategoryForeignKey TestCategory = "foreign_key" - CategorySnapshot TestCategory = "snapshot" - CategoryPITR TestCategory = "pitr" - CategorySequence TestCategory = "sequence" - CategoryProcedure TestCategory = "procedure" - CategoryPrepare TestCategory = "prepare" - CategorySecurity TestCategory = "security" - CategorySystem TestCategory = "system" - CategoryFulltext TestCategory = "fulltext" - CategoryUDF TestCategory = "udf" - CategoryVector TestCategory = "vector" - CategoryArray TestCategory = "array" - CategoryStage TestCategory = "stage" - CategoryHint TestCategory = "hint" - CategoryAutoIncrement TestCategory = "auto_increment" - CategoryCharsetCollation TestCategory = "charset_collation" - CategoryTenant TestCategory = "tenant" - CategoryPlugin TestCategory = "plugin" - CategoryAccessControl TestCategory = "zz_accesscontrol" - CategoryCDC TestCategory = "cdc" - CategorySet TestCategory = "set" - CategorySystemVariable TestCategory = "system_variable" -) - -// FileChange represents a single file changed in a PR diff. -type FileChange struct { - Path string `json:"path"` - ChangeKind string `json:"change_kind"` // added, modified, deleted, renamed - Package string `json:"package"` // Go package path, e.g. "pkg/sql/plan" - Functions []string `json:"functions"` // changed function names (best effort) -} - -// DiffSummary contains the parsed result of a PR's code changes. -type DiffSummary struct { - PRNumber int `json:"pr_number"` - BaseBranch string `json:"base_branch"` - HeadBranch string `json:"head_branch"` - Files []FileChange `json:"files"` - TotalAdded int `json:"total_added"` - TotalDeleted int `json:"total_deleted"` -} - -// AffectedPackages returns the deduplicated set of Go packages affected. -func (d *DiffSummary) AffectedPackages() []string { - seen := make(map[string]struct{}) - var pkgs []string - for _, f := range d.Files { - if f.Package != "" { - if _, ok := seen[f.Package]; !ok { - seen[f.Package] = struct{}{} - pkgs = append(pkgs, f.Package) - } - } - } - return pkgs -} - -// TestTask represents a single executable test task within a TestPlan. -type TestTask struct { - ID string `json:"id"` - Type TestType `json:"type"` - Category TestCategory `json:"category,omitempty"` - Package string `json:"package,omitempty"` // Go package for UT - TestFile string `json:"test_file,omitempty"` // BVT .test file path - Priority Priority `json:"priority"` - EstDuration string `json:"est_duration,omitempty"` - Status TaskStatus `json:"status"` - Reason string `json:"reason"` // why this task is included -} - -// TestPlan is the structured output produced by the planner. It describes -// which tests should be run for a given PR. -type TestPlan struct { - ID string `json:"id"` - PRNumber int `json:"pr_number"` - BaseBranch string `json:"base_branch"` - HeadBranch string `json:"head_branch"` - CreatedAt time.Time `json:"created_at"` - Summary string `json:"summary"` - DiffSummary DiffSummary `json:"diff_summary"` - Tasks []TestTask `json:"tasks"` -} - -// ToJSON serializes the TestPlan to indented JSON. -func (tp *TestPlan) ToJSON() ([]byte, error) { - return json.MarshalIndent(tp, "", " ") -} - -// FromJSON deserializes a TestPlan from JSON bytes. -func FromJSON(data []byte) (*TestPlan, error) { - var tp TestPlan - if err := json.Unmarshal(data, &tp); err != nil { - return nil, err - } - return &tp, nil -} - -// TaskCountByStatus returns a map of status → count for quick summaries. -func (tp *TestPlan) TaskCountByStatus() map[TaskStatus]int { - m := make(map[TaskStatus]int) - for _, t := range tp.Tasks { - m[t.Status]++ - } - return m -} - -// TaskCountByType returns a map of type → count. -func (tp *TestPlan) TaskCountByType() map[TestType]int { - m := make(map[TestType]int) - for _, t := range tp.Tasks { - m[t.Type]++ - } - return m -} diff --git a/pkg/testinfra/types/types_test.go b/pkg/testinfra/types/types_test.go deleted file mode 100644 index 83f2ce163ae2f..0000000000000 --- a/pkg/testinfra/types/types_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2024 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "encoding/json" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPriorityString(t *testing.T) { - tests := []struct { - p Priority - want string - }{ - {PriorityCritical, "critical"}, - {PriorityHigh, "high"}, - {PriorityMedium, "medium"}, - {PriorityLow, "low"}, - {Priority(99), "unknown"}, - } - for _, tt := range tests { - assert.Equal(t, tt.want, tt.p.String()) - } -} - -func TestDiffSummaryAffectedPackages(t *testing.T) { - ds := DiffSummary{ - Files: []FileChange{ - {Path: "pkg/sql/plan/build.go", Package: "pkg/sql/plan"}, - {Path: "pkg/sql/plan/optimize.go", Package: "pkg/sql/plan"}, - {Path: "pkg/vm/engine/disttae/txn.go", Package: "pkg/vm/engine/disttae"}, - {Path: "README.md", Package: ""}, - }, - } - pkgs := ds.AffectedPackages() - assert.Equal(t, 2, len(pkgs)) - assert.Contains(t, pkgs, "pkg/sql/plan") - assert.Contains(t, pkgs, "pkg/vm/engine/disttae") -} - -func TestTestPlanJSON(t *testing.T) { - plan := &TestPlan{ - ID: "tp-001", - PRNumber: 12345, - BaseBranch: "main", - HeadBranch: "feature/test", - CreatedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), - Summary: "Test plan for PR #12345", - Tasks: []TestTask{ - { - ID: "task-1", - Type: TestTypeUT, - Package: "pkg/sql/plan", - Priority: PriorityHigh, - Status: TaskStatusPending, - Reason: "pkg/sql/plan modified", - }, - { - ID: "task-2", - Type: TestTypeBVT, - Category: CategoryOptimizer, - Priority: PriorityMedium, - Status: TaskStatusPending, - Reason: "optimizer category mapped from pkg/sql/plan", - }, - }, - } - - data, err := plan.ToJSON() - require.NoError(t, err) - - var parsed map[string]interface{} - err = json.Unmarshal(data, &parsed) - require.NoError(t, err) - assert.Equal(t, "tp-001", parsed["id"]) - assert.Equal(t, float64(12345), parsed["pr_number"]) - - restored, err := FromJSON(data) - require.NoError(t, err) - assert.Equal(t, plan.ID, restored.ID) - assert.Equal(t, plan.PRNumber, restored.PRNumber) - assert.Equal(t, len(plan.Tasks), len(restored.Tasks)) - assert.Equal(t, plan.Tasks[0].Package, restored.Tasks[0].Package) -} - -func TestTestPlanTaskCounts(t *testing.T) { - plan := &TestPlan{ - Tasks: []TestTask{ - {Status: TaskStatusPending, Type: TestTypeUT}, - {Status: TaskStatusPending, Type: TestTypeBVT}, - {Status: TaskStatusRunning, Type: TestTypeUT}, - {Status: TaskStatusPassed, Type: TestTypeBVT}, - {Status: TaskStatusFailed, Type: TestTypeSCA}, - }, - } - - byStatus := plan.TaskCountByStatus() - assert.Equal(t, 2, byStatus[TaskStatusPending]) - assert.Equal(t, 1, byStatus[TaskStatusRunning]) - assert.Equal(t, 1, byStatus[TaskStatusPassed]) - assert.Equal(t, 1, byStatus[TaskStatusFailed]) - - byType := plan.TaskCountByType() - assert.Equal(t, 2, byType[TestTypeUT]) - assert.Equal(t, 2, byType[TestTypeBVT]) - assert.Equal(t, 1, byType[TestTypeSCA]) -} - -func TestFromJSONInvalid(t *testing.T) { - _, err := FromJSON([]byte("not json")) - assert.Error(t, err) -} From 01d18025c236a7f940fe96d50173c68f49d2c297 Mon Sep 17 00:00:00 2001 From: "Ariznawl@163.com" Date: Thu, 9 Apr 2026 15:42:55 +0800 Subject: [PATCH 10/10] feat(testinfra): V2 AI-powered test coverage analyzer with dedup - 6 test types: BVT, stability, chaos, bigdata, PITR, snapshot - LLM-powered analysis via GitHub Models API (gpt-4o-mini) - Smart category inference from changed file paths - Sample case loading for LLM format learning - SQL deduplication against existing test files (>50% overlap filter) - Precise targeting: only generates cases for types that truly need coverage - Writer module for auto-writing cases to correct locations - CLI: mo-testplan --pr N [--write] [--create-pr] [--json] - Expanded testing-guide.md with real formats for all 6 test types - 35+ unit tests across 6 packages --- cmd/mo-testplan/main.go | 177 ++++++++++ docs/ai-skills/testing-guide.md | 193 ++++++++--- pkg/testinfra/analyzer/analyzer.go | 432 ++++++++++++++++++++++++ pkg/testinfra/analyzer/analyzer_test.go | 379 +++++++++++++++++++++ pkg/testinfra/dedup/dedup.go | 200 +++++++++++ pkg/testinfra/dedup/dedup_test.go | 245 ++++++++++++++ pkg/testinfra/llm/client.go | 104 ++++++ pkg/testinfra/llm/client_test.go | 118 +++++++ pkg/testinfra/scanner/scanner.go | 172 ++++++++++ pkg/testinfra/scanner/scanner_test.go | 169 +++++++++ pkg/testinfra/types/types.go | 61 ++++ pkg/testinfra/types/types_test.go | 101 ++++++ pkg/testinfra/writer/writer.go | 120 +++++++ pkg/testinfra/writer/writer_test.go | 235 +++++++++++++ 14 files changed, 2655 insertions(+), 51 deletions(-) create mode 100644 cmd/mo-testplan/main.go create mode 100644 pkg/testinfra/analyzer/analyzer.go create mode 100644 pkg/testinfra/analyzer/analyzer_test.go create mode 100644 pkg/testinfra/dedup/dedup.go create mode 100644 pkg/testinfra/dedup/dedup_test.go create mode 100644 pkg/testinfra/llm/client.go create mode 100644 pkg/testinfra/llm/client_test.go create mode 100644 pkg/testinfra/scanner/scanner.go create mode 100644 pkg/testinfra/scanner/scanner_test.go create mode 100644 pkg/testinfra/types/types.go create mode 100644 pkg/testinfra/types/types_test.go create mode 100644 pkg/testinfra/writer/writer.go create mode 100644 pkg/testinfra/writer/writer_test.go diff --git a/cmd/mo-testplan/main.go b/cmd/mo-testplan/main.go new file mode 100644 index 0000000000000..e5fca973ccda9 --- /dev/null +++ b/cmd/mo-testplan/main.go @@ -0,0 +1,177 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// mo-testplan analyzes PR test coverage against MO's 6 test types +// (BVT, stability, chaos, big data, PITR, snapshot) using AI. +// +// Usage: +// +// mo-testplan --pr 24088 +// mo-testplan --pr 24088 --json +// mo-testplan --pr 24088 --write +// mo-testplan --pr 24088 --write --create-pr +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/matrixorigin/matrixone/pkg/testinfra/analyzer" + "github.com/matrixorigin/matrixone/pkg/testinfra/writer" +) + +const ( + defaultAPIURL = "https://models.github.ai/inference/chat/completions" + defaultModel = "openai/gpt-4o-mini" +) + +func main() { + var ( + prNumber int + repo string + apiURL string + apiKey string + model string + jsonOut bool + write bool + createPR bool + ) + + flag.IntVar(&prNumber, "pr", 0, "PR number to analyze") + flag.StringVar(&repo, "repo", "matrixorigin/matrixone", "GitHub repository (owner/name)") + flag.StringVar(&apiURL, "api-url", defaultAPIURL, "LLM API endpoint") + flag.StringVar(&apiKey, "api-key", "", "API key (default: $GITHUB_TOKEN or gh auth token)") + flag.StringVar(&model, "model", defaultModel, "LLM model name") + flag.BoolVar(&jsonOut, "json", false, "output raw JSON instead of formatted report") + flag.BoolVar(&write, "write", false, "write suggested BVT cases to test/distributed/cases/") + flag.BoolVar(&createPR, "create-pr", false, "create a PR with written cases (implies --write)") + flag.Parse() + + if prNumber == 0 { + fmt.Fprintf(os.Stderr, "Usage: mo-testplan --pr [--repo owner/name] [--json]\n") + fmt.Fprintf(os.Stderr, "\nFlags:\n") + flag.PrintDefaults() + os.Exit(1) + } + + if apiKey == "" { + apiKey = resolveAPIKey() + } + if apiKey == "" { + fmt.Fprintf(os.Stderr, "Error: no API key found.\n") + fmt.Fprintf(os.Stderr, "Set --api-key, $GITHUB_TOKEN, or login with: gh auth login\n") + os.Exit(1) + } + + repoRoot := detectRepoRoot() + if repoRoot == "" { + fmt.Fprintf(os.Stderr, "Error: cannot detect repo root (no go.mod found).\n") + os.Exit(1) + } + + cfg := analyzer.Config{ + Repo: repo, + RepoRoot: repoRoot, + APIURL: apiURL, + APIKey: apiKey, + Model: model, + } + + a := analyzer.New(cfg) + report, err := a.Analyze(context.Background(), prNumber) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + if jsonOut { + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + os.Stdout.Write(data) + os.Stdout.WriteString("\n") + } else { + os.Stdout.WriteString(analyzer.FormatReport(report)) + } + + // --create-pr implies --write + if createPR { + write = true + } + + if write && len(report.SuggestedCases) > 0 { + written, err := writer.WriteCases(repoRoot, report.SuggestedCases) + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing cases: %v\n", err) + os.Exit(1) + } + if len(written) > 0 { + os.Stdout.WriteString("\n### 已写入文件\n") + for _, f := range written { + os.Stdout.WriteString("- ") + os.Stdout.WriteString(f) + os.Stdout.WriteString("\n") + } + + if createPR { + prURL, err := writer.CreatePR(repoRoot, repo, prNumber, written) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating PR: %v\n", err) + os.Exit(1) + } + os.Stdout.WriteString("\n### 已创建 PR\n") + os.Stdout.WriteString(strings.TrimSpace(prURL)) + os.Stdout.WriteString("\n") + } + } else { + os.Stdout.WriteString("\n无新增文件（已有 case 或建议为空）\n") + } + } +} + +func resolveAPIKey() string { + if key := os.Getenv("GITHUB_TOKEN"); key != "" { + return key + } + out, err := exec.Command("gh", "auth", "token").Output() + if err == nil { + return strings.TrimSpace(string(out)) + } + return "" +} + +func detectRepoRoot() string { + dir, err := os.Getwd() + if err != nil { + return "" + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + return "" + } + dir = parent + } +} diff --git a/docs/ai-skills/testing-guide.md b/docs/ai-skills/testing-guide.md index 79f079f19d648..c1b096c9998ce 100644 --- a/docs/ai-skills/testing-guide.md +++ b/docs/ai-skills/testing-guide.md @@ -4,78 +4,169 @@ ### 1. BVT 测试（轻量级回归） - **仓库：** matrixone -- **路径：** `test/distributed/cases/` +- **路径：** `test/distributed/cases/{category}/` - **工具：** mo-tester -- **Case 格式：** `.test` 文件（SQL + 标签） +- **Case 格式：** `.test`/`.sql` + 对应 `.result` 文件 - **运行时机：** 每个 PR 的 CI +- **运行命令：** `cd mo-tester && ./run.sh -n -g -p /path/to/test.test` +- **生成 result：** `./run.sh -m genrs -n -g -p /path/to/test.test` ### 2. 稳定性测试（长时间运行） - **仓库：** mo-nightly-regression (main) +- **Workflow：** `stability-test-on-distributed.yaml` - **测试项：** TPCH, TPCC, Sysbench, Fulltext-vector, Vector IVF+DML +- **配置：** `cases/sysbench/{scenario}/run.yml` — mo-load YAML 配置 +- **参数：** TPCHScale, LoadDataScale, TPCC_WarehouseNum, OLTPThreads, RunMinutes - **目的：** 验证长时间运行下的稳定性 -- **运行时机：** Nightly ### 3. Chaos 测试（故障注入） - **仓库：** mo-nightly-regression (main) -- **配置：** `mo-chaos-config/` -- **测试项：** Sysbench, TPCC, Fulltext（叠加故障注入） -- **故障类型：** 杀 CN、杀 TN、杀 LogService、网络分区 -- **运行时机：** Nightly +- **配置：** `mo-chaos-config/chaos_regression.yaml` — Chaos Mesh YAML +- **工作负载：** `mo-chaos-config/chaos_test_case.yaml` — 测试任务列表 +- **故障类型：** + - `PodChaos/pod-kill` — 杀 CN/TN/LogService Pod + - `StressChaos/memory` — 内存压力 + - `NetworkChaos` — 网络分区 +- **测试任务：** mo-tpcc, mo-load(sysbench), mo-cdc-test, mo-vector-fulltext-test +- **工作负载配置格式：** `mo-chaos-config/mo-load-insert-case/run.yml` ### 4. 大数据量测试 -- **仓库：** mo-nightly-regression (**big_data 分支**) -- **路径：** `tools/mo-regression-test/cases/big_data_test/` -- **特点：** 从云端 load 大规模数据后执行测试 -- **运行时机：** 定期 +- **仓库：** mo-nightly-regression (main) +- **Workflow：** `big-data-test.yml` +- **路径：** `tools/mo-regression-test/cases/{customer_name}/` +- **配置格式：** `mo_ddl.yaml`（DDL source）, `mo_load_*.yaml`（数据加载）, `q00.sql`（查询） +- **Suite：** suite_1y（1年数据）, suite_10y（10年数据）, all +- **数据来源：** COS 对象存储 + load data ### 5. PITR 测试 -- **仓库：** mo-nightly-regression (main) -- **Workflow：** `pitr-backup-restore-regression-main.yml` -- **内容：** Point-In-Time Recovery 完整流程验证 -- **运行时机：** Nightly +- **仓库：** matrixone（BVT 级别）+ mo-nightly-regression（完整流程） +- **BVT 路径：** `test/distributed/cases/pitr/` — pitr.sql, pitr_basic.sql, pitr_inherit.sql +- **内容：** CREATE PITR → 操作数据 → RESTORE → 验证数据一致性 ### 6. Snapshot 测试 -- **仓库：** mo-nightly-regression (main) -- **Workflow：** `snapshot_backup_restore_main.yml` -- **内容：** Snapshot 备份恢复完整流程验证 -- **运行时机：** Nightly +- **仓库：** matrixone（BVT 级别）+ mo-nightly-regression（完整流程） +- **BVT 路径：** `test/distributed/cases/snapshot/` — 多层级 snapshot 测试 +- **场景：** cluster/account/database/table 级别的 snapshot 创建和恢复 +- **内容：** CREATE SNAPSHOT → 操作数据 → RESTORE ACCOUNT → 验证 + +## BVT Case 格式详解 -## BVT Case 格式 +### 文件结构 +- `.test`/`.sql` — 测试文件（SQL + mo-tester 标签） +- `.result` — 期望输出（含列元数据 `column[type,precision,scale]`） +- 文件对应关系：`func_sum.test` ↔ `func_sum.result` + +### 标签语法 + +**文件级标签：** +```sql +-- @skip:issue#16438 -- 跳过整个文件 +--- @metacmp(false) -- 关闭元数据比较（三个 -） +``` -`.test` 文件示例： +**SQL 级标签：** ```sql --- @bvt:issue#12345 -SELECT 1; --- @bvt:issue#12345 +-- @bvt:issue#3185 -- 开始跳过区块 +SELECT * FROM t1; +-- @bvt:issue -- 结束跳过区块 + +-- @ignore:0 -- 忽略第 0 列（0-indexed） +select time(now()); + +-- @ignore:5,6 -- 忽略多列 +show publications; + +-- @sortkey:0,1 -- 按第 0,1 列排序后比较 +SELECT col1, col2 FROM t1; --- @session:id=1 -BEGIN; -INSERT INTO t1 VALUES (1); --- @session +-- @regex("pattern", true) -- 结果必须匹配 pattern +show accounts; --- @sortkey:0,1 -SELECT * FROM t1 ORDER BY id; +-- @regex("error", false) -- 结果不能匹配 pattern +SHOW TABLES; --- @ignore:1 -SELECT NOW(), COUNT(*) FROM t1; +-- @metacmp(true) -- 单条 SQL 开启元数据比较 +SELECT * FROM t1; +``` + +**会话标签（并发测试）：** +```sql +begin; +select * from t1; +-- @session:id=1&user=acc:admin&password=111{ +insert into t1 values (100); +-- @wait:0:commit -- 等待 session 0 commit +select * from t1; +-- @session} +commit; ``` -**常用标签：** -| 标签 | 说明 | -|------|------| -| `@bvt:issue#N` | 跳过指定区块 | -| `@skip:issue#N` | 跳过整个文件 | -| `@session:id=N` | 并发会话 | -| `@wait:session:commit/rollback` | 等待会话提交/回滚 | -| `@sortkey:cols` | 结果排序（消除不确定顺序）| -| `@ignore:cols` | 忽略指定列（如时间戳）| - -## BVT 类别（70+） - -核心 SQL: `dml`, `ddl`, `database`, `table`, `view`, `sequence` -查询: `expression`, `function`, `subquery`, `cte`, `recursive_cte`, `window`, `join` -事务: `pessimistic_transaction`, `optimistic`, `snapshot` -高级功能: `fulltext`, `vector`, `udf`, `procedure` -特性: `partition`, `charset_collation`, `foreign_key`, `temporary`, `tenant` -数据操作: `load_data`, `replace_statement`, `prepare`, `hint` -基础设施: `pitr`, `disttae`, `log`, `metadata`, `stage` +### BVT Case 编写规范 +1. **自包含** — 每个 test 文件独立运行 +2. **资源清理** — 结尾 drop 创建的表/库 +3. **确定性** — 不确定列用 `@ignore`，不确定顺序用 `@sortkey` +4. **复用库** — 避免创建过多临时 database + +## 稳定性测试 Case 格式（mo-load） + +```yaml +# cases/sysbench/simple_insert_10_100000/run.yml +duration: 10 # 运行分钟数 +stdout: "console" +transaction: + - name: "simple_insert" + vuser: 10 # 并发数 + mode: 0 # 0=顺序执行, 1=封装为事务 + prepared: "false" + script: + - sql: "insert into sbtest{tbx} values({i_id},{kvalue},'...');" +``` + +## Chaos 测试配置格式 + +### 故障定义 (chaos_regression.yaml) +```yaml +chaos: + cm-chaos: + - name: task_kill_cn + kubectl_yaml: | + apiVersion: chaos-mesh.org/v1alpha1 + kind: PodChaos + spec: + action: pod-kill + selector: + labelSelectors: + 'matrixorigin.io/component': 'CNSet' # 或 DNSet/LogSet + times: 1 + interval: 30 + is_delete_after_apply: true +``` + +### 工作负载定义 (chaos_test_case.yaml) +```yaml +tasks: + - name: mo-tpcc + work-path: mo-tpcc + run-steps: + - command: ./runBenchmark.sh props.mo > tpcc.log 2>&1 + verify: + - command: ./runVerify.sh props.mo >> check.log 2>&1 + verify-mode: parallel # parallel(边跑边验)/after(跑完再验) +``` + +## 大数据测试 Case 格式 + +```yaml +# tools/mo-regression-test/cases/tpch_1g/mo_ddl.yaml +source_path: "/data/customer/tpch_1g/ddl/mo.sql" +load_type: "ddl" + +# tools/mo-regression-test/cases/tpch_1g/mo_load_server_serial.yaml +s3_path: "cos://bucket/path/to/data.csv" +load_type: "load_server" +count: "6001215" + +# tools/mo-regression-test/cases/tpch_1g/q00.sql +-- 标准 SQL 查询文件 +SELECT * FROM lineitem WHERE l_shipdate <= '1998-09-02'; +``` diff --git a/pkg/testinfra/analyzer/analyzer.go b/pkg/testinfra/analyzer/analyzer.go new file mode 100644 index 0000000000000..4771125c35586 --- /dev/null +++ b/pkg/testinfra/analyzer/analyzer.go @@ -0,0 +1,432 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analyzer + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/testinfra/dedup" + "github.com/matrixorigin/matrixone/pkg/testinfra/llm" + "github.com/matrixorigin/matrixone/pkg/testinfra/scanner" + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +const maxDiffLines = 300 + +// Config holds analyzer configuration. +type Config struct { + Repo string + RepoRoot string + APIURL string + APIKey string + Model string +} + +// Analyzer orchestrates PR analysis: diff → skills → cases → LLM → report. +type Analyzer struct { + config Config + client *llm.Client +} + +// New creates an Analyzer. +func New(cfg Config) *Analyzer { + return &Analyzer{ + config: cfg, + client: llm.NewClient(cfg.APIURL, cfg.APIKey, cfg.Model), + } +} + +// Analyze fetches the PR diff, loads skill docs and existing cases, +// calls the LLM, and returns a structured coverage report. +func (a *Analyzer) Analyze(ctx context.Context, prNumber int) (*types.CoverageReport, error) { + diff, err := a.getDiff(prNumber) + if err != nil { + return nil, err + } + + // Extract changed file paths from diff for category inference + changedPaths := extractChangedPaths(diff) + + skills, err := a.loadSkills() + if err != nil { + return nil, err + } + + cases, err := scanner.ScanBVTCases(a.config.RepoRoot) + if err != nil { + return nil, err + } + caseSummary := scanner.FormatCaseSummary(cases) + + // Read actual case examples from relevant categories + sampleCases := scanner.ReadSampleCases(a.config.RepoRoot, changedPaths, 3, 40) + + systemMsg := buildSystemPrompt(skills) + userMsg := buildUserPrompt(prNumber, diff, caseSummary, sampleCases) + + response, err := a.client.Chat(ctx, []llm.Message{ + {Role: "system", Content: systemMsg}, + {Role: "user", Content: userMsg}, + }) + if err != nil { + return nil, err + } + + report, err := parseReport(response, prNumber) + if err != nil { + return nil, err + } + + // Deduplicate suggested cases against existing test files + d := dedup.New(a.config.RepoRoot) + kept, skipped := d.Filter(report.SuggestedCases) + report.SuggestedCases = kept + if len(skipped) > 0 { + report.Summary += fmt.Sprintf("\n\n[Dedup] Filtered %d case(s): %s", len(skipped), strings.Join(skipped, "; ")) + } + + return report, nil +} + +// extractChangedPaths parses diff headers to get file paths. +func extractChangedPaths(diff string) []string { + var paths []string + for _, line := range strings.Split(diff, "\n") { + if strings.HasPrefix(line, "+++ b/") { + paths = append(paths, line[6:]) + } + } + return paths +} + +func (a *Analyzer) getDiff(prNumber int) (string, error) { + cmd := exec.Command("gh", "pr", "diff", strconv.Itoa(prNumber), "--repo", a.config.Repo) + out, err := cmd.Output() + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("gh pr diff failed for #%d: %v", prNumber, err) + } + + diff := string(out) + lines := strings.Split(diff, "\n") + if len(lines) > maxDiffLines { + diff = strings.Join(lines[:maxDiffLines], "\n") + diff += fmt.Sprintf("\n\n... (truncated, showing first %d of %d lines)", maxDiffLines, len(lines)) + } + return diff, nil +} + +// prioritySkills are the most important skill docs to include in the prompt. +// These are loaded first; others are added only if token budget allows. +var prioritySkills = []string{ + "module-test-mapping.md", + "testing-guide.md", +} + +func (a *Analyzer) loadSkills() (string, error) { + skillsDir := filepath.Join(a.config.RepoRoot, "docs", "ai-skills") + + var b strings.Builder + // Load priority docs first + for _, name := range prioritySkills { + data, err := os.ReadFile(filepath.Join(skillsDir, name)) + if err != nil { + continue + } + b.WriteString("## ") + b.WriteString(name) + b.WriteString("\n") + b.Write(data) + b.WriteString("\n\n") + } + + // Load remaining docs if total is under 6KB + entries, err := os.ReadDir(skillsDir) + if err != nil { + return b.String(), nil + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") { + continue + } + // Skip already loaded + skip := false + for _, p := range prioritySkills { + if entry.Name() == p { + skip = true + break + } + } + if skip { + continue + } + if b.Len() > 6000 { + break + } + data, err := os.ReadFile(filepath.Join(skillsDir, entry.Name())) + if err != nil { + continue + } + b.WriteString("## ") + b.WriteString(entry.Name()) + b.WriteString("\n") + b.Write(data) + b.WriteString("\n\n") + } + return b.String(), nil +} + +func buildSystemPrompt(skills string) string { + return `你是 MatrixOne 数据库的测试覆盖分析 Agent。分析 PR 代码变更，判断 6 类测试的覆盖情况，并**生成完整可运行的测试用例**。 + +## 6 类测试 + +### 1. BVT（轻量级回归，matrixone 仓库） +- 路径: test/distributed/cases/{category}/{name}.sql +- 格式: SQL + mo-tester 标签 +- 标签: @bvt:issue#N（跳过区块）、@session:id=N（并发会话）、@wait:N:commit（等待提交）、@sortkey:cols（排序）、@ignore:cols（忽略列）、@regex("pattern",true/false) +- 结尾必须 drop 创建的表/数据库 +- 每个文件必须自包含、可独立运行 +- 不确定顺序用 @sortkey，不确定值（如时间）用 @ignore + +### 2. 稳定性测试（mo-nightly-regression 仓库，main 分支） +- 路径: cases/sysbench/{scenario}/run.yml +- 格式: YAML (mo-load 配置) +- 内容: duration, transaction[].name/vuser/mode/script[].sql +- 场景: TPCH、TPCC、Sysbench(insert/select/update/delete/mixed)、Fulltext-vector + +### 3. Chaos 测试（mo-nightly-regression 仓库，main 分支） +- 故障定义: mo-chaos-config/chaos_regression.yaml (Chaos Mesh YAML) +- 工作负载: mo-chaos-config/chaos_test_case.yaml (tasks[].run-steps/verify) +- 故障类型: PodChaos/pod-kill(杀CN/TN/LogService)、StressChaos/memory、NetworkChaos +- 工作负载: mo-tpcc、mo-load(sysbench)、mo-cdc-test、mo-vector-fulltext-test + +### 4. 大数据量测试（mo-nightly-regression 仓库，main 分支） +- 路径: tools/mo-regression-test/cases/{test_name}/ +- 格式: mo_ddl.yaml + mo_load_*.yaml + q00.sql +- 数据来源: COS 对象存储 load + +### 5. PITR 测试（matrixone 仓库 BVT 级别） +- 路径: test/distributed/cases/pitr/ +- 格式: .sql + mo-tester 标签 +- 流程: CREATE PITR → 操作数据 → RESTORE → 验证一致性 + +### 6. Snapshot 测试（matrixone 仓库 BVT 级别） +- 路径: test/distributed/cases/snapshot/ +- 格式: .sql + mo-tester 标签 +- 场景: cluster/account/database/table 级别 snapshot 创建和恢复 + +## MO 知识文档 +` + skills +} + +func buildUserPrompt(prNumber int, diff, caseSummary, sampleCases string) string { + var b strings.Builder + b.WriteString(fmt.Sprintf("分析 PR #%d 的测试覆盖情况。\n\n", prNumber)) + b.WriteString("**关键原则**: 只为真正缺少测试覆盖的类型生成 case。如果某类测试已有覆盖或与本次变更完全无关，status 设为 covered 或 not_related，不要生成 case。不要盲目为每种测试类型都生成 case。\n\n") + + b.WriteString("## PR Diff:\n") + b.WriteString(diff) + b.WriteString("\n\n") + + b.WriteString("## 已有 BVT 测试目录（category/文件数）:\n") + b.WriteString(caseSummary) + b.WriteString("\n\n") + + if sampleCases != "" { + b.WriteString("## 目标目录已有 case 示例（学习格式，避免与这些已有 case 重复）:\n") + b.WriteString(sampleCases) + b.WriteString("\n") + } + + b.WriteString(`## 输出要求 + +请严格输出以下 JSON 格式（不要添加 markdown 代码块或其他文字）: +{ + "summary": "一句话描述改了什么", + "affected_modules": ["受影响的模块路径"], + "coverage": [ + {"type": "bvt", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "stability", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "chaos", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "bigdata", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "pitr", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "snapshot", "status": "covered|needs_attention|not_related", "description": "说明"} + ], + "suggested_cases": [ + { + "type": "bvt|stability|chaos|bigdata|pitr|snapshot", + "category": "BVT子目录名 或 nightly场景名", + "filename": "文件名", + "content": "完整的可运行内容", + "reason": "补充原因" + } + ] +} + +## 生成 case 规则 + +### BVT case 生成规则: +1. 必须是完整可运行的 .sql 文件 +2. 参考上面的已有 case 示例的格式和风格 +3. 结尾必须 drop 创建的表/数据库 +4. 不确定的结果列用 @ignore，不确定的行序用 @sortkey +5. category 填对应的 BVT 子目录名（如 function, ddl, window, expression 等） + +### 稳定性 case 生成规则: +1. 格式为 mo-load run.yml +2. 包含 duration、transaction 配置 +3. category 填 "sysbench/{场景名}" + +### Chaos case 生成规则: +1. 如果需要新增故障类型，生成 Chaos Mesh YAML +2. 如果需要新增工作负载，生成 chaos_test_case.yaml 格式的 task +3. category 填 "mo-chaos-config" + +### PITR/Snapshot case 生成规则: +1. 与 BVT 同格式（.sql + mo-tester 标签） +2. PITR: CREATE PITR → DML → RESTORE → SELECT 验证 +3. Snapshot: CREATE SNAPSHOT → DML → RESTORE ACCOUNT → SELECT 验证 +4. category 分别填 "pitr" 或 "snapshot" + +注意： +- coverage 必须包含全部 6 种类型 +- suggested_cases 只列 status 为 needs_attention 的测试类型，绝不要为 not_related 或 covered 的类型生成 case +- 如果本次变更只涉及 BVT 相关代码（如 SQL 函数、DDL 等），就只生成 BVT case，不要强行凑其他类型 +- 生成的 SQL 不要与已有示例中的内容重复，关注新增/修改代码路径的覆盖 +- BVT/PITR/Snapshot case 的 content 必须是完整可运行的 SQL +- 稳定性/Chaos case 的 content 必须是完整的 YAML +`) + return b.String() +} + +func parseReport(response string, prNumber int) (*types.CoverageReport, error) { + jsonStr := extractJSON(response) + + var report types.CoverageReport + if err := json.Unmarshal([]byte(jsonStr), &report); err != nil { + return nil, moerr.NewInternalErrorNoCtxf("parse LLM JSON: %v\nRaw:\n%s", err, response) + } + report.PRNumber = prNumber + return &report, nil +} + +func extractJSON(s string) string { + s = strings.TrimSpace(s) + if idx := strings.Index(s, "```json"); idx >= 0 { + s = s[idx+7:] + if end := strings.Index(s, "```"); end >= 0 { + s = s[:end] + } + } else if idx := strings.Index(s, "```"); idx >= 0 { + s = s[idx+3:] + if nl := strings.Index(s, "\n"); nl >= 0 { + s = s[nl:] + } + if end := strings.Index(s, "```"); end >= 0 { + s = s[:end] + } + } + // Fix LLM producing string concatenation instead of proper JSON strings. + // e.g. "line1\n" +\n "line2\n" → "line1\nline2\n" + s = sanitizeJSONConcat(s) + return strings.TrimSpace(s) +} + +// sanitizeJSONConcat merges broken "..." + "..." patterns that some LLMs produce. +func sanitizeJSONConcat(s string) string { + for { + // Match: " +\n{whitespace}" or " +\n{whitespace}' + idx := strings.Index(s, "\" +") + if idx < 0 { + break + } + // Find the next quote after " +" + rest := s[idx+3:] + rest = strings.TrimLeft(rest, " \t\n\r") + if len(rest) == 0 || rest[0] != '"' { + break + } + // Merge: replace the closing quote + concat + opening quote with nothing + s = s[:idx] + rest[1:] + } + return s +} + +// FormatReport produces a terminal-friendly coverage report. +func FormatReport(r *types.CoverageReport) string { + var b strings.Builder + + b.WriteString(fmt.Sprintf("## PR #%d 测试覆盖分析\n\n", r.PRNumber)) + b.WriteString("### 变更摘要\n") + b.WriteString(r.Summary) + b.WriteString("\n\n") + + if len(r.AffectedModules) > 0 { + b.WriteString("### 受影响模块\n") + for _, m := range r.AffectedModules { + b.WriteString("- ") + b.WriteString(m) + b.WriteString("\n") + } + b.WriteString("\n") + } + + b.WriteString("### 6 类测试覆盖情况\n\n") + b.WriteString("| 测试类型 | 状态 | 说明 |\n") + b.WriteString("|---------|------|------|\n") + typeNames := map[types.TestType]string{ + types.TestBVT: "BVT", types.TestStability: "稳定性", + types.TestChaos: "Chaos", types.TestBigData: "大数据", + types.TestPITR: "PITR", types.TestSnapshot: "Snapshot", + } + for _, c := range r.Coverage { + icon := "➖" + switch c.Status { + case types.StatusCovered: + icon = "✅" + case types.StatusNeedsAttention: + icon = "⚠️" + } + name := typeNames[c.Type] + if name == "" { + name = string(c.Type) + } + b.WriteString(fmt.Sprintf("| %s | %s | %s |\n", name, icon, c.Description)) + } + b.WriteString("\n") + + if len(r.SuggestedCases) > 0 { + b.WriteString(fmt.Sprintf("### 建议补充的 Case（%d 个）\n\n", len(r.SuggestedCases))) + for i, sc := range r.SuggestedCases { + path := "test/distributed/cases/" + sc.Category + "/" + sc.Filename + b.WriteString(fmt.Sprintf("#### %d. %s\n", i+1, path)) + b.WriteString("原因: ") + b.WriteString(sc.Reason) + b.WriteString("\n\n```sql\n") + b.WriteString(sc.Content) + b.WriteString("\n```\n\n") + } + } + + return b.String() +} diff --git a/pkg/testinfra/analyzer/analyzer_test.go b/pkg/testinfra/analyzer/analyzer_test.go new file mode 100644 index 0000000000000..3b855cb2793f5 --- /dev/null +++ b/pkg/testinfra/analyzer/analyzer_test.go @@ -0,0 +1,379 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analyzer + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +func TestExtractChangedPaths(t *testing.T) { + diff := `diff --git a/pkg/sql/plan/foo.go b/pkg/sql/plan/foo.go +--- a/pkg/sql/plan/foo.go ++++ b/pkg/sql/plan/foo.go +@@ -1,3 +1,4 @@ ++import "fmt" +diff --git a/pkg/txn/bar.go b/pkg/txn/bar.go +--- a/pkg/txn/bar.go ++++ b/pkg/txn/bar.go +` + paths := extractChangedPaths(diff) + if len(paths) != 2 { + t.Fatalf("paths = %v, want 2", paths) + } + if paths[0] != "pkg/sql/plan/foo.go" { + t.Errorf("paths[0] = %q", paths[0]) + } + if paths[1] != "pkg/txn/bar.go" { + t.Errorf("paths[1] = %q", paths[1]) + } +} + +func TestExtractJSON_Plain(t *testing.T) { + input := `{"summary":"test","affected_modules":[],"coverage":[],"suggested_cases":[]}` + result := extractJSON(input) + if result != input { + t.Errorf("extractJSON plain = %q", result) + } +} + +func TestExtractJSON_Markdown(t *testing.T) { + input := "Some text\n```json\n{\"summary\":\"test\"}\n```\nmore text" + result := extractJSON(input) + if result != `{"summary":"test"}` { + t.Errorf("extractJSON markdown = %q", result) + } +} + +func TestExtractJSON_MarkdownNoLang(t *testing.T) { + input := "```\n{\"summary\":\"test\"}\n```" + result := extractJSON(input) + if result != `{"summary":"test"}` { + t.Errorf("extractJSON no lang = %q", result) + } +} + +func TestExtractJSON_ConcatStrings(t *testing.T) { + input := `{ + "content": "line1\n" + + "line2\n" + + "line3" +}` + result := extractJSON(input) + var m map[string]string + if err := json.Unmarshal([]byte(result), &m); err != nil { + t.Fatalf("should produce valid JSON: %v\nGot: %s", err, result) + } + if m["content"] != "line1\nline2\nline3" { + t.Errorf("content = %q", m["content"]) + } +} + +func TestSanitizeJSONConcat(t *testing.T) { + input := `"hello\n" + + "world"` + got := sanitizeJSONConcat(input) + want := `"hello\nworld"` + if got != want { + t.Errorf("sanitizeJSONConcat = %q, want %q", got, want) + } +} + +func TestBuildSystemPrompt(t *testing.T) { + result := buildSystemPrompt("skill content here") + if !strings.Contains(result, "MatrixOne") { + t.Error("system prompt should mention MatrixOne") + } + if !strings.Contains(result, "skill content here") { + t.Error("system prompt should include skills") + } + if !strings.Contains(result, "BVT") { + t.Error("system prompt should mention BVT") + } +} + +func TestBuildUserPrompt(t *testing.T) { + result := buildUserPrompt(123, "diff content", "function/10 ddl/5", "### sample\n```sql\nSELECT 1;\n```") + if !strings.Contains(result, "PR #123") { + t.Error("user prompt should mention PR number") + } + if !strings.Contains(result, "diff content") { + t.Error("user prompt should include diff") + } + if !strings.Contains(result, "function/10") { + t.Error("user prompt should include case summary") + } + if !strings.Contains(result, "sample") { + t.Error("user prompt should include sample cases") + } + if !strings.Contains(result, "stability") { + t.Error("user prompt should mention stability test type") + } + if !strings.Contains(result, "mo-chaos-config") { + t.Error("user prompt should mention chaos config") + } +} + +func TestLoadSkills(t *testing.T) { + tmp := t.TempDir() + skillsDir := filepath.Join(tmp, "docs", "ai-skills") + if err := os.MkdirAll(skillsDir, 0o755); err != nil { + t.Fatal(err) + } + + // Create priority skill doc + if err := os.WriteFile(filepath.Join(skillsDir, "module-test-mapping.md"), []byte("mapping content"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillsDir, "testing-guide.md"), []byte("testing content"), 0o644); err != nil { + t.Fatal(err) + } + // Create a non-priority doc + if err := os.WriteFile(filepath.Join(skillsDir, "architecture.md"), []byte("arch content"), 0o644); err != nil { + t.Fatal(err) + } + + a := &Analyzer{config: Config{RepoRoot: tmp}} + skills, err := a.loadSkills() + if err != nil { + t.Fatalf("loadSkills: %v", err) + } + + if !strings.Contains(skills, "mapping content") { + t.Error("should include module-test-mapping") + } + if !strings.Contains(skills, "testing content") { + t.Error("should include testing-guide") + } + if !strings.Contains(skills, "arch content") { + t.Error("should include architecture (under budget)") + } +} + +func TestLoadSkills_BudgetCap(t *testing.T) { + tmp := t.TempDir() + skillsDir := filepath.Join(tmp, "docs", "ai-skills") + if err := os.MkdirAll(skillsDir, 0o755); err != nil { + t.Fatal(err) + } + + // Create a priority doc that's already 5KB + bigContent := strings.Repeat("x", 5000) + if err := os.WriteFile(filepath.Join(skillsDir, "module-test-mapping.md"), []byte(bigContent), 0o644); err != nil { + t.Fatal(err) + } + // testing-guide pushes us over 6KB + if err := os.WriteFile(filepath.Join(skillsDir, "testing-guide.md"), []byte(strings.Repeat("y", 2000)), 0o644); err != nil { + t.Fatal(err) + } + // This extra doc should be skipped due to budget + if err := os.WriteFile(filepath.Join(skillsDir, "extra.md"), []byte("extra"), 0o644); err != nil { + t.Fatal(err) + } + + a := &Analyzer{config: Config{RepoRoot: tmp}} + skills, err := a.loadSkills() + if err != nil { + t.Fatalf("loadSkills: %v", err) + } + + // Priority docs always loaded + if !strings.Contains(skills, bigContent) { + t.Error("priority doc should always be loaded") + } + // Extra should be skipped (budget >6000) + if strings.Contains(skills, "extra") { + t.Error("extra doc should be skipped (over budget)") + } +} + +func TestLoadSkills_NoDir(t *testing.T) { + tmp := t.TempDir() + a := &Analyzer{config: Config{RepoRoot: tmp}} + skills, err := a.loadSkills() + if err != nil { + t.Fatalf("loadSkills: %v", err) + } + if skills != "" { + t.Errorf("expected empty skills, got %q", skills) + } +} + +func TestParseReport(t *testing.T) { + jsonStr := `{ + "summary": "fix decimal compare", + "affected_modules": ["pkg/sql"], + "coverage": [ + {"type": "bvt", "status": "covered", "description": "ok"}, + {"type": "stability", "status": "not_related", "description": "n/a"}, + {"type": "chaos", "status": "not_related", "description": "n/a"}, + {"type": "bigdata", "status": "not_related", "description": "n/a"}, + {"type": "pitr", "status": "not_related", "description": "n/a"}, + {"type": "snapshot", "status": "not_related", "description": "n/a"} + ], + "suggested_cases": [] + }` + + report, err := parseReport(jsonStr, 100) + if err != nil { + t.Fatalf("parseReport: %v", err) + } + if report.PRNumber != 100 { + t.Errorf("PRNumber = %d, want 100", report.PRNumber) + } + if report.Summary != "fix decimal compare" { + t.Errorf("Summary = %q", report.Summary) + } + if len(report.Coverage) != 6 { + t.Errorf("Coverage len = %d, want 6", len(report.Coverage)) + } +} + +func TestParseReport_WithMarkdown(t *testing.T) { + input := "Here is the analysis:\n```json\n" + + `{"summary":"test","affected_modules":[],"coverage":[],"suggested_cases":[]}` + + "\n```\n" + report, err := parseReport(input, 1) + if err != nil { + t.Fatalf("parseReport with markdown: %v", err) + } + if report.Summary != "test" { + t.Errorf("Summary = %q", report.Summary) + } +} + +func TestParseReport_InvalidJSON(t *testing.T) { + _, err := parseReport("not json", 1) + if err == nil { + t.Fatal("expected error for invalid JSON") + } +} + +func TestFormatReport(t *testing.T) { + report := &types.CoverageReport{ + PRNumber: 42, + Summary: "test PR", + AffectedModules: []string{"pkg/sql", "pkg/vm"}, + Coverage: []types.CoverageItem{ + {Type: types.TestBVT, Status: types.StatusCovered, Description: "ok"}, + {Type: types.TestStability, Status: types.StatusNeedsAttention, Description: "needs work"}, + {Type: types.TestChaos, Status: types.StatusNotRelated, Description: "n/a"}, + }, + SuggestedCases: []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "decimal_compare.test", + Content: "SELECT 1.0 = 1;", + Reason: "missing decimal test", + }, + }, + } + + output := FormatReport(report) + + checks := []string{ + "PR #42", + "test PR", + "pkg/sql", + "pkg/vm", + "BVT", + "✅", + "⚠️", + "➖", + "needs work", + "decimal_compare.test", + "SELECT 1.0 = 1;", + "missing decimal test", + } + for _, check := range checks { + if !strings.Contains(output, check) { + t.Errorf("output should contain %q", check) + } + } +} + +func TestAnalyze_EndToEnd(t *testing.T) { + // Mock LLM server + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := map[string]interface{}{ + "choices": []map[string]interface{}{ + { + "message": map[string]interface{}{ + "content": `{"summary":"mock","affected_modules":["pkg/test"],"coverage":[{"type":"bvt","status":"covered","description":"ok"},{"type":"stability","status":"not_related","description":"n/a"},{"type":"chaos","status":"not_related","description":"n/a"},{"type":"bigdata","status":"not_related","description":"n/a"},{"type":"pitr","status":"not_related","description":"n/a"},{"type":"snapshot","status":"not_related","description":"n/a"}],"suggested_cases":[]}`, + }, + }, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + // Create temp repo structure + tmp := t.TempDir() + // Skills + skillsDir := filepath.Join(tmp, "docs", "ai-skills") + if err := os.MkdirAll(skillsDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillsDir, "testing-guide.md"), []byte("guide"), 0o644); err != nil { + t.Fatal(err) + } + // Cases + casesDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(casesDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(casesDir, "test.test"), []byte("SELECT 1;"), 0o644); err != nil { + t.Fatal(err) + } + + // This test requires `gh` CLI to get the diff - skip if not available + if _, err := os.Stat("/opt/homebrew/bin/gh"); err != nil { + t.Skip("gh CLI not available, skipping end-to-end test") + } + + cfg := Config{ + Repo: "matrixorigin/matrixone", + RepoRoot: tmp, + APIURL: srv.URL, + APIKey: "test-key", + Model: "test-model", + } + + a := New(cfg) + report, err := a.Analyze(context.Background(), 24088) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if report.PRNumber != 24088 { + t.Errorf("PRNumber = %d", report.PRNumber) + } + if report.Summary != "mock" { + t.Errorf("Summary = %q", report.Summary) + } + if len(report.Coverage) != 6 { + t.Errorf("Coverage len = %d", len(report.Coverage)) + } +} diff --git a/pkg/testinfra/dedup/dedup.go b/pkg/testinfra/dedup/dedup.go new file mode 100644 index 0000000000000..ebc1bf59b652d --- /dev/null +++ b/pkg/testinfra/dedup/dedup.go @@ -0,0 +1,200 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dedup + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// Deduplicator checks whether suggested cases overlap with existing test files. +type Deduplicator struct { + repoRoot string + // existing holds normalized SQL statement sets per category dir. + // Key: directory path relative to repo root. Value: set of normalized SQL strings. + existing map[string]map[string]bool +} + +// New creates a Deduplicator that lazily loads existing cases. +func New(repoRoot string) *Deduplicator { + return &Deduplicator{ + repoRoot: repoRoot, + existing: make(map[string]map[string]bool), + } +} + +// Filter removes suggested cases whose SQL content significantly overlaps +// with existing test files in the target directory. +// Returns the filtered list and a list of skip reasons. +func (d *Deduplicator) Filter(cases []types.SuggestedCase) (kept []types.SuggestedCase, skipped []string) { + for _, sc := range cases { + dir := d.targetDir(sc) + if dir == "" { + kept = append(kept, sc) + continue + } + + existingSQL := d.loadDir(dir) + newStatements := extractStatements(sc.Content) + + if d.isDuplicate(newStatements, existingSQL) { + skipped = append(skipped, sc.Category+"/"+sc.Filename+": overlaps with existing case") + continue + } + kept = append(kept, sc) + } + return +} + +// targetDir returns the actual directory to check for duplicates. +func (d *Deduplicator) targetDir(sc types.SuggestedCase) string { + switch sc.Type { + case types.TestBVT, types.TestPITR, types.TestSnapshot: + return filepath.Join("test", "distributed", "cases", sc.Category) + default: + // Nightly regression cases are in a separate repo; skip dedup for those. + return "" + } +} + +// loadDir lazily scans all .sql/.test files in dir and extracts normalized SQL. +func (d *Deduplicator) loadDir(relDir string) map[string]bool { + if stmts, ok := d.existing[relDir]; ok { + return stmts + } + + stmts := make(map[string]bool) + absDir := filepath.Join(d.repoRoot, relDir) + + _ = filepath.WalkDir(absDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return nil + } + name := entry.Name() + if !(strings.HasSuffix(name, ".sql") || strings.HasSuffix(name, ".test")) { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil + } + for _, stmt := range extractStatements(string(data)) { + stmts[stmt] = true + } + return nil + }) + + d.existing[relDir] = stmts + return stmts +} + +// isDuplicate returns true if more than half of the new SQL statements +// already appear in the existing set. +func (d *Deduplicator) isDuplicate(newStatements []string, existing map[string]bool) bool { + if len(newStatements) == 0 { + return false + } + + matches := 0 + for _, stmt := range newStatements { + if existing[stmt] { + matches++ + } + } + + // >50% overlap means duplicate + return matches*2 > len(newStatements) +} + +// extractStatements parses SQL text into normalized statement strings. +// Strips comments, whitespace, and mo-tester tags to focus on actual SQL. +func extractStatements(content string) []string { + var stmts []string + var current strings.Builder + + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + + // Skip empty lines, comments, and mo-tester tags + if trimmed == "" { + continue + } + if strings.HasPrefix(trimmed, "--") { + continue + } + if strings.HasPrefix(trimmed, "#") { + continue + } + + current.WriteString(trimmed) + current.WriteString(" ") + + // Statement ends with semicolon + if strings.HasSuffix(trimmed, ";") { + stmt := normalizeSQL(current.String()) + if stmt != "" && !isBoilerplate(stmt) { + stmts = append(stmts, stmt) + } + current.Reset() + } + } + + // Handle last statement without trailing semicolon + if current.Len() > 0 { + stmt := normalizeSQL(current.String()) + if stmt != "" && !isBoilerplate(stmt) { + stmts = append(stmts, stmt) + } + } + + return stmts +} + +// normalizeSQL lowercases and collapses whitespace for comparison. +func normalizeSQL(sql string) string { + sql = strings.ToLower(strings.TrimSpace(sql)) + // Collapse multiple spaces + for strings.Contains(sql, " ") { + sql = strings.ReplaceAll(sql, " ", " ") + } + return sql +} + +// isBoilerplate returns true for common setup/teardown SQL that shouldn't +// contribute to duplicate detection. +func isBoilerplate(sql string) bool { + prefixes := []string{ + "drop table ", + "drop database ", + "drop account ", + "drop pitr ", + "drop snapshot ", + "use ", + "set ", + "begin;", + "commit;", + "rollback;", + } + for _, p := range prefixes { + if strings.HasPrefix(sql, p) { + return true + } + } + return false +} diff --git a/pkg/testinfra/dedup/dedup_test.go b/pkg/testinfra/dedup/dedup_test.go new file mode 100644 index 0000000000000..9f2c4a9a3f8b1 --- /dev/null +++ b/pkg/testinfra/dedup/dedup_test.go @@ -0,0 +1,245 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dedup + +import ( + "os" + "path/filepath" + "testing" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +func TestExtractStatements(t *testing.T) { + content := `-- this is a comment +-- @bvt:issue#123 +drop table if exists t1; +create table t1 (a int, b varchar(100)); +insert into t1 values (1, 'hello'); +SELECT a, b FROM t1 WHERE a > 0; +-- @bvt:issue +drop table t1; +` + stmts := extractStatements(content) + + // Should skip: comments, drop table if exists (boilerplate), drop table (boilerplate) + // Should keep: create table, insert, select + if len(stmts) != 3 { + t.Fatalf("got %d statements, want 3: %v", len(stmts), stmts) + } + if stmts[0] != "create table t1 (a int, b varchar(100));" { + t.Errorf("stmts[0] = %q", stmts[0]) + } + if stmts[1] != "insert into t1 values (1, 'hello');" { + t.Errorf("stmts[1] = %q", stmts[1]) + } + if stmts[2] != "select a, b from t1 where a > 0;" { + t.Errorf("stmts[2] = %q", stmts[2]) + } +} + +func TestExtractStatements_MultiLine(t *testing.T) { + content := `CREATE TABLE t1 ( + a INT, + b VARCHAR(100) +); +SELECT * FROM t1;` + stmts := extractStatements(content) + if len(stmts) != 2 { + t.Fatalf("got %d statements, want 2: %v", len(stmts), stmts) + } +} + +func TestNormalizeSQL(t *testing.T) { + cases := []struct { + in, want string + }{ + {" SELECT * FROM t1 ; ", "select * from t1 ;"}, + {"INSERT INTO t1 VALUES (1);", "insert into t1 values (1);"}, + } + for _, tc := range cases { + got := normalizeSQL(tc.in) + if got != tc.want { + t.Errorf("normalizeSQL(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestIsBoilerplate(t *testing.T) { + boilerplate := []string{ + "drop table if exists t1;", + "drop database if exists db1;", + "use db1;", + "set global var = 1;", + "begin;", + "commit;", + } + for _, s := range boilerplate { + if !isBoilerplate(s) { + t.Errorf("%q should be boilerplate", s) + } + } + + notBoilerplate := []string{ + "create table t1 (a int);", + "select * from t1;", + "insert into t1 values (1);", + } + for _, s := range notBoilerplate { + if isBoilerplate(s) { + t.Errorf("%q should NOT be boilerplate", s) + } + } +} + +func TestFilter_NoDuplicate(t *testing.T) { + tmp := t.TempDir() + catDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(catDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(catDir, "existing.sql"), []byte("SELECT 1;\nSELECT 2;\n"), 0o644); err != nil { + t.Fatal(err) + } + + d := New(tmp) + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "new_case.sql", + Content: "CREATE TABLE t1 (a int);\nSELECT a FROM t1;\ndrop table t1;\n", + }, + } + + kept, skipped := d.Filter(cases) + if len(kept) != 1 { + t.Errorf("kept = %d, want 1", len(kept)) + } + if len(skipped) != 0 { + t.Errorf("skipped = %v", skipped) + } +} + +func TestFilter_Duplicate(t *testing.T) { + tmp := t.TempDir() + catDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(catDir, 0o755); err != nil { + t.Fatal(err) + } + // Existing file has these SQL statements + existing := "CREATE TABLE t1 (a INT);\nINSERT INTO t1 VALUES (1);\nSELECT * FROM t1;\ndrop table t1;\n" + if err := os.WriteFile(filepath.Join(catDir, "existing.sql"), []byte(existing), 0o644); err != nil { + t.Fatal(err) + } + + d := New(tmp) + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "duplicate.sql", + // Same SQL, slightly different formatting + Content: "create table t1 (a int);\ninsert into t1 values (1);\nselect * from t1;\ndrop table t1;\n", + }, + } + + kept, skipped := d.Filter(cases) + if len(kept) != 0 { + t.Errorf("kept = %d, want 0 (should be filtered)", len(kept)) + } + if len(skipped) != 1 { + t.Errorf("skipped = %d, want 1", len(skipped)) + } +} + +func TestFilter_PartialOverlap(t *testing.T) { + tmp := t.TempDir() + catDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(catDir, 0o755); err != nil { + t.Fatal(err) + } + existing := "SELECT 1;\nSELECT 2;\n" + if err := os.WriteFile(filepath.Join(catDir, "existing.sql"), []byte(existing), 0o644); err != nil { + t.Fatal(err) + } + + d := New(tmp) + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "partial.sql", + // 4 statements, only 1 overlaps (SELECT 1) = 25% < 50%, so keep + Content: "SELECT 1;\nSELECT 3;\nSELECT 4;\nSELECT 5;\n", + }, + } + + kept, skipped := d.Filter(cases) + if len(kept) != 1 { + t.Errorf("kept = %d, want 1 (25%% overlap should pass)", len(kept)) + } + if len(skipped) != 0 { + t.Errorf("skipped = %v", skipped) + } +} + +func TestFilter_NightlySkipsDedup(t *testing.T) { + tmp := t.TempDir() + d := New(tmp) + + cases := []types.SuggestedCase{ + { + Type: types.TestStability, + Category: "sysbench/new_scenario", + Filename: "run.yml", + Content: "duration: 10\n", + }, + { + Type: types.TestChaos, + Category: "mo-chaos-config", + Filename: "new_chaos.yaml", + Content: "chaos: test\n", + }, + } + + kept, skipped := d.Filter(cases) + if len(kept) != 2 { + t.Errorf("kept = %d, want 2 (nightly cases skip dedup)", len(kept)) + } + if len(skipped) != 0 { + t.Errorf("skipped = %v", skipped) + } +} + +func TestFilter_EmptyNewCase(t *testing.T) { + tmp := t.TempDir() + d := New(tmp) + + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "empty.sql", + Content: "-- just comments\n-- nothing here\n", + }, + } + + kept, _ := d.Filter(cases) + // Empty SQL → 0 statements → not duplicate → kept + if len(kept) != 1 { + t.Errorf("kept = %d, want 1", len(kept)) + } +} diff --git a/pkg/testinfra/llm/client.go b/pkg/testinfra/llm/client.go new file mode 100644 index 0000000000000..d283905d63a7a --- /dev/null +++ b/pkg/testinfra/llm/client.go @@ -0,0 +1,104 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package llm + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// Message represents a chat message. +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type chatRequest struct { + Model string `json:"model"` + Messages []Message `json:"messages"` + Temperature float64 `json:"temperature"` +} + +type chatResponse struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` +} + +// Client is an OpenAI-compatible LLM API client. +type Client struct { + apiURL string + apiKey string + model string +} + +// NewClient creates an LLM client. +func NewClient(apiURL, apiKey, model string) *Client { + return &Client{apiURL: apiURL, apiKey: apiKey, model: model} +} + +// Chat sends messages to the LLM and returns the response text. +func (c *Client) Chat(ctx context.Context, messages []Message) (string, error) { + reqBody := chatRequest{ + Model: c.model, + Messages: messages, + Temperature: 0.1, + } + + data, err := json.Marshal(reqBody) + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("marshal request: %v", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "POST", c.apiURL, bytes.NewReader(data)) + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("create request: %v", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("LLM API call failed: %v", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("read response: %v", err) + } + + if resp.StatusCode != http.StatusOK { + return "", moerr.NewInternalErrorNoCtxf("LLM API %d: %s", resp.StatusCode, string(body)) + } + + var chatResp chatResponse + if err := json.Unmarshal(body, &chatResp); err != nil { + return "", moerr.NewInternalErrorNoCtxf("parse response: %v", err) + } + + if len(chatResp.Choices) == 0 { + return "", moerr.NewInternalErrorNoCtx("LLM returned no choices") + } + + return chatResp.Choices[0].Message.Content, nil +} diff --git a/pkg/testinfra/llm/client_test.go b/pkg/testinfra/llm/client_test.go new file mode 100644 index 0000000000000..1651a1b1d0007 --- /dev/null +++ b/pkg/testinfra/llm/client_test.go @@ -0,0 +1,118 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package llm + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestNewClient(t *testing.T) { + c := NewClient("http://example.com/v1", "key123", "gpt-4") + if c.apiURL != "http://example.com/v1" { + t.Errorf("apiURL = %q", c.apiURL) + } + if c.apiKey != "key123" { + t.Errorf("apiKey = %q", c.apiKey) + } + if c.model != "gpt-4" { + t.Errorf("model = %q", c.model) + } +} + +func TestChat_Success(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify request + if r.Method != "POST" { + t.Errorf("method = %s, want POST", r.Method) + } + if r.Header.Get("Authorization") != "Bearer test-key" { + t.Errorf("auth header = %q", r.Header.Get("Authorization")) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Errorf("content-type = %q", r.Header.Get("Content-Type")) + } + + var req chatRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if req.Model != "test-model" { + t.Errorf("model = %q, want test-model", req.Model) + } + if len(req.Messages) != 1 { + t.Fatalf("messages len = %d, want 1", len(req.Messages)) + } + if req.Messages[0].Content != "hello" { + t.Errorf("message content = %q", req.Messages[0].Content) + } + + resp := chatResponse{ + Choices: []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + }{ + {Message: struct { + Content string `json:"content"` + }{Content: "world"}}, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + c := NewClient(srv.URL, "test-key", "test-model") + result, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hello"}}) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if result != "world" { + t.Errorf("result = %q, want %q", result, "world") + } +} + +func TestChat_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte("rate limited")) + })) + defer srv.Close() + + c := NewClient(srv.URL, "key", "model") + _, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}) + if err == nil { + t.Fatal("expected error for 429") + } +} + +func TestChat_EmptyChoices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := chatResponse{Choices: nil} + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + c := NewClient(srv.URL, "key", "model") + _, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}) + if err == nil { + t.Fatal("expected error for empty choices") + } +} diff --git a/pkg/testinfra/scanner/scanner.go b/pkg/testinfra/scanner/scanner.go new file mode 100644 index 0000000000000..c80304066f49e --- /dev/null +++ b/pkg/testinfra/scanner/scanner.go @@ -0,0 +1,172 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// CaseInfo holds the test files for a single BVT category. +type CaseInfo struct { + Category string + Files []string +} + +// ScanBVTCases scans test/distributed/cases/ and returns case info per category. +func ScanBVTCases(repoRoot string) ([]CaseInfo, error) { + casesDir := filepath.Join(repoRoot, "test", "distributed", "cases") + entries, err := os.ReadDir(casesDir) + if err != nil { + return nil, moerr.NewInternalErrorNoCtxf("read cases dir: %v", err) + } + + result := make([]CaseInfo, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + info := CaseInfo{Category: entry.Name()} + catDir := filepath.Join(casesDir, entry.Name()) + _ = filepath.WalkDir(catDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil || d.IsDir() { + return nil + } + if strings.HasSuffix(d.Name(), ".test") || strings.HasSuffix(d.Name(), ".sql") { + info.Files = append(info.Files, d.Name()) + } + return nil + }) + sort.Strings(info.Files) + result = append(result, info) + } + return result, nil +} + +// FormatCaseSummary returns a compact text listing of BVT cases for the LLM prompt. +// Uses compact mode by default to save tokens. +func FormatCaseSummary(cases []CaseInfo) string { + var b strings.Builder + for _, c := range cases { + b.WriteString(c.Category) + b.WriteString("/") + b.WriteString(strconv.Itoa(len(c.Files))) + b.WriteString(" ") + } + return b.String() +} + +// ReadSampleCases reads the first N lines from BVT .test/.sql files +// in categories relevant to the given changed paths. +// This gives the LLM concrete examples of existing case format. +func ReadSampleCases(repoRoot string, changedPaths []string, maxSamples int, maxLinesPerFile int) string { + if maxSamples <= 0 { + maxSamples = 3 + } + if maxLinesPerFile <= 0 { + maxLinesPerFile = 40 + } + + // Determine relevant categories from changed paths + categories := inferCategories(changedPaths) + if len(categories) == 0 { + return "" + } + + casesDir := filepath.Join(repoRoot, "test", "distributed", "cases") + var b strings.Builder + sampled := 0 + + for _, cat := range categories { + if sampled >= maxSamples { + break + } + catDir := filepath.Join(casesDir, cat) + entries, err := os.ReadDir(catDir) + if err != nil { + continue + } + for _, e := range entries { + if sampled >= maxSamples { + break + } + name := e.Name() + if e.IsDir() || !(strings.HasSuffix(name, ".test") || strings.HasSuffix(name, ".sql")) { + continue + } + data, err := os.ReadFile(filepath.Join(catDir, name)) + if err != nil { + continue + } + lines := strings.Split(string(data), "\n") + if len(lines) > maxLinesPerFile { + lines = lines[:maxLinesPerFile] + } + b.WriteString("### ") + b.WriteString(cat) + b.WriteString("/") + b.WriteString(name) + b.WriteString("\n```sql\n") + b.WriteString(strings.Join(lines, "\n")) + b.WriteString("\n```\n\n") + sampled++ + } + } + return b.String() +} + +// inferCategories maps changed file paths to likely BVT categories. +func inferCategories(paths []string) []string { + seen := make(map[string]bool) + mappings := map[string][]string{ + "pkg/sql/plan": {"function", "optimizer", "subquery", "join", "window", "cte"}, + "pkg/sql/compile": {"function", "expression", "dml"}, + "pkg/sql/parsers": {"ddl", "dml", "expression"}, + "pkg/vm/engine/tae": {"disttae", "dml"}, + "pkg/vm/engine/dist": {"disttae"}, + "pkg/txn": {"pessimistic_transaction", "optimistic"}, + "pkg/lockservice": {"pessimistic_transaction"}, + "pkg/frontend": {"snapshot", "pitr", "tenant"}, + "pkg/cdc": {"dml"}, + "pkg/fulltext": {"fulltext"}, + "pkg/vectorindex": {"vector"}, + "pkg/vectorize": {"function", "expression"}, + "pkg/partition": {"ddl"}, + "pkg/backup": {"snapshot", "pitr"}, + } + + for _, p := range paths { + for prefix, cats := range mappings { + if strings.HasPrefix(p, prefix) { + for _, c := range cats { + seen[c] = true + } + } + } + } + + result := make([]string, 0, len(seen)) + for c := range seen { + result = append(result, c) + } + sort.Strings(result) + return result +} diff --git a/pkg/testinfra/scanner/scanner_test.go b/pkg/testinfra/scanner/scanner_test.go new file mode 100644 index 0000000000000..ab176e90ef6c8 --- /dev/null +++ b/pkg/testinfra/scanner/scanner_test.go @@ -0,0 +1,169 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestScanBVTCases(t *testing.T) { + // Create a temporary directory structure + tmp := t.TempDir() + casesDir := filepath.Join(tmp, "test", "distributed", "cases") + funcDir := filepath.Join(casesDir, "function") + ddlDir := filepath.Join(casesDir, "ddl") + + if err := os.MkdirAll(funcDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(ddlDir, 0o755); err != nil { + t.Fatal(err) + } + + // Create test files + for _, f := range []string{"test1.test", "test2.test", "other.txt"} { + if err := os.WriteFile(filepath.Join(funcDir, f), []byte("SELECT 1;"), 0o644); err != nil { + t.Fatal(err) + } + } + for _, f := range []string{"create.sql", "drop.test"} { + if err := os.WriteFile(filepath.Join(ddlDir, f), []byte("CREATE TABLE t(a INT);"), 0o644); err != nil { + t.Fatal(err) + } + } + + cases, err := ScanBVTCases(tmp) + if err != nil { + t.Fatalf("ScanBVTCases: %v", err) + } + + if len(cases) != 2 { + t.Fatalf("got %d categories, want 2", len(cases)) + } + + // Find each category + caseMap := make(map[string]CaseInfo) + for _, c := range cases { + caseMap[c.Category] = c + } + + funcInfo, ok := caseMap["function"] + if !ok { + t.Fatal("missing 'function' category") + } + // Should have 2 .test files, not the .txt + if len(funcInfo.Files) != 2 { + t.Errorf("function files = %d, want 2: %v", len(funcInfo.Files), funcInfo.Files) + } + + ddlInfo, ok := caseMap["ddl"] + if !ok { + t.Fatal("missing 'ddl' category") + } + // Should have 1 .sql + 1 .test = 2 + if len(ddlInfo.Files) != 2 { + t.Errorf("ddl files = %d, want 2: %v", len(ddlInfo.Files), ddlInfo.Files) + } +} + +func TestScanBVTCases_MissingDir(t *testing.T) { + _, err := ScanBVTCases("/nonexistent/path") + if err == nil { + t.Fatal("expected error for missing dir") + } +} + +func TestFormatCaseSummary(t *testing.T) { + cases := []CaseInfo{ + {Category: "function", Files: []string{"a.test", "b.test", "c.test"}}, + {Category: "ddl", Files: []string{"x.sql"}}, + } + + summary := FormatCaseSummary(cases) + + if !strings.Contains(summary, "function/3") { + t.Errorf("summary should contain 'function/3', got: %q", summary) + } + if !strings.Contains(summary, "ddl/1") { + t.Errorf("summary should contain 'ddl/1', got: %q", summary) + } +} + +func TestFormatCaseSummary_Empty(t *testing.T) { + summary := FormatCaseSummary(nil) + if summary != "" { + t.Errorf("expected empty summary, got: %q", summary) + } +} + +func TestReadSampleCases(t *testing.T) { + tmp := t.TempDir() + funcDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(funcDir, 0o755); err != nil { + t.Fatal(err) + } + + content := "-- test sample\nSELECT 1;\nSELECT 2;\ndrop table if exists t1;\n" + if err := os.WriteFile(filepath.Join(funcDir, "sample.test"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + // Changed path in pkg/sql/plan should map to "function" category + result := ReadSampleCases(tmp, []string{"pkg/sql/plan/some_file.go"}, 3, 40) + if !strings.Contains(result, "function/sample.test") { + t.Errorf("should include function/sample.test, got: %q", result) + } + if !strings.Contains(result, "SELECT 1;") { + t.Errorf("should include file content, got: %q", result) + } +} + +func TestReadSampleCases_NoMatch(t *testing.T) { + tmp := t.TempDir() + result := ReadSampleCases(tmp, []string{"unknown/path.go"}, 3, 40) + if result != "" { + t.Errorf("expected empty result for unknown path, got: %q", result) + } +} + +func TestInferCategories(t *testing.T) { + cases := []struct { + path string + want []string + }{ + {"pkg/sql/plan/foo.go", []string{"cte", "function", "join", "optimizer", "subquery", "window"}}, + {"pkg/txn/foo.go", []string{"optimistic", "pessimistic_transaction"}}, + {"pkg/frontend/foo.go", []string{"pitr", "snapshot", "tenant"}}, + {"pkg/fulltext/foo.go", []string{"fulltext"}}, + {"pkg/cdc/foo.go", []string{"dml"}}, + {"unknown/foo.go", nil}, + } + + for _, tc := range cases { + got := inferCategories([]string{tc.path}) + if len(got) != len(tc.want) { + t.Errorf("inferCategories(%q) = %v, want %v", tc.path, got, tc.want) + continue + } + for i, g := range got { + if g != tc.want[i] { + t.Errorf("inferCategories(%q)[%d] = %q, want %q", tc.path, i, g, tc.want[i]) + } + } + } +} diff --git a/pkg/testinfra/types/types.go b/pkg/testinfra/types/types.go new file mode 100644 index 0000000000000..949385b99bda3 --- /dev/null +++ b/pkg/testinfra/types/types.go @@ -0,0 +1,61 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +// CoverageStatus represents the test coverage state for a test category. +type CoverageStatus string + +const ( + StatusCovered CoverageStatus = "covered" + StatusNeedsAttention CoverageStatus = "needs_attention" + StatusNotRelated CoverageStatus = "not_related" +) + +// TestType represents the 6 test categories in MO's test infrastructure. +type TestType string + +const ( + TestBVT TestType = "bvt" + TestStability TestType = "stability" + TestChaos TestType = "chaos" + TestBigData TestType = "bigdata" + TestPITR TestType = "pitr" + TestSnapshot TestType = "snapshot" +) + +// CoverageItem describes coverage status for one test type. +type CoverageItem struct { + Type TestType `json:"type"` + Status CoverageStatus `json:"status"` + Description string `json:"description"` +} + +// SuggestedCase is a test case the AI suggests adding. +type SuggestedCase struct { + Type TestType `json:"type"` + Category string `json:"category"` + Filename string `json:"filename"` + Content string `json:"content"` + Reason string `json:"reason"` +} + +// CoverageReport is the structured output of the analysis. +type CoverageReport struct { + PRNumber int `json:"pr_number"` + Summary string `json:"summary"` + AffectedModules []string `json:"affected_modules"` + Coverage []CoverageItem `json:"coverage"` + SuggestedCases []SuggestedCase `json:"suggested_cases"` +} diff --git a/pkg/testinfra/types/types_test.go b/pkg/testinfra/types/types_test.go new file mode 100644 index 0000000000000..5e21e85fe3926 --- /dev/null +++ b/pkg/testinfra/types/types_test.go @@ -0,0 +1,101 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "encoding/json" + "testing" +) + +func TestCoverageStatusValues(t *testing.T) { + if StatusCovered != "covered" { + t.Errorf("StatusCovered = %q, want %q", StatusCovered, "covered") + } + if StatusNeedsAttention != "needs_attention" { + t.Errorf("StatusNeedsAttention = %q, want %q", StatusNeedsAttention, "needs_attention") + } + if StatusNotRelated != "not_related" { + t.Errorf("StatusNotRelated = %q, want %q", StatusNotRelated, "not_related") + } +} + +func TestTestTypeValues(t *testing.T) { + expected := map[TestType]string{ + TestBVT: "bvt", + TestStability: "stability", + TestChaos: "chaos", + TestBigData: "bigdata", + TestPITR: "pitr", + TestSnapshot: "snapshot", + } + for tt, want := range expected { + if string(tt) != want { + t.Errorf("TestType %v = %q, want %q", tt, string(tt), want) + } + } +} + +func TestCoverageReportJSON(t *testing.T) { + report := CoverageReport{ + PRNumber: 123, + Summary: "test summary", + AffectedModules: []string{"pkg/sql"}, + Coverage: []CoverageItem{ + {Type: TestBVT, Status: StatusCovered, Description: "ok"}, + {Type: TestStability, Status: StatusNotRelated, Description: "n/a"}, + }, + SuggestedCases: []SuggestedCase{ + { + Type: TestBVT, + Category: "function", + Filename: "test1.test", + Content: "SELECT 1;", + Reason: "missing coverage", + }, + }, + } + + data, err := json.Marshal(report) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + var got CoverageReport + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + if got.PRNumber != 123 { + t.Errorf("PRNumber = %d, want 123", got.PRNumber) + } + if got.Summary != "test summary" { + t.Errorf("Summary = %q, want %q", got.Summary, "test summary") + } + if len(got.Coverage) != 2 { + t.Fatalf("Coverage len = %d, want 2", len(got.Coverage)) + } + if got.Coverage[0].Type != TestBVT { + t.Errorf("Coverage[0].Type = %q, want %q", got.Coverage[0].Type, TestBVT) + } + if got.Coverage[0].Status != StatusCovered { + t.Errorf("Coverage[0].Status = %q, want %q", got.Coverage[0].Status, StatusCovered) + } + if len(got.SuggestedCases) != 1 { + t.Fatalf("SuggestedCases len = %d, want 1", len(got.SuggestedCases)) + } + if got.SuggestedCases[0].Content != "SELECT 1;" { + t.Errorf("SuggestedCases[0].Content = %q", got.SuggestedCases[0].Content) + } +} diff --git a/pkg/testinfra/writer/writer.go b/pkg/testinfra/writer/writer.go new file mode 100644 index 0000000000000..0e9a73b54a912 --- /dev/null +++ b/pkg/testinfra/writer/writer.go @@ -0,0 +1,120 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package writer + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// WriteCases writes suggested BVT cases to disk under test/distributed/cases/. +// Returns the list of written file paths (relative to repoRoot). +// WriteCases writes suggested test cases to disk. +// BVT/PITR/Snapshot → test/distributed/cases/{category}/ +// Stability/Chaos/BigData → output/{type}/{category}/ (for manual copy to nightly repo) +func WriteCases(repoRoot string, cases []types.SuggestedCase) ([]string, error) { + if len(cases) == 0 { + return nil, nil + } + + var written []string + for _, sc := range cases { + if sc.Category == "" || sc.Filename == "" || sc.Content == "" { + continue + } + + var dir, relPath string + switch sc.Type { + case types.TestBVT, types.TestPITR, types.TestSnapshot: + // These go directly into the matrixone repo + dir = filepath.Join(repoRoot, "test", "distributed", "cases", sc.Category) + relPath = filepath.Join("test", "distributed", "cases", sc.Category, sc.Filename) + case types.TestStability, types.TestChaos, types.TestBigData: + // These are staged in output/ for manual copy to mo-nightly-regression + dir = filepath.Join(repoRoot, "output", string(sc.Type), sc.Category) + relPath = filepath.Join("output", string(sc.Type), sc.Category, sc.Filename) + default: + continue + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return written, moerr.NewInternalErrorNoCtxf("mkdir %s: %v", dir, err) + } + + fpath := filepath.Join(dir, sc.Filename) + + // Don't overwrite existing files + if _, err := os.Stat(fpath); err == nil { + fmt.Fprintf(os.Stderr, "skip: %s already exists\n", relPath) + continue + } + + if err := os.WriteFile(fpath, []byte(sc.Content), 0o644); err != nil { + return written, moerr.NewInternalErrorNoCtxf("write %s: %v", relPath, err) + } + written = append(written, relPath) + } + return written, nil +} + +// CreatePR creates a new branch, commits the written case files, and opens a PR. +// Returns the PR URL. +func CreatePR(repoRoot string, repo string, prNumber int, files []string) (string, error) { + if len(files) == 0 { + return "", moerr.NewInternalErrorNoCtx("no files to commit") + } + + branch := "testinfra/pr-" + strconv.Itoa(prNumber) + "-cases" + title := fmt.Sprintf("test: add BVT cases for PR #%d", prNumber) + body := fmt.Sprintf("Auto-generated BVT test cases for PR #%d.\n\nGenerated by `mo-testplan`.", prNumber) + + type step struct { + name string + args []string + } + + steps := []step{ + {"git", []string{"-C", repoRoot, "checkout", "-b", branch}}, + } + for _, f := range files { + steps = append(steps, step{"git", []string{"-C", repoRoot, "add", f}}) + } + steps = append(steps, + step{"git", []string{"-C", repoRoot, "commit", "-m", title}}, + step{"git", []string{"-C", repoRoot, "push", "origin", branch}}, + step{"gh", []string{"pr", "create", "--repo", repo, "--head", branch, + "--title", title, "--body", body}}, + ) + + for _, s := range steps { + cmd := exec.Command(s.name, s.args...) + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("%s %v failed: %v", s.name, s.args, err) + } + // The last command (gh pr create) outputs the PR URL + if s.name == "gh" { + return string(out), nil + } + } + return "", nil +} diff --git a/pkg/testinfra/writer/writer_test.go b/pkg/testinfra/writer/writer_test.go new file mode 100644 index 0000000000000..3e543e039d687 --- /dev/null +++ b/pkg/testinfra/writer/writer_test.go @@ -0,0 +1,235 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package writer + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +func TestWriteCases_Basic(t *testing.T) { + tmp := t.TempDir() + + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "decimal_compare.test", + Content: "-- test decimal vs integer compare\nSELECT 1.0 = 1;\nSELECT 2.5 > 2;\n", + Reason: "missing coverage", + }, + { + Type: types.TestBVT, + Category: "window", + Filename: "window_decimal.test", + Content: "-- test window function with decimal\nSELECT SUM(1.0) OVER();\n", + Reason: "missing coverage", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + + if len(written) != 2 { + t.Fatalf("written = %d, want 2", len(written)) + } + + // Verify file content + data, err := os.ReadFile(filepath.Join(tmp, "test", "distributed", "cases", "function", "decimal_compare.test")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != cases[0].Content { + t.Errorf("content mismatch: %q", string(data)) + } + + // Verify second file + data2, err := os.ReadFile(filepath.Join(tmp, "test", "distributed", "cases", "window", "window_decimal.test")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data2) != cases[1].Content { + t.Errorf("content mismatch: %q", string(data2)) + } +} + +func TestWriteCases_SkipExisting(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + // Pre-create the file + if err := os.WriteFile(filepath.Join(dir, "existing.test"), []byte("original"), 0o644); err != nil { + t.Fatal(err) + } + + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "existing.test", + Content: "new content", + Reason: "test", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + + if len(written) != 0 { + t.Errorf("should skip existing file, written = %v", written) + } + + // Verify original content preserved + data, err := os.ReadFile(filepath.Join(dir, "existing.test")) + if err != nil { + t.Fatal(err) + } + if string(data) != "original" { + t.Errorf("existing file was overwritten: %q", string(data)) + } +} + +func TestWriteCases_SkipNonBVT(t *testing.T) { + tmp := t.TempDir() + + // Stability cases should go to output/ directory, not be skipped + cases := []types.SuggestedCase{ + { + Type: types.TestStability, + Category: "sysbench/mixed", + Filename: "run.yml", + Content: "duration: 10\ntransaction:\n - name: test\n", + Reason: "test", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if len(written) != 1 { + t.Fatalf("should write stability case to output/, written = %v", written) + } + if !strings.Contains(written[0], "output") { + t.Errorf("stability case should be in output/, got: %s", written[0]) + } + + // Verify file exists + data, err := os.ReadFile(filepath.Join(tmp, written[0])) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !strings.Contains(string(data), "duration") { + t.Errorf("content mismatch: %q", string(data)) + } +} + +func TestWriteCases_PITRAndSnapshot(t *testing.T) { + tmp := t.TempDir() + + cases := []types.SuggestedCase{ + { + Type: types.TestPITR, + Category: "pitr", + Filename: "pitr_new.sql", + Content: "CREATE PITR p1;\nSELECT 1;\ndrop pitr p1;\n", + Reason: "test", + }, + { + Type: types.TestSnapshot, + Category: "snapshot", + Filename: "snap_new.sql", + Content: "CREATE SNAPSHOT s1;\nSELECT 1;\n", + Reason: "test", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if len(written) != 2 { + t.Fatalf("written = %d, want 2", len(written)) + } + // Both should be in test/distributed/cases/ + for _, w := range written { + if !strings.HasPrefix(w, "test/distributed/cases/") { + t.Errorf("PITR/Snapshot should be in test/distributed/cases/, got: %s", w) + } + } +} + +func TestWriteCases_ChaosOutput(t *testing.T) { + tmp := t.TempDir() + + cases := []types.SuggestedCase{ + { + Type: types.TestChaos, + Category: "mo-chaos-config", + Filename: "chaos_new_scenario.yaml", + Content: "chaos:\n cm-chaos:\n - name: test\n", + Reason: "test", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if len(written) != 1 { + t.Fatalf("written = %d, want 1", len(written)) + } + if !strings.Contains(written[0], "output/chaos") { + t.Errorf("chaos case should be in output/chaos/, got: %s", written[0]) + } +} + +func TestWriteCases_SkipEmpty(t *testing.T) { + tmp := t.TempDir() + + cases := []types.SuggestedCase{ + {Type: types.TestBVT, Category: "", Filename: "a.test", Content: "x"}, + {Type: types.TestBVT, Category: "f", Filename: "", Content: "x"}, + {Type: types.TestBVT, Category: "f", Filename: "a.test", Content: ""}, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if len(written) != 0 { + t.Errorf("should skip cases with empty fields, written = %v", written) + } +} + +func TestWriteCases_EmptyList(t *testing.T) { + written, err := WriteCases(t.TempDir(), nil) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if written != nil { + t.Errorf("expected nil, got %v", written) + } +}