diff --git a/cmd/mo-testplan/main.go b/cmd/mo-testplan/main.go new file mode 100644 index 0000000000000..e5fca973ccda9 --- /dev/null +++ b/cmd/mo-testplan/main.go @@ -0,0 +1,177 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// mo-testplan analyzes PR test coverage against MO's 6 test types +// (BVT, stability, chaos, big data, PITR, snapshot) using AI. +// +// Usage: +// +// mo-testplan --pr 24088 +// mo-testplan --pr 24088 --json +// mo-testplan --pr 24088 --write +// mo-testplan --pr 24088 --write --create-pr +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/matrixorigin/matrixone/pkg/testinfra/analyzer" + "github.com/matrixorigin/matrixone/pkg/testinfra/writer" +) + +const ( + defaultAPIURL = "https://models.github.ai/inference/chat/completions" + defaultModel = "openai/gpt-4o-mini" +) + +func main() { + var ( + prNumber int + repo string + apiURL string + apiKey string + model string + jsonOut bool + write bool + createPR bool + ) + + flag.IntVar(&prNumber, "pr", 0, "PR number to analyze") + flag.StringVar(&repo, "repo", "matrixorigin/matrixone", "GitHub repository (owner/name)") + flag.StringVar(&apiURL, "api-url", defaultAPIURL, "LLM API endpoint") + flag.StringVar(&apiKey, "api-key", "", "API key (default: $GITHUB_TOKEN or gh auth token)") + flag.StringVar(&model, "model", defaultModel, "LLM model name") + flag.BoolVar(&jsonOut, "json", false, "output raw JSON instead of formatted report") + flag.BoolVar(&write, "write", false, "write suggested BVT cases to test/distributed/cases/") + flag.BoolVar(&createPR, "create-pr", false, "create a PR with written cases (implies --write)") + flag.Parse() + + if prNumber == 0 { + fmt.Fprintf(os.Stderr, "Usage: mo-testplan --pr [--repo owner/name] [--json]\n") + fmt.Fprintf(os.Stderr, "\nFlags:\n") + flag.PrintDefaults() + os.Exit(1) + } + + if apiKey == "" { + apiKey = resolveAPIKey() + } + if apiKey == "" { + fmt.Fprintf(os.Stderr, "Error: no API key found.\n") + fmt.Fprintf(os.Stderr, "Set --api-key, $GITHUB_TOKEN, or login with: gh auth login\n") + os.Exit(1) + } + + repoRoot := detectRepoRoot() + if repoRoot == "" { + fmt.Fprintf(os.Stderr, "Error: cannot detect repo root (no go.mod found).\n") + os.Exit(1) + } + + cfg := analyzer.Config{ + Repo: repo, + RepoRoot: repoRoot, + APIURL: apiURL, + APIKey: apiKey, + Model: model, + } + + a := analyzer.New(cfg) + report, err := a.Analyze(context.Background(), prNumber) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + if jsonOut { + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + os.Stdout.Write(data) + os.Stdout.WriteString("\n") + } else { + os.Stdout.WriteString(analyzer.FormatReport(report)) + } + + // --create-pr implies --write + if createPR { + write = true + } + + if write && len(report.SuggestedCases) > 0 { + written, err := writer.WriteCases(repoRoot, report.SuggestedCases) + if err != nil { + fmt.Fprintf(os.Stderr, "Error writing cases: %v\n", err) + os.Exit(1) + } + if len(written) > 0 { + os.Stdout.WriteString("\n### 已写入文件\n") + for _, f := range written { + os.Stdout.WriteString("- ") + os.Stdout.WriteString(f) + os.Stdout.WriteString("\n") + } + + if createPR { + prURL, err := writer.CreatePR(repoRoot, repo, prNumber, written) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating PR: %v\n", err) + os.Exit(1) + } + os.Stdout.WriteString("\n### 已创建 PR\n") + os.Stdout.WriteString(strings.TrimSpace(prURL)) + os.Stdout.WriteString("\n") + } + } else { + os.Stdout.WriteString("\n无新增文件(已有 case 或建议为空)\n") + } + } +} + +func resolveAPIKey() string { + if key := os.Getenv("GITHUB_TOKEN"); key != "" { + return key + } + out, err := exec.Command("gh", "auth", "token").Output() + if err == nil { + return strings.TrimSpace(string(out)) + } + return "" +} + +func detectRepoRoot() string { + dir, err := os.Getwd() + if err != nil { + return "" + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + return "" + } + dir = parent + } +} diff --git a/docs/ai-skills/architecture.md b/docs/ai-skills/architecture.md new file mode 100644 index 0000000000000..abbffd7090959 --- /dev/null +++ b/docs/ai-skills/architecture.md @@ -0,0 +1,47 @@ +# MO 整体架构 + +## 核心组件 + +MO 采用存算分离架构,由 4 个核心服务组成: + +| 组件 | 包路径 | 职责 | +|------|--------|------| +| **CN (Compute Node)** | `pkg/cnservice/` | SQL 解析、计划生成、编译执行。无状态,可水平扩展 | +| **TN (Transaction Node)** | `pkg/tnservice/` | 事务处理、数据持久化、WAL 管理 | +| **LogService** | `pkg/logservice/` | 分布式日志(基于 Dragonboat/Raft),提供数据一致性保证 | +| **Proxy** | `pkg/proxy/` | 连接路由、负载均衡,将客户端请求分发到不同 CN | + +## 辅助组件 + +| 组件 | 包路径 | 职责 | +|------|--------|------| +| **HAKeeper** | `pkg/hakeeper/` | 集群编排与健康检查,基于 Raft 状态机 | +| **ClusterService** | `pkg/clusterservice/` | 集群拓扑管理、节点发现 | +| **Gossip** | `pkg/gossip/` | 节点间元数据传播(基于 memberlist) | +| **TaskService** | `pkg/taskservice/` | 后台任务调度(CDC、自增 ID 等) | +| **Bootstrap** | `pkg/bootstrap/` | 集群初始化、版本管理 | + +## 启动流程 + +入口:`cmd/mo-service/main.go` → `launch.go` + +``` +启动 → 读配置 → 启动 LogService → 启动 TN → 启动 CN → 启动 Proxy +``` + +配置文件位于 `etc/launch/`: +- `launch.toml` — 集群协调配置 +- `cn.toml` — CN 配置(端口、引擎类型) +- `tn.toml` — TN 配置 +- `log.toml` — LogService 配置 + +## 数据流 + +``` +客户端 → Proxy → CN (SQL解析/计划/编译/执行) + ↕ + DisttaE (分布式引擎) ←→ TN (事务/存储) + ↕ ↕ + FileService LogService + (对象存储) (WAL/Raft) +``` diff --git a/docs/ai-skills/backup-restore.md b/docs/ai-skills/backup-restore.md new file mode 100644 index 0000000000000..df2bfdc8c93b7 --- /dev/null +++ b/docs/ai-skills/backup-restore.md @@ -0,0 +1,58 @@ +# 备份恢复(PITR / Snapshot) + +## 概述 + +MO 支持两种备份恢复机制: +- **PITR (Point-In-Time Recovery)** — 恢复到任意时间点 +- **Snapshot** — 恢复到指定快照 + +## PITR + +### 原理 +- 基于 Logtail 的增量日志回放 +- 记录事务提交时间戳,恢复时回放到指定时间点 +- 依赖 TN 的 checkpoint + WAL + +### 关键路径 +- `pkg/backup/` — 备份配置与元数据 +- `pkg/frontend/` — 前端 PITR SQL 命令处理 +- `pkg/vm/engine/tae/` — 存储层 checkpoint 管理 +- `pkg/logservice/` — WAL 日志持久化 + +### SQL 语法 +```sql +CREATE PITR pitr_name FOR ACCOUNT account_name RANGE value unit; +ALTER PITR pitr_name ...; +DROP PITR pitr_name; +RESTORE ACCOUNT account_name FROM PITR pitr_name TIMESTAMP '2024-01-01 00:00:00'; +``` + +## Snapshot + +### 原理 +- 创建数据库/表的一致性快照 +- 基于 MVCC 时间戳实现 +- 快照元数据存储在系统表中 + +### SQL 语法 +```sql +CREATE SNAPSHOT snapshot_name FOR ACCOUNT account_name; +RESTORE ACCOUNT account_name FROM SNAPSHOT snapshot_name; +DROP SNAPSHOT snapshot_name; +``` + +## Backup 包(`pkg/backup/`) + +- `BackupType` — 备份类型 +- `BackupTs` — 备份时间戳 +- `BackupObject` (objectio) — 备份对象元数据 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| PITR 核心逻辑 | PITR 测试; BVT: pitr | +| Snapshot 核心逻辑 | Snapshot 测试; BVT: snapshot | +| Logtail/Checkpoint | PITR + Snapshot 都受影响 | +| 备份元数据 | BVT: pitr, snapshot | +| 多租户备份 | BVT: tenant; PITR/Snapshot 多租户场景 | diff --git a/docs/ai-skills/cdc.md b/docs/ai-skills/cdc.md new file mode 100644 index 0000000000000..e170622d919ab --- /dev/null +++ b/docs/ai-skills/cdc.md @@ -0,0 +1,47 @@ +# CDC(Change Data Capture) + +## 概述 + +MO 的 CDC 模块基于 Logtail 捕获数据变更,将变更同步到下游系统。 + +## 核心组件(`pkg/cdc/`) + +| 组件 | 职责 | +|------|------| +| `CDCStateManager` | 每个 publication 的状态跟踪 | +| `CDCStatementBuilder` | 为下游生成复制 SQL | +| `WatermarkUpdater` | 同步进度(水位线)管理 | +| Table Scanner | 扫描表变更 | + +## 工作流程 + +``` +表数据变更 → Logtail 捕获 → CDC 模块消费 → 生成下游 SQL → 同步到下游 + ↓ + WatermarkUpdater 更新水位线 +``` + +## 后台任务类型 + +- `JT_CDC_GetOrAddCommittedWM` — 获取/添加已提交水位线 +- `JT_CDC_CommittingWM` — 提交中的水位线 +- `JT_CDC_UpdateWMErrMsg` — 更新水位线错误信息 +- `JT_CDC_RemoveCachedWM` — 清除缓存水位线 + +## SQL 语法 + +```sql +CREATE CDC cdc_name ...; +ALTER CDC cdc_name ...; +DROP CDC cdc_name; +SHOW CDC; +``` + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| CDC 状态管理 | BVT: cdc | +| 水位线逻辑 | BVT: cdc; 稳定性: 长时间 CDC 同步 | +| Logtail 消费 | CDC + PITR + Snapshot | +| 杀节点后 CDC 恢复 | Chaos: 杀 CN/TN 后 CDC 恢复 | diff --git a/docs/ai-skills/fileservice.md b/docs/ai-skills/fileservice.md new file mode 100644 index 0000000000000..a49e3578ef6a7 --- /dev/null +++ b/docs/ai-skills/fileservice.md @@ -0,0 +1,43 @@ +# FileService + +## 概述 + +FileService 是 MO 的对象存储抽象层,屏蔽底层存储差异(本地磁盘 / S3 / MinIO)。 + +## 核心接口(`pkg/fileservice/`) + +| 接口 | 职责 | +|------|------| +| `FileService` | 基础读写操作 | +| `MutableFileService` | 追加/删除操作 | +| `ReaderWriterFileService` | 流式 I/O | +| `CacheDataAllocator` | 内存管理 | +| `FileCache` | 本地缓存层 | +| `ETLFileService` | ETL 专用操作 | + +## 存储后端 + +- **本地文件系统** — 开发/测试环境 +- **S3 / MinIO** — 生产环境对象存储 +- **HTTP** — 远程文件访问 + +## 配置 + +```toml +# etc/launch/tn.toml +[fileservice.s3] +bucket = "my-bucket" +key-prefix = "mo-data" +endpoint = "http://minio:9000" +``` + +本地存储模式:数据存储在 `mo-data/` 目录下。 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| 读写逻辑 | BVT: load_data, stage | +| 缓存层 | 稳定性: tpch(大量读取)| +| S3 对接 | 大数据量测试(云端 load)| +| ETL 功能 | BVT: load_data | diff --git a/docs/ai-skills/fulltext-vector.md b/docs/ai-skills/fulltext-vector.md new file mode 100644 index 0000000000000..8385e16a95264 --- /dev/null +++ b/docs/ai-skills/fulltext-vector.md @@ -0,0 +1,44 @@ +# 全文检索 + 向量索引 + +## 全文检索(`pkg/fulltext/`) + +- **核心类型:** + - `FullTextScoreAlgo` — 相关性评分算法 + - `FullTextParserParam` — 解析配置 + - `FullTextBooleanOperator` — 布尔查询算子(AND/OR/NOT) + +- **SQL 语法:** +```sql +CREATE FULLTEXT INDEX idx ON t(col); +SELECT * FROM t WHERE MATCH(col) AGAINST('keyword' IN BOOLEAN MODE); +``` + +## 向量索引(`pkg/vectorindex/`) + +- **核心类型:** + - `IndexTableConfig` — 索引配置 + - `IvfflatIndexConfig` — IVF-Flat 算法配置 + - `VectorIndexCdc[T]` — 向量索引的 CDC 更新 + - `SearchResultIf` — 搜索结果接口 + +- **支持算法:** IVF-Flat, HNSW + +- **SQL 语法:** +```sql +CREATE INDEX idx USING IVFFLAT ON t(embedding_col) LISTS = 100; +SELECT * FROM t ORDER BY l2_distance(embedding_col, '[1,2,3]') LIMIT 10; +``` + +## 向量化层(`pkg/vectorize/`) + +- 表达式向量化执行 +- 批量向量运算优化 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| 全文检索 | BVT: fulltext; 稳定性: fulltext-vector | +| 向量索引 | BVT: vector; 稳定性: vector IVF+DML concurrency | +| 向量索引 CDC | Chaos: fulltext 故障场景 | +| 评分算法 | BVT: fulltext | diff --git a/docs/ai-skills/module-test-mapping.md b/docs/ai-skills/module-test-mapping.md new file mode 100644 index 0000000000000..be046570419b3 --- /dev/null +++ b/docs/ai-skills/module-test-mapping.md @@ -0,0 +1,141 @@ +# 模块 → 测试关联 + +本文档描述 MO 各代码模块与 6 类测试的关联关系。AI 分析 diff 时,根据变更文件所属模块查找需要关注的测试类型。 + +## SQL 引擎 + +### pkg/sql/plan/ +- **BVT:** optimizer, plan_cache, join, subquery, cte, recursive_cte, hint, window +- **稳定性:** tpch(复杂查询计划) +- **大数据:** 大数据量下查询计划选择 + +### pkg/sql/compile/ +- **BVT:** ddl, dml, function, expression +- **稳定性:** tpch, tpcc + +### pkg/sql/colexec/ +- **BVT:** function, expression, join, window + +### pkg/sql/parsers/ +- **BVT:** ddl, dml, prepare + +## 存储引擎 + +### pkg/vm/engine/disttae/ +- **BVT:** disttae, pessimistic_transaction, optimistic +- **稳定性:** tpcc, sysbench +- **Chaos:** 杀 TN 后数据一致性 +- **Snapshot:** 存储层快照一致性 +- **PITR:** 存储层恢复 + +### pkg/vm/engine/tae/ +- **BVT:** disttae, pessimistic_transaction +- **稳定性:** tpcc(频繁写入 flush) +- **Chaos:** 杀 TN 后恢复 +- **PITR:** checkpoint 恢复 + +### pkg/objectio/ +- **BVT:** load_data, disttae +- **大数据:** 大规模数据读写 + +## 事务 + +### pkg/txn/ +- **BVT:** pessimistic_transaction, optimistic +- **稳定性:** tpcc, sysbench(高并发事务) +- **Chaos:** 杀 TN 后事务恢复 + +### pkg/lockservice/ +- **BVT:** pessimistic_transaction +- **Chaos:** 死锁场景、杀节点后锁恢复 + +## 前端 + +### pkg/frontend/ +- **BVT:** security, tenant, zz_accesscontrol, snapshot, pitr, system_variable, set +- **PITR:** 前端 PITR 命令处理 +- **Snapshot:** 前端 snapshot 命令处理 + +## 数据类型 + +### pkg/container/ +- **BVT:** dtype, array, vector + +## 全文 / 向量 + +### pkg/fulltext/ +- **BVT:** fulltext +- **稳定性:** fulltext-vector + +### pkg/vectorindex/ +- **BVT:** vector +- **稳定性:** vector IVF+DML concurrency +- **Chaos:** fulltext 故障场景 + +## CDC + +### pkg/cdc/ +- **BVT:** cdc +- **稳定性:** 长时间 CDC 同步 +- **Chaos:** 杀 CN/TN 后 CDC 恢复 + +## 备份恢复 + +### pkg/backup/ +- **PITR:** 备份元数据 +- **Snapshot:** 快照管理 + +## 基础设施 + +### pkg/fileservice/ +- **BVT:** stage, load_data +- **大数据:** 云端数据 load + +### pkg/partition/ +- **BVT:** ddl, dml(分区表) + +### pkg/proxy/ +- **BVT:** tenant(路由) +- **Chaos:** 杀 CN 后连接恢复 + +### pkg/bootstrap/ +- **BVT:** system + +### pkg/catalog/ +- **BVT:** ddl, database, table, system + +### pkg/udf/ +- **BVT:** udf + +### pkg/stage/ +- **BVT:** stage + +### pkg/logservice/ +- **Chaos:** 杀 LogService 场景 +- **PITR:** WAL 日志完整性 + +## BVT 测试目录索引 + +``` +test/distributed/cases/ +├── ddl/ # CREATE/ALTER/DROP 语句 +├── dml/ # INSERT/UPDATE/DELETE +├── optimizer/ # 查询优化器 +├── join/ # JOIN 语法和优化 +├── subquery/ # 子查询 +├── window/ # 窗口函数 +├── function/ # 内置函数 +├── expression/ # 表达式计算 +├── pessimistic_transaction/ # 悲观事务 +├── optimistic/ # 乐观事务 +├── disttae/ # 分布式存储引擎 +├── fulltext/ # 全文检索 +├── vector/ # 向量索引 +├── cdc/ # CDC +├── pitr/ # PITR +├── snapshot/ # Snapshot +├── load_data/ # 数据加载 +├── tenant/ # 多租户 +├── security/ # 安全/权限 +└── ... # 其他 50+ 类别 +``` diff --git a/docs/ai-skills/multi-cn.md b/docs/ai-skills/multi-cn.md new file mode 100644 index 0000000000000..7ef810da79f8f --- /dev/null +++ b/docs/ai-skills/multi-cn.md @@ -0,0 +1,46 @@ +# 多 CN 架构 + +## 概述 + +MO 的 CN 是无状态的,可以水平扩展。Proxy 负责将客户端连接路由到合适的 CN。 + +## Proxy(`pkg/proxy/`) + +- `Router` 接口 — CN 选择策略 +- `RefreshableRouter` — 动态刷新集群拓扑 +- 连接路由 + 负载均衡 +- 支持基于标签(label)的路由 + +## ClusterService(`pkg/clusterservice/`) + +- `MOCluster` 接口 — 集群拓扑和元数据 +- `ClusterClient` 接口 — CN 间通信 +- `labelSupportedClient` — 基于标签路由的客户端 + +## QueryService(`pkg/queryservice/`) + +- `QueryService` 接口 — 分布式查询处理 +- `Session` 接口 — 会话管理 +- 节点间消息路由 + +## Gossip(`pkg/gossip/`) + +- 基于 memberlist 的节点发现 +- 节点心跳 + 元数据交换 + +## 多 CN 部署配置 + +参考 `etc/launch-multi-cn/`,每个 CN 独立配置: +- 不同端口 +- 相同的 TN 和 LogService 地址 +- 可配置不同标签用于路由 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| Proxy 路由逻辑 | BVT: tenant(多租户路由)| +| 连接均衡 | 稳定性: sysbench(高并发连接)| +| 杀 CN 恢复 | Chaos: 杀 CN 场景 | +| 节点发现 | Chaos: 网络分区场景 | +| 跨 CN 查询 | 大数据量测试(分布式执行)| diff --git a/docs/ai-skills/sql-engine.md b/docs/ai-skills/sql-engine.md new file mode 100644 index 0000000000000..c5a6d151bac60 --- /dev/null +++ b/docs/ai-skills/sql-engine.md @@ -0,0 +1,55 @@ +# SQL 引擎 + +## 执行流水线 + +``` +SQL 文本 → Parser → AST → Planner → Plan Tree → Compiler → Scope/Pipeline → Executor → 结果 +``` + +## 各阶段详解 + +### Parser(`pkg/sql/parsers/`) +- SQL → AST 转换(基于 bison/flex) +- MySQL 方言兼容 +- 输出:语法树节点 + +### Planner(`pkg/sql/plan/`) +- 逻辑计划生成 +- 类型推导与验证 +- 查询优化(代价估算、谓词下推、连接重排) +- 分区裁剪集成 +- 关键子模块: + - `function/` — 函数注册与类型匹配 + - `opt_misc.go` — 各种优化 pass(HAVING remap、窗口函数 remap 等) + +### Compiler(`pkg/sql/compile/`) +- Plan Tree → 可执行 Pipeline +- 核心类型:`Compile` 结构体 +- `Scope` — 执行单元(本地/远程/Load/表函数) +- 负责跨 CN 的远程执行规划 +- DDL/DML/TCL 翻译 + +### Executor(`pkg/sql/colexec/`) +- 列式执行引擎 +- `ExpressionExecutor` 接口 +- 实现:`FixedVectorExpressionExecutor`, `FunctionExpressionExecutor`, `ColumnExpressionExecutor` +- 批量向量化处理 + +## VM Pipeline(`pkg/vm/`) + +``` +Scope → Operator → Operator → ... → Output +``` + +- 每个 SQL 算子对应一个 Operator(scan, filter, join, agg, sort, limit...) +- Pipeline 模式执行(pull-based) + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| Parser | BVT: ddl, dml, prepare | +| Plan 优化 | BVT: optimizer, hint, plan_cache, join, subquery, cte, window | +| Compile/Scope | BVT: expression, function; 稳定性: tpch | +| 类型系统 | BVT: dtype, pg_cast, array, vector | +| 执行引擎 | 大数据量测试(大规模扫描/聚合) | diff --git a/docs/ai-skills/storage-engine.md b/docs/ai-skills/storage-engine.md new file mode 100644 index 0000000000000..a2d5457353c43 --- /dev/null +++ b/docs/ai-skills/storage-engine.md @@ -0,0 +1,54 @@ +# 存储引擎 + +## 引擎分层 + +``` +CN 侧: DisttaE (分布式事务引擎) + ↕ +TN 侧: TAE (本地存储引擎) + ↕ +底层: FileService (对象存储抽象) +``` + +## DisttaE(分布式 TAE) + +- **路径:** `pkg/vm/engine/disttae/` +- **角色:** CN 侧使用的分布式引擎,负责跨节点事务读写 +- **核心类型:** + - `Engine` — 分布式引擎实例 + - `txnDatabase` — 事务级数据库视图 + - `txnTable` / `txnTableDelegate` — 事务级表操作 +- **特性:** MVCC 快照隔离,支持乐观/悲观事务 + +## TAE(Transaction Aware Engine) + +- **路径:** `pkg/vm/engine/tae/` +- **角色:** TN 侧的本地存储引擎 +- **特性:** + - 块状列存格式 + - Checkpoint / Flush 管理 + - Logtail 增量变更捕获 + +## ObjectIO(对象存储模型) + +- **路径:** `pkg/objectio/` +- **核心类型:** + - `BlockInfo` — 存储块元数据 + - `ObjectLocation` — 对象存储位置引用 + - `BackupObject` — 备份对象元数据 +- **说明:** 通过 FileService 执行实际的对象 I/O + +## Engine 接口 + +- **路径:** `pkg/vm/engine/types.go` +- **抽象:** Engine → Database → Relation(Table) → Reader +- **可插拔:** 支持 DisttaE、MemoryEngine (测试用) 等实现 + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| DisttaE 事务逻辑 | BVT: disttae, pessimistic_transaction, optimistic | +| TAE flush/checkpoint | 稳定性: tpcc/sysbench 长时间写入 | +| ObjectIO 读写 | BVT: load_data; 大数据量测试 | +| Logtail 变更捕获 | PITR, Snapshot, CDC | diff --git a/docs/ai-skills/test-env-setup.md b/docs/ai-skills/test-env-setup.md new file mode 100644 index 0000000000000..9b26b8f9d2098 --- /dev/null +++ b/docs/ai-skills/test-env-setup.md @@ -0,0 +1,76 @@ +# 测试环境配置 + +## 本地单机部署 + +```bash +# 编译 +make build + +# 启动(单机集群:LogService + TN + CN) +./mo-service -launch etc/launch/launch.toml +``` + +默认端口:`6001`(MySQL 协议) + +## Docker 部署 + +```bash +# 使用 docker compose +cd etc/launch-tae-compose/ +docker compose up -d +``` + +## 本地多 CN 部署 + +配置目录:`etc/launch-multi-cn/` + +```bash +# 启动集群(1 LogService + 1 TN + 多 CN) +./mo-service -launch etc/launch-multi-cn/launch.toml +``` + +每个 CN 使用不同端口基数,共享同一个 TN 和 LogService。 + +## 加速 Flush / GC(测试调参) + +在 TN 配置中调整: + +```toml +# tn.toml - 加快 flush +[tn.Txn.Storage] +# 减小 checkpoint 间隔 +checkpoint-flush-interval = "1s" +# 减小 GC 间隔 +gc-check-interval = "1s" +``` + +**目的:** 测试时加快数据落盘和垃圾回收,缩短测试等待时间。 + +## BVT 测试运行 + +```bash +# 需要 mo-tester 工具 +cd mo-tester/ +./run.sh -p /path/to/matrixone/test/distributed/cases/optimizer/ + +# 跑单个 .test 文件 +./run.sh -p /path/to/test/distributed/cases/window/window_basic.test +``` + +## 稳定性/Chaos/大数据测试 + +这些测试通过 mo-nightly-regression 仓库的 GitHub Actions Workflow 运行: + +```bash +# 触发方式:手动或定时(nightly) +# 仓库:matrixorigin/mo-nightly-regression +# 分支:main (稳定性/chaos/PITR/snapshot), big_data (大数据量) +``` + +## 环境变量 + +| 变量 | 说明 | +|------|------| +| `MO_WORKSPACE` | MO 工作目录 | +| `MO_LOG_LEVEL` | 日志级别 (debug/info/warn/error) | +| `CGO_CFLAGS` | CGO 编译标志(含 thirdparties 路径)| diff --git a/docs/ai-skills/testing-guide.md b/docs/ai-skills/testing-guide.md new file mode 100644 index 0000000000000..c1b096c9998ce --- /dev/null +++ b/docs/ai-skills/testing-guide.md @@ -0,0 +1,172 @@ +# 测试体系总览 + +## 6 类测试 + +### 1. BVT 测试(轻量级回归) +- **仓库:** matrixone +- **路径:** `test/distributed/cases/{category}/` +- **工具:** mo-tester +- **Case 格式:** `.test`/`.sql` + 对应 `.result` 文件 +- **运行时机:** 每个 PR 的 CI +- **运行命令:** `cd mo-tester && ./run.sh -n -g -p /path/to/test.test` +- **生成 result:** `./run.sh -m genrs -n -g -p /path/to/test.test` + +### 2. 稳定性测试(长时间运行) +- **仓库:** mo-nightly-regression (main) +- **Workflow:** `stability-test-on-distributed.yaml` +- **测试项:** TPCH, TPCC, Sysbench, Fulltext-vector, Vector IVF+DML +- **配置:** `cases/sysbench/{scenario}/run.yml` — mo-load YAML 配置 +- **参数:** TPCHScale, LoadDataScale, TPCC_WarehouseNum, OLTPThreads, RunMinutes +- **目的:** 验证长时间运行下的稳定性 + +### 3. Chaos 测试(故障注入) +- **仓库:** mo-nightly-regression (main) +- **配置:** `mo-chaos-config/chaos_regression.yaml` — Chaos Mesh YAML +- **工作负载:** `mo-chaos-config/chaos_test_case.yaml` — 测试任务列表 +- **故障类型:** + - `PodChaos/pod-kill` — 杀 CN/TN/LogService Pod + - `StressChaos/memory` — 内存压力 + - `NetworkChaos` — 网络分区 +- **测试任务:** mo-tpcc, mo-load(sysbench), mo-cdc-test, mo-vector-fulltext-test +- **工作负载配置格式:** `mo-chaos-config/mo-load-insert-case/run.yml` + +### 4. 大数据量测试 +- **仓库:** mo-nightly-regression (main) +- **Workflow:** `big-data-test.yml` +- **路径:** `tools/mo-regression-test/cases/{customer_name}/` +- **配置格式:** `mo_ddl.yaml`(DDL source), `mo_load_*.yaml`(数据加载), `q00.sql`(查询) +- **Suite:** suite_1y(1年数据), suite_10y(10年数据), all +- **数据来源:** COS 对象存储 + load data + +### 5. PITR 测试 +- **仓库:** matrixone(BVT 级别)+ mo-nightly-regression(完整流程) +- **BVT 路径:** `test/distributed/cases/pitr/` — pitr.sql, pitr_basic.sql, pitr_inherit.sql +- **内容:** CREATE PITR → 操作数据 → RESTORE → 验证数据一致性 + +### 6. Snapshot 测试 +- **仓库:** matrixone(BVT 级别)+ mo-nightly-regression(完整流程) +- **BVT 路径:** `test/distributed/cases/snapshot/` — 多层级 snapshot 测试 +- **场景:** cluster/account/database/table 级别的 snapshot 创建和恢复 +- **内容:** CREATE SNAPSHOT → 操作数据 → RESTORE ACCOUNT → 验证 + +## BVT Case 格式详解 + +### 文件结构 +- `.test`/`.sql` — 测试文件(SQL + mo-tester 标签) +- `.result` — 期望输出(含列元数据 `column[type,precision,scale]`) +- 文件对应关系:`func_sum.test` ↔ `func_sum.result` + +### 标签语法 + +**文件级标签:** +```sql +-- @skip:issue#16438 -- 跳过整个文件 +--- @metacmp(false) -- 关闭元数据比较(三个 -) +``` + +**SQL 级标签:** +```sql +-- @bvt:issue#3185 -- 开始跳过区块 +SELECT * FROM t1; +-- @bvt:issue -- 结束跳过区块 + +-- @ignore:0 -- 忽略第 0 列(0-indexed) +select time(now()); + +-- @ignore:5,6 -- 忽略多列 +show publications; + +-- @sortkey:0,1 -- 按第 0,1 列排序后比较 +SELECT col1, col2 FROM t1; + +-- @regex("pattern", true) -- 结果必须匹配 pattern +show accounts; + +-- @regex("error", false) -- 结果不能匹配 pattern +SHOW TABLES; + +-- @metacmp(true) -- 单条 SQL 开启元数据比较 +SELECT * FROM t1; +``` + +**会话标签(并发测试):** +```sql +begin; +select * from t1; +-- @session:id=1&user=acc:admin&password=111{ +insert into t1 values (100); +-- @wait:0:commit -- 等待 session 0 commit +select * from t1; +-- @session} +commit; +``` + +### BVT Case 编写规范 +1. **自包含** — 每个 test 文件独立运行 +2. **资源清理** — 结尾 drop 创建的表/库 +3. **确定性** — 不确定列用 `@ignore`,不确定顺序用 `@sortkey` +4. **复用库** — 避免创建过多临时 database + +## 稳定性测试 Case 格式(mo-load) + +```yaml +# cases/sysbench/simple_insert_10_100000/run.yml +duration: 10 # 运行分钟数 +stdout: "console" +transaction: + - name: "simple_insert" + vuser: 10 # 并发数 + mode: 0 # 0=顺序执行, 1=封装为事务 + prepared: "false" + script: + - sql: "insert into sbtest{tbx} values({i_id},{kvalue},'...');" +``` + +## Chaos 测试配置格式 + +### 故障定义 (chaos_regression.yaml) +```yaml +chaos: + cm-chaos: + - name: task_kill_cn + kubectl_yaml: | + apiVersion: chaos-mesh.org/v1alpha1 + kind: PodChaos + spec: + action: pod-kill + selector: + labelSelectors: + 'matrixorigin.io/component': 'CNSet' # 或 DNSet/LogSet + times: 1 + interval: 30 + is_delete_after_apply: true +``` + +### 工作负载定义 (chaos_test_case.yaml) +```yaml +tasks: + - name: mo-tpcc + work-path: mo-tpcc + run-steps: + - command: ./runBenchmark.sh props.mo > tpcc.log 2>&1 + verify: + - command: ./runVerify.sh props.mo >> check.log 2>&1 + verify-mode: parallel # parallel(边跑边验)/after(跑完再验) +``` + +## 大数据测试 Case 格式 + +```yaml +# tools/mo-regression-test/cases/tpch_1g/mo_ddl.yaml +source_path: "/data/customer/tpch_1g/ddl/mo.sql" +load_type: "ddl" + +# tools/mo-regression-test/cases/tpch_1g/mo_load_server_serial.yaml +s3_path: "cos://bucket/path/to/data.csv" +load_type: "load_server" +count: "6001215" + +# tools/mo-regression-test/cases/tpch_1g/q00.sql +-- 标准 SQL 查询文件 +SELECT * FROM lineitem WHERE l_shipdate <= '1998-09-02'; +``` diff --git a/docs/ai-skills/transaction.md b/docs/ai-skills/transaction.md new file mode 100644 index 0000000000000..fe893d51bb29e --- /dev/null +++ b/docs/ai-skills/transaction.md @@ -0,0 +1,49 @@ +# 事务模型 + +## 两种事务模式 + +| 模式 | 说明 | 核心包 | +|------|------|--------| +| **悲观事务**(默认) | 写前加锁,冲突立即报错 | `pkg/lockservice/` | +| **乐观事务** | 提交时检测冲突,冲突则回滚 | `pkg/txn/` | + +## 核心组件 + +### TxnClient(`pkg/txn/client/`) +- `TxnClient` 接口 — 创建和管理事务 +- `TxnOperator` 接口 — 单个事务的读写操作 +- `TxnTimestampAware` — MVCC 时间戳管理 + +### LockService(`pkg/lockservice/`) +- `LockService` 接口 — 分布式锁协调 +- `LockTableAllocator` — 按表分配锁表实例 +- `lockTable` 接口 — 行级/表级锁 +- 死锁检测:`lockNode`, `deadlockTxn` + +## MVCC 机制 + +``` +写入 → 分配时间戳 → 写入新版本 → 提交 +读取 → 使用快照时间戳 → 读取可见版本 +``` + +- 快照时间戳由时钟服务提供 +- 行版本存储在 TAE 中 +- 增量变更通过 Logtail 系统传播 + +## 事务状态 + +``` +Active → Committing → Committed + ↘ Aborting → Aborted +``` + +## 与测试的关联 + +| 变更范围 | 影响的测试 | +|---------|----------| +| 悲观事务/锁服务 | BVT: pessimistic_transaction | +| 乐观事务/冲突检测 | BVT: optimistic | +| MVCC/快照隔离 | BVT: snapshot; Snapshot 测试 | +| 事务恢复 | Chaos: 杀 TN 后事务恢复; PITR | +| 高并发事务 | 稳定性: tpcc, sysbench | diff --git a/pkg/testinfra/analyzer/analyzer.go b/pkg/testinfra/analyzer/analyzer.go new file mode 100644 index 0000000000000..4771125c35586 --- /dev/null +++ b/pkg/testinfra/analyzer/analyzer.go @@ -0,0 +1,432 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analyzer + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/testinfra/dedup" + "github.com/matrixorigin/matrixone/pkg/testinfra/llm" + "github.com/matrixorigin/matrixone/pkg/testinfra/scanner" + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +const maxDiffLines = 300 + +// Config holds analyzer configuration. +type Config struct { + Repo string + RepoRoot string + APIURL string + APIKey string + Model string +} + +// Analyzer orchestrates PR analysis: diff → skills → cases → LLM → report. +type Analyzer struct { + config Config + client *llm.Client +} + +// New creates an Analyzer. +func New(cfg Config) *Analyzer { + return &Analyzer{ + config: cfg, + client: llm.NewClient(cfg.APIURL, cfg.APIKey, cfg.Model), + } +} + +// Analyze fetches the PR diff, loads skill docs and existing cases, +// calls the LLM, and returns a structured coverage report. +func (a *Analyzer) Analyze(ctx context.Context, prNumber int) (*types.CoverageReport, error) { + diff, err := a.getDiff(prNumber) + if err != nil { + return nil, err + } + + // Extract changed file paths from diff for category inference + changedPaths := extractChangedPaths(diff) + + skills, err := a.loadSkills() + if err != nil { + return nil, err + } + + cases, err := scanner.ScanBVTCases(a.config.RepoRoot) + if err != nil { + return nil, err + } + caseSummary := scanner.FormatCaseSummary(cases) + + // Read actual case examples from relevant categories + sampleCases := scanner.ReadSampleCases(a.config.RepoRoot, changedPaths, 3, 40) + + systemMsg := buildSystemPrompt(skills) + userMsg := buildUserPrompt(prNumber, diff, caseSummary, sampleCases) + + response, err := a.client.Chat(ctx, []llm.Message{ + {Role: "system", Content: systemMsg}, + {Role: "user", Content: userMsg}, + }) + if err != nil { + return nil, err + } + + report, err := parseReport(response, prNumber) + if err != nil { + return nil, err + } + + // Deduplicate suggested cases against existing test files + d := dedup.New(a.config.RepoRoot) + kept, skipped := d.Filter(report.SuggestedCases) + report.SuggestedCases = kept + if len(skipped) > 0 { + report.Summary += fmt.Sprintf("\n\n[Dedup] Filtered %d case(s): %s", len(skipped), strings.Join(skipped, "; ")) + } + + return report, nil +} + +// extractChangedPaths parses diff headers to get file paths. +func extractChangedPaths(diff string) []string { + var paths []string + for _, line := range strings.Split(diff, "\n") { + if strings.HasPrefix(line, "+++ b/") { + paths = append(paths, line[6:]) + } + } + return paths +} + +func (a *Analyzer) getDiff(prNumber int) (string, error) { + cmd := exec.Command("gh", "pr", "diff", strconv.Itoa(prNumber), "--repo", a.config.Repo) + out, err := cmd.Output() + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("gh pr diff failed for #%d: %v", prNumber, err) + } + + diff := string(out) + lines := strings.Split(diff, "\n") + if len(lines) > maxDiffLines { + diff = strings.Join(lines[:maxDiffLines], "\n") + diff += fmt.Sprintf("\n\n... (truncated, showing first %d of %d lines)", maxDiffLines, len(lines)) + } + return diff, nil +} + +// prioritySkills are the most important skill docs to include in the prompt. +// These are loaded first; others are added only if token budget allows. +var prioritySkills = []string{ + "module-test-mapping.md", + "testing-guide.md", +} + +func (a *Analyzer) loadSkills() (string, error) { + skillsDir := filepath.Join(a.config.RepoRoot, "docs", "ai-skills") + + var b strings.Builder + // Load priority docs first + for _, name := range prioritySkills { + data, err := os.ReadFile(filepath.Join(skillsDir, name)) + if err != nil { + continue + } + b.WriteString("## ") + b.WriteString(name) + b.WriteString("\n") + b.Write(data) + b.WriteString("\n\n") + } + + // Load remaining docs if total is under 6KB + entries, err := os.ReadDir(skillsDir) + if err != nil { + return b.String(), nil + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") { + continue + } + // Skip already loaded + skip := false + for _, p := range prioritySkills { + if entry.Name() == p { + skip = true + break + } + } + if skip { + continue + } + if b.Len() > 6000 { + break + } + data, err := os.ReadFile(filepath.Join(skillsDir, entry.Name())) + if err != nil { + continue + } + b.WriteString("## ") + b.WriteString(entry.Name()) + b.WriteString("\n") + b.Write(data) + b.WriteString("\n\n") + } + return b.String(), nil +} + +func buildSystemPrompt(skills string) string { + return `你是 MatrixOne 数据库的测试覆盖分析 Agent。分析 PR 代码变更,判断 6 类测试的覆盖情况,并**生成完整可运行的测试用例**。 + +## 6 类测试 + +### 1. BVT(轻量级回归,matrixone 仓库) +- 路径: test/distributed/cases/{category}/{name}.sql +- 格式: SQL + mo-tester 标签 +- 标签: @bvt:issue#N(跳过区块)、@session:id=N(并发会话)、@wait:N:commit(等待提交)、@sortkey:cols(排序)、@ignore:cols(忽略列)、@regex("pattern",true/false) +- 结尾必须 drop 创建的表/数据库 +- 每个文件必须自包含、可独立运行 +- 不确定顺序用 @sortkey,不确定值(如时间)用 @ignore + +### 2. 稳定性测试(mo-nightly-regression 仓库,main 分支) +- 路径: cases/sysbench/{scenario}/run.yml +- 格式: YAML (mo-load 配置) +- 内容: duration, transaction[].name/vuser/mode/script[].sql +- 场景: TPCH、TPCC、Sysbench(insert/select/update/delete/mixed)、Fulltext-vector + +### 3. Chaos 测试(mo-nightly-regression 仓库,main 分支) +- 故障定义: mo-chaos-config/chaos_regression.yaml (Chaos Mesh YAML) +- 工作负载: mo-chaos-config/chaos_test_case.yaml (tasks[].run-steps/verify) +- 故障类型: PodChaos/pod-kill(杀CN/TN/LogService)、StressChaos/memory、NetworkChaos +- 工作负载: mo-tpcc、mo-load(sysbench)、mo-cdc-test、mo-vector-fulltext-test + +### 4. 大数据量测试(mo-nightly-regression 仓库,main 分支) +- 路径: tools/mo-regression-test/cases/{test_name}/ +- 格式: mo_ddl.yaml + mo_load_*.yaml + q00.sql +- 数据来源: COS 对象存储 load + +### 5. PITR 测试(matrixone 仓库 BVT 级别) +- 路径: test/distributed/cases/pitr/ +- 格式: .sql + mo-tester 标签 +- 流程: CREATE PITR → 操作数据 → RESTORE → 验证一致性 + +### 6. Snapshot 测试(matrixone 仓库 BVT 级别) +- 路径: test/distributed/cases/snapshot/ +- 格式: .sql + mo-tester 标签 +- 场景: cluster/account/database/table 级别 snapshot 创建和恢复 + +## MO 知识文档 +` + skills +} + +func buildUserPrompt(prNumber int, diff, caseSummary, sampleCases string) string { + var b strings.Builder + b.WriteString(fmt.Sprintf("分析 PR #%d 的测试覆盖情况。\n\n", prNumber)) + b.WriteString("**关键原则**: 只为真正缺少测试覆盖的类型生成 case。如果某类测试已有覆盖或与本次变更完全无关,status 设为 covered 或 not_related,不要生成 case。不要盲目为每种测试类型都生成 case。\n\n") + + b.WriteString("## PR Diff:\n") + b.WriteString(diff) + b.WriteString("\n\n") + + b.WriteString("## 已有 BVT 测试目录(category/文件数):\n") + b.WriteString(caseSummary) + b.WriteString("\n\n") + + if sampleCases != "" { + b.WriteString("## 目标目录已有 case 示例(学习格式,避免与这些已有 case 重复):\n") + b.WriteString(sampleCases) + b.WriteString("\n") + } + + b.WriteString(`## 输出要求 + +请严格输出以下 JSON 格式(不要添加 markdown 代码块或其他文字): +{ + "summary": "一句话描述改了什么", + "affected_modules": ["受影响的模块路径"], + "coverage": [ + {"type": "bvt", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "stability", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "chaos", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "bigdata", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "pitr", "status": "covered|needs_attention|not_related", "description": "说明"}, + {"type": "snapshot", "status": "covered|needs_attention|not_related", "description": "说明"} + ], + "suggested_cases": [ + { + "type": "bvt|stability|chaos|bigdata|pitr|snapshot", + "category": "BVT子目录名 或 nightly场景名", + "filename": "文件名", + "content": "完整的可运行内容", + "reason": "补充原因" + } + ] +} + +## 生成 case 规则 + +### BVT case 生成规则: +1. 必须是完整可运行的 .sql 文件 +2. 参考上面的已有 case 示例的格式和风格 +3. 结尾必须 drop 创建的表/数据库 +4. 不确定的结果列用 @ignore,不确定的行序用 @sortkey +5. category 填对应的 BVT 子目录名(如 function, ddl, window, expression 等) + +### 稳定性 case 生成规则: +1. 格式为 mo-load run.yml +2. 包含 duration、transaction 配置 +3. category 填 "sysbench/{场景名}" + +### Chaos case 生成规则: +1. 如果需要新增故障类型,生成 Chaos Mesh YAML +2. 如果需要新增工作负载,生成 chaos_test_case.yaml 格式的 task +3. category 填 "mo-chaos-config" + +### PITR/Snapshot case 生成规则: +1. 与 BVT 同格式(.sql + mo-tester 标签) +2. PITR: CREATE PITR → DML → RESTORE → SELECT 验证 +3. Snapshot: CREATE SNAPSHOT → DML → RESTORE ACCOUNT → SELECT 验证 +4. category 分别填 "pitr" 或 "snapshot" + +注意: +- coverage 必须包含全部 6 种类型 +- suggested_cases 只列 status 为 needs_attention 的测试类型,绝不要为 not_related 或 covered 的类型生成 case +- 如果本次变更只涉及 BVT 相关代码(如 SQL 函数、DDL 等),就只生成 BVT case,不要强行凑其他类型 +- 生成的 SQL 不要与已有示例中的内容重复,关注新增/修改代码路径的覆盖 +- BVT/PITR/Snapshot case 的 content 必须是完整可运行的 SQL +- 稳定性/Chaos case 的 content 必须是完整的 YAML +`) + return b.String() +} + +func parseReport(response string, prNumber int) (*types.CoverageReport, error) { + jsonStr := extractJSON(response) + + var report types.CoverageReport + if err := json.Unmarshal([]byte(jsonStr), &report); err != nil { + return nil, moerr.NewInternalErrorNoCtxf("parse LLM JSON: %v\nRaw:\n%s", err, response) + } + report.PRNumber = prNumber + return &report, nil +} + +func extractJSON(s string) string { + s = strings.TrimSpace(s) + if idx := strings.Index(s, "```json"); idx >= 0 { + s = s[idx+7:] + if end := strings.Index(s, "```"); end >= 0 { + s = s[:end] + } + } else if idx := strings.Index(s, "```"); idx >= 0 { + s = s[idx+3:] + if nl := strings.Index(s, "\n"); nl >= 0 { + s = s[nl:] + } + if end := strings.Index(s, "```"); end >= 0 { + s = s[:end] + } + } + // Fix LLM producing string concatenation instead of proper JSON strings. + // e.g. "line1\n" +\n "line2\n" → "line1\nline2\n" + s = sanitizeJSONConcat(s) + return strings.TrimSpace(s) +} + +// sanitizeJSONConcat merges broken "..." + "..." patterns that some LLMs produce. +func sanitizeJSONConcat(s string) string { + for { + // Match: " +\n{whitespace}" or " +\n{whitespace}' + idx := strings.Index(s, "\" +") + if idx < 0 { + break + } + // Find the next quote after " +" + rest := s[idx+3:] + rest = strings.TrimLeft(rest, " \t\n\r") + if len(rest) == 0 || rest[0] != '"' { + break + } + // Merge: replace the closing quote + concat + opening quote with nothing + s = s[:idx] + rest[1:] + } + return s +} + +// FormatReport produces a terminal-friendly coverage report. +func FormatReport(r *types.CoverageReport) string { + var b strings.Builder + + b.WriteString(fmt.Sprintf("## PR #%d 测试覆盖分析\n\n", r.PRNumber)) + b.WriteString("### 变更摘要\n") + b.WriteString(r.Summary) + b.WriteString("\n\n") + + if len(r.AffectedModules) > 0 { + b.WriteString("### 受影响模块\n") + for _, m := range r.AffectedModules { + b.WriteString("- ") + b.WriteString(m) + b.WriteString("\n") + } + b.WriteString("\n") + } + + b.WriteString("### 6 类测试覆盖情况\n\n") + b.WriteString("| 测试类型 | 状态 | 说明 |\n") + b.WriteString("|---------|------|------|\n") + typeNames := map[types.TestType]string{ + types.TestBVT: "BVT", types.TestStability: "稳定性", + types.TestChaos: "Chaos", types.TestBigData: "大数据", + types.TestPITR: "PITR", types.TestSnapshot: "Snapshot", + } + for _, c := range r.Coverage { + icon := "➖" + switch c.Status { + case types.StatusCovered: + icon = "✅" + case types.StatusNeedsAttention: + icon = "⚠️" + } + name := typeNames[c.Type] + if name == "" { + name = string(c.Type) + } + b.WriteString(fmt.Sprintf("| %s | %s | %s |\n", name, icon, c.Description)) + } + b.WriteString("\n") + + if len(r.SuggestedCases) > 0 { + b.WriteString(fmt.Sprintf("### 建议补充的 Case(%d 个)\n\n", len(r.SuggestedCases))) + for i, sc := range r.SuggestedCases { + path := "test/distributed/cases/" + sc.Category + "/" + sc.Filename + b.WriteString(fmt.Sprintf("#### %d. %s\n", i+1, path)) + b.WriteString("原因: ") + b.WriteString(sc.Reason) + b.WriteString("\n\n```sql\n") + b.WriteString(sc.Content) + b.WriteString("\n```\n\n") + } + } + + return b.String() +} diff --git a/pkg/testinfra/analyzer/analyzer_test.go b/pkg/testinfra/analyzer/analyzer_test.go new file mode 100644 index 0000000000000..3b855cb2793f5 --- /dev/null +++ b/pkg/testinfra/analyzer/analyzer_test.go @@ -0,0 +1,379 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analyzer + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +func TestExtractChangedPaths(t *testing.T) { + diff := `diff --git a/pkg/sql/plan/foo.go b/pkg/sql/plan/foo.go +--- a/pkg/sql/plan/foo.go ++++ b/pkg/sql/plan/foo.go +@@ -1,3 +1,4 @@ ++import "fmt" +diff --git a/pkg/txn/bar.go b/pkg/txn/bar.go +--- a/pkg/txn/bar.go ++++ b/pkg/txn/bar.go +` + paths := extractChangedPaths(diff) + if len(paths) != 2 { + t.Fatalf("paths = %v, want 2", paths) + } + if paths[0] != "pkg/sql/plan/foo.go" { + t.Errorf("paths[0] = %q", paths[0]) + } + if paths[1] != "pkg/txn/bar.go" { + t.Errorf("paths[1] = %q", paths[1]) + } +} + +func TestExtractJSON_Plain(t *testing.T) { + input := `{"summary":"test","affected_modules":[],"coverage":[],"suggested_cases":[]}` + result := extractJSON(input) + if result != input { + t.Errorf("extractJSON plain = %q", result) + } +} + +func TestExtractJSON_Markdown(t *testing.T) { + input := "Some text\n```json\n{\"summary\":\"test\"}\n```\nmore text" + result := extractJSON(input) + if result != `{"summary":"test"}` { + t.Errorf("extractJSON markdown = %q", result) + } +} + +func TestExtractJSON_MarkdownNoLang(t *testing.T) { + input := "```\n{\"summary\":\"test\"}\n```" + result := extractJSON(input) + if result != `{"summary":"test"}` { + t.Errorf("extractJSON no lang = %q", result) + } +} + +func TestExtractJSON_ConcatStrings(t *testing.T) { + input := `{ + "content": "line1\n" + + "line2\n" + + "line3" +}` + result := extractJSON(input) + var m map[string]string + if err := json.Unmarshal([]byte(result), &m); err != nil { + t.Fatalf("should produce valid JSON: %v\nGot: %s", err, result) + } + if m["content"] != "line1\nline2\nline3" { + t.Errorf("content = %q", m["content"]) + } +} + +func TestSanitizeJSONConcat(t *testing.T) { + input := `"hello\n" + + "world"` + got := sanitizeJSONConcat(input) + want := `"hello\nworld"` + if got != want { + t.Errorf("sanitizeJSONConcat = %q, want %q", got, want) + } +} + +func TestBuildSystemPrompt(t *testing.T) { + result := buildSystemPrompt("skill content here") + if !strings.Contains(result, "MatrixOne") { + t.Error("system prompt should mention MatrixOne") + } + if !strings.Contains(result, "skill content here") { + t.Error("system prompt should include skills") + } + if !strings.Contains(result, "BVT") { + t.Error("system prompt should mention BVT") + } +} + +func TestBuildUserPrompt(t *testing.T) { + result := buildUserPrompt(123, "diff content", "function/10 ddl/5", "### sample\n```sql\nSELECT 1;\n```") + if !strings.Contains(result, "PR #123") { + t.Error("user prompt should mention PR number") + } + if !strings.Contains(result, "diff content") { + t.Error("user prompt should include diff") + } + if !strings.Contains(result, "function/10") { + t.Error("user prompt should include case summary") + } + if !strings.Contains(result, "sample") { + t.Error("user prompt should include sample cases") + } + if !strings.Contains(result, "stability") { + t.Error("user prompt should mention stability test type") + } + if !strings.Contains(result, "mo-chaos-config") { + t.Error("user prompt should mention chaos config") + } +} + +func TestLoadSkills(t *testing.T) { + tmp := t.TempDir() + skillsDir := filepath.Join(tmp, "docs", "ai-skills") + if err := os.MkdirAll(skillsDir, 0o755); err != nil { + t.Fatal(err) + } + + // Create priority skill doc + if err := os.WriteFile(filepath.Join(skillsDir, "module-test-mapping.md"), []byte("mapping content"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillsDir, "testing-guide.md"), []byte("testing content"), 0o644); err != nil { + t.Fatal(err) + } + // Create a non-priority doc + if err := os.WriteFile(filepath.Join(skillsDir, "architecture.md"), []byte("arch content"), 0o644); err != nil { + t.Fatal(err) + } + + a := &Analyzer{config: Config{RepoRoot: tmp}} + skills, err := a.loadSkills() + if err != nil { + t.Fatalf("loadSkills: %v", err) + } + + if !strings.Contains(skills, "mapping content") { + t.Error("should include module-test-mapping") + } + if !strings.Contains(skills, "testing content") { + t.Error("should include testing-guide") + } + if !strings.Contains(skills, "arch content") { + t.Error("should include architecture (under budget)") + } +} + +func TestLoadSkills_BudgetCap(t *testing.T) { + tmp := t.TempDir() + skillsDir := filepath.Join(tmp, "docs", "ai-skills") + if err := os.MkdirAll(skillsDir, 0o755); err != nil { + t.Fatal(err) + } + + // Create a priority doc that's already 5KB + bigContent := strings.Repeat("x", 5000) + if err := os.WriteFile(filepath.Join(skillsDir, "module-test-mapping.md"), []byte(bigContent), 0o644); err != nil { + t.Fatal(err) + } + // testing-guide pushes us over 6KB + if err := os.WriteFile(filepath.Join(skillsDir, "testing-guide.md"), []byte(strings.Repeat("y", 2000)), 0o644); err != nil { + t.Fatal(err) + } + // This extra doc should be skipped due to budget + if err := os.WriteFile(filepath.Join(skillsDir, "extra.md"), []byte("extra"), 0o644); err != nil { + t.Fatal(err) + } + + a := &Analyzer{config: Config{RepoRoot: tmp}} + skills, err := a.loadSkills() + if err != nil { + t.Fatalf("loadSkills: %v", err) + } + + // Priority docs always loaded + if !strings.Contains(skills, bigContent) { + t.Error("priority doc should always be loaded") + } + // Extra should be skipped (budget >6000) + if strings.Contains(skills, "extra") { + t.Error("extra doc should be skipped (over budget)") + } +} + +func TestLoadSkills_NoDir(t *testing.T) { + tmp := t.TempDir() + a := &Analyzer{config: Config{RepoRoot: tmp}} + skills, err := a.loadSkills() + if err != nil { + t.Fatalf("loadSkills: %v", err) + } + if skills != "" { + t.Errorf("expected empty skills, got %q", skills) + } +} + +func TestParseReport(t *testing.T) { + jsonStr := `{ + "summary": "fix decimal compare", + "affected_modules": ["pkg/sql"], + "coverage": [ + {"type": "bvt", "status": "covered", "description": "ok"}, + {"type": "stability", "status": "not_related", "description": "n/a"}, + {"type": "chaos", "status": "not_related", "description": "n/a"}, + {"type": "bigdata", "status": "not_related", "description": "n/a"}, + {"type": "pitr", "status": "not_related", "description": "n/a"}, + {"type": "snapshot", "status": "not_related", "description": "n/a"} + ], + "suggested_cases": [] + }` + + report, err := parseReport(jsonStr, 100) + if err != nil { + t.Fatalf("parseReport: %v", err) + } + if report.PRNumber != 100 { + t.Errorf("PRNumber = %d, want 100", report.PRNumber) + } + if report.Summary != "fix decimal compare" { + t.Errorf("Summary = %q", report.Summary) + } + if len(report.Coverage) != 6 { + t.Errorf("Coverage len = %d, want 6", len(report.Coverage)) + } +} + +func TestParseReport_WithMarkdown(t *testing.T) { + input := "Here is the analysis:\n```json\n" + + `{"summary":"test","affected_modules":[],"coverage":[],"suggested_cases":[]}` + + "\n```\n" + report, err := parseReport(input, 1) + if err != nil { + t.Fatalf("parseReport with markdown: %v", err) + } + if report.Summary != "test" { + t.Errorf("Summary = %q", report.Summary) + } +} + +func TestParseReport_InvalidJSON(t *testing.T) { + _, err := parseReport("not json", 1) + if err == nil { + t.Fatal("expected error for invalid JSON") + } +} + +func TestFormatReport(t *testing.T) { + report := &types.CoverageReport{ + PRNumber: 42, + Summary: "test PR", + AffectedModules: []string{"pkg/sql", "pkg/vm"}, + Coverage: []types.CoverageItem{ + {Type: types.TestBVT, Status: types.StatusCovered, Description: "ok"}, + {Type: types.TestStability, Status: types.StatusNeedsAttention, Description: "needs work"}, + {Type: types.TestChaos, Status: types.StatusNotRelated, Description: "n/a"}, + }, + SuggestedCases: []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "decimal_compare.test", + Content: "SELECT 1.0 = 1;", + Reason: "missing decimal test", + }, + }, + } + + output := FormatReport(report) + + checks := []string{ + "PR #42", + "test PR", + "pkg/sql", + "pkg/vm", + "BVT", + "✅", + "⚠️", + "➖", + "needs work", + "decimal_compare.test", + "SELECT 1.0 = 1;", + "missing decimal test", + } + for _, check := range checks { + if !strings.Contains(output, check) { + t.Errorf("output should contain %q", check) + } + } +} + +func TestAnalyze_EndToEnd(t *testing.T) { + // Mock LLM server + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := map[string]interface{}{ + "choices": []map[string]interface{}{ + { + "message": map[string]interface{}{ + "content": `{"summary":"mock","affected_modules":["pkg/test"],"coverage":[{"type":"bvt","status":"covered","description":"ok"},{"type":"stability","status":"not_related","description":"n/a"},{"type":"chaos","status":"not_related","description":"n/a"},{"type":"bigdata","status":"not_related","description":"n/a"},{"type":"pitr","status":"not_related","description":"n/a"},{"type":"snapshot","status":"not_related","description":"n/a"}],"suggested_cases":[]}`, + }, + }, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + // Create temp repo structure + tmp := t.TempDir() + // Skills + skillsDir := filepath.Join(tmp, "docs", "ai-skills") + if err := os.MkdirAll(skillsDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(skillsDir, "testing-guide.md"), []byte("guide"), 0o644); err != nil { + t.Fatal(err) + } + // Cases + casesDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(casesDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(casesDir, "test.test"), []byte("SELECT 1;"), 0o644); err != nil { + t.Fatal(err) + } + + // This test requires `gh` CLI to get the diff - skip if not available + if _, err := os.Stat("/opt/homebrew/bin/gh"); err != nil { + t.Skip("gh CLI not available, skipping end-to-end test") + } + + cfg := Config{ + Repo: "matrixorigin/matrixone", + RepoRoot: tmp, + APIURL: srv.URL, + APIKey: "test-key", + Model: "test-model", + } + + a := New(cfg) + report, err := a.Analyze(context.Background(), 24088) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if report.PRNumber != 24088 { + t.Errorf("PRNumber = %d", report.PRNumber) + } + if report.Summary != "mock" { + t.Errorf("Summary = %q", report.Summary) + } + if len(report.Coverage) != 6 { + t.Errorf("Coverage len = %d", len(report.Coverage)) + } +} diff --git a/pkg/testinfra/dedup/dedup.go b/pkg/testinfra/dedup/dedup.go new file mode 100644 index 0000000000000..ebc1bf59b652d --- /dev/null +++ b/pkg/testinfra/dedup/dedup.go @@ -0,0 +1,200 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dedup + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// Deduplicator checks whether suggested cases overlap with existing test files. +type Deduplicator struct { + repoRoot string + // existing holds normalized SQL statement sets per category dir. + // Key: directory path relative to repo root. Value: set of normalized SQL strings. + existing map[string]map[string]bool +} + +// New creates a Deduplicator that lazily loads existing cases. +func New(repoRoot string) *Deduplicator { + return &Deduplicator{ + repoRoot: repoRoot, + existing: make(map[string]map[string]bool), + } +} + +// Filter removes suggested cases whose SQL content significantly overlaps +// with existing test files in the target directory. +// Returns the filtered list and a list of skip reasons. +func (d *Deduplicator) Filter(cases []types.SuggestedCase) (kept []types.SuggestedCase, skipped []string) { + for _, sc := range cases { + dir := d.targetDir(sc) + if dir == "" { + kept = append(kept, sc) + continue + } + + existingSQL := d.loadDir(dir) + newStatements := extractStatements(sc.Content) + + if d.isDuplicate(newStatements, existingSQL) { + skipped = append(skipped, sc.Category+"/"+sc.Filename+": overlaps with existing case") + continue + } + kept = append(kept, sc) + } + return +} + +// targetDir returns the actual directory to check for duplicates. +func (d *Deduplicator) targetDir(sc types.SuggestedCase) string { + switch sc.Type { + case types.TestBVT, types.TestPITR, types.TestSnapshot: + return filepath.Join("test", "distributed", "cases", sc.Category) + default: + // Nightly regression cases are in a separate repo; skip dedup for those. + return "" + } +} + +// loadDir lazily scans all .sql/.test files in dir and extracts normalized SQL. +func (d *Deduplicator) loadDir(relDir string) map[string]bool { + if stmts, ok := d.existing[relDir]; ok { + return stmts + } + + stmts := make(map[string]bool) + absDir := filepath.Join(d.repoRoot, relDir) + + _ = filepath.WalkDir(absDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return nil + } + name := entry.Name() + if !(strings.HasSuffix(name, ".sql") || strings.HasSuffix(name, ".test")) { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil + } + for _, stmt := range extractStatements(string(data)) { + stmts[stmt] = true + } + return nil + }) + + d.existing[relDir] = stmts + return stmts +} + +// isDuplicate returns true if more than half of the new SQL statements +// already appear in the existing set. +func (d *Deduplicator) isDuplicate(newStatements []string, existing map[string]bool) bool { + if len(newStatements) == 0 { + return false + } + + matches := 0 + for _, stmt := range newStatements { + if existing[stmt] { + matches++ + } + } + + // >50% overlap means duplicate + return matches*2 > len(newStatements) +} + +// extractStatements parses SQL text into normalized statement strings. +// Strips comments, whitespace, and mo-tester tags to focus on actual SQL. +func extractStatements(content string) []string { + var stmts []string + var current strings.Builder + + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + + // Skip empty lines, comments, and mo-tester tags + if trimmed == "" { + continue + } + if strings.HasPrefix(trimmed, "--") { + continue + } + if strings.HasPrefix(trimmed, "#") { + continue + } + + current.WriteString(trimmed) + current.WriteString(" ") + + // Statement ends with semicolon + if strings.HasSuffix(trimmed, ";") { + stmt := normalizeSQL(current.String()) + if stmt != "" && !isBoilerplate(stmt) { + stmts = append(stmts, stmt) + } + current.Reset() + } + } + + // Handle last statement without trailing semicolon + if current.Len() > 0 { + stmt := normalizeSQL(current.String()) + if stmt != "" && !isBoilerplate(stmt) { + stmts = append(stmts, stmt) + } + } + + return stmts +} + +// normalizeSQL lowercases and collapses whitespace for comparison. +func normalizeSQL(sql string) string { + sql = strings.ToLower(strings.TrimSpace(sql)) + // Collapse multiple spaces + for strings.Contains(sql, " ") { + sql = strings.ReplaceAll(sql, " ", " ") + } + return sql +} + +// isBoilerplate returns true for common setup/teardown SQL that shouldn't +// contribute to duplicate detection. +func isBoilerplate(sql string) bool { + prefixes := []string{ + "drop table ", + "drop database ", + "drop account ", + "drop pitr ", + "drop snapshot ", + "use ", + "set ", + "begin;", + "commit;", + "rollback;", + } + for _, p := range prefixes { + if strings.HasPrefix(sql, p) { + return true + } + } + return false +} diff --git a/pkg/testinfra/dedup/dedup_test.go b/pkg/testinfra/dedup/dedup_test.go new file mode 100644 index 0000000000000..9f2c4a9a3f8b1 --- /dev/null +++ b/pkg/testinfra/dedup/dedup_test.go @@ -0,0 +1,245 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dedup + +import ( + "os" + "path/filepath" + "testing" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +func TestExtractStatements(t *testing.T) { + content := `-- this is a comment +-- @bvt:issue#123 +drop table if exists t1; +create table t1 (a int, b varchar(100)); +insert into t1 values (1, 'hello'); +SELECT a, b FROM t1 WHERE a > 0; +-- @bvt:issue +drop table t1; +` + stmts := extractStatements(content) + + // Should skip: comments, drop table if exists (boilerplate), drop table (boilerplate) + // Should keep: create table, insert, select + if len(stmts) != 3 { + t.Fatalf("got %d statements, want 3: %v", len(stmts), stmts) + } + if stmts[0] != "create table t1 (a int, b varchar(100));" { + t.Errorf("stmts[0] = %q", stmts[0]) + } + if stmts[1] != "insert into t1 values (1, 'hello');" { + t.Errorf("stmts[1] = %q", stmts[1]) + } + if stmts[2] != "select a, b from t1 where a > 0;" { + t.Errorf("stmts[2] = %q", stmts[2]) + } +} + +func TestExtractStatements_MultiLine(t *testing.T) { + content := `CREATE TABLE t1 ( + a INT, + b VARCHAR(100) +); +SELECT * FROM t1;` + stmts := extractStatements(content) + if len(stmts) != 2 { + t.Fatalf("got %d statements, want 2: %v", len(stmts), stmts) + } +} + +func TestNormalizeSQL(t *testing.T) { + cases := []struct { + in, want string + }{ + {" SELECT * FROM t1 ; ", "select * from t1 ;"}, + {"INSERT INTO t1 VALUES (1);", "insert into t1 values (1);"}, + } + for _, tc := range cases { + got := normalizeSQL(tc.in) + if got != tc.want { + t.Errorf("normalizeSQL(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestIsBoilerplate(t *testing.T) { + boilerplate := []string{ + "drop table if exists t1;", + "drop database if exists db1;", + "use db1;", + "set global var = 1;", + "begin;", + "commit;", + } + for _, s := range boilerplate { + if !isBoilerplate(s) { + t.Errorf("%q should be boilerplate", s) + } + } + + notBoilerplate := []string{ + "create table t1 (a int);", + "select * from t1;", + "insert into t1 values (1);", + } + for _, s := range notBoilerplate { + if isBoilerplate(s) { + t.Errorf("%q should NOT be boilerplate", s) + } + } +} + +func TestFilter_NoDuplicate(t *testing.T) { + tmp := t.TempDir() + catDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(catDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(catDir, "existing.sql"), []byte("SELECT 1;\nSELECT 2;\n"), 0o644); err != nil { + t.Fatal(err) + } + + d := New(tmp) + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "new_case.sql", + Content: "CREATE TABLE t1 (a int);\nSELECT a FROM t1;\ndrop table t1;\n", + }, + } + + kept, skipped := d.Filter(cases) + if len(kept) != 1 { + t.Errorf("kept = %d, want 1", len(kept)) + } + if len(skipped) != 0 { + t.Errorf("skipped = %v", skipped) + } +} + +func TestFilter_Duplicate(t *testing.T) { + tmp := t.TempDir() + catDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(catDir, 0o755); err != nil { + t.Fatal(err) + } + // Existing file has these SQL statements + existing := "CREATE TABLE t1 (a INT);\nINSERT INTO t1 VALUES (1);\nSELECT * FROM t1;\ndrop table t1;\n" + if err := os.WriteFile(filepath.Join(catDir, "existing.sql"), []byte(existing), 0o644); err != nil { + t.Fatal(err) + } + + d := New(tmp) + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "duplicate.sql", + // Same SQL, slightly different formatting + Content: "create table t1 (a int);\ninsert into t1 values (1);\nselect * from t1;\ndrop table t1;\n", + }, + } + + kept, skipped := d.Filter(cases) + if len(kept) != 0 { + t.Errorf("kept = %d, want 0 (should be filtered)", len(kept)) + } + if len(skipped) != 1 { + t.Errorf("skipped = %d, want 1", len(skipped)) + } +} + +func TestFilter_PartialOverlap(t *testing.T) { + tmp := t.TempDir() + catDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(catDir, 0o755); err != nil { + t.Fatal(err) + } + existing := "SELECT 1;\nSELECT 2;\n" + if err := os.WriteFile(filepath.Join(catDir, "existing.sql"), []byte(existing), 0o644); err != nil { + t.Fatal(err) + } + + d := New(tmp) + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "partial.sql", + // 4 statements, only 1 overlaps (SELECT 1) = 25% < 50%, so keep + Content: "SELECT 1;\nSELECT 3;\nSELECT 4;\nSELECT 5;\n", + }, + } + + kept, skipped := d.Filter(cases) + if len(kept) != 1 { + t.Errorf("kept = %d, want 1 (25%% overlap should pass)", len(kept)) + } + if len(skipped) != 0 { + t.Errorf("skipped = %v", skipped) + } +} + +func TestFilter_NightlySkipsDedup(t *testing.T) { + tmp := t.TempDir() + d := New(tmp) + + cases := []types.SuggestedCase{ + { + Type: types.TestStability, + Category: "sysbench/new_scenario", + Filename: "run.yml", + Content: "duration: 10\n", + }, + { + Type: types.TestChaos, + Category: "mo-chaos-config", + Filename: "new_chaos.yaml", + Content: "chaos: test\n", + }, + } + + kept, skipped := d.Filter(cases) + if len(kept) != 2 { + t.Errorf("kept = %d, want 2 (nightly cases skip dedup)", len(kept)) + } + if len(skipped) != 0 { + t.Errorf("skipped = %v", skipped) + } +} + +func TestFilter_EmptyNewCase(t *testing.T) { + tmp := t.TempDir() + d := New(tmp) + + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "empty.sql", + Content: "-- just comments\n-- nothing here\n", + }, + } + + kept, _ := d.Filter(cases) + // Empty SQL → 0 statements → not duplicate → kept + if len(kept) != 1 { + t.Errorf("kept = %d, want 1", len(kept)) + } +} diff --git a/pkg/testinfra/llm/client.go b/pkg/testinfra/llm/client.go new file mode 100644 index 0000000000000..d283905d63a7a --- /dev/null +++ b/pkg/testinfra/llm/client.go @@ -0,0 +1,104 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package llm + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// Message represents a chat message. +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type chatRequest struct { + Model string `json:"model"` + Messages []Message `json:"messages"` + Temperature float64 `json:"temperature"` +} + +type chatResponse struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` +} + +// Client is an OpenAI-compatible LLM API client. +type Client struct { + apiURL string + apiKey string + model string +} + +// NewClient creates an LLM client. +func NewClient(apiURL, apiKey, model string) *Client { + return &Client{apiURL: apiURL, apiKey: apiKey, model: model} +} + +// Chat sends messages to the LLM and returns the response text. +func (c *Client) Chat(ctx context.Context, messages []Message) (string, error) { + reqBody := chatRequest{ + Model: c.model, + Messages: messages, + Temperature: 0.1, + } + + data, err := json.Marshal(reqBody) + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("marshal request: %v", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "POST", c.apiURL, bytes.NewReader(data)) + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("create request: %v", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("LLM API call failed: %v", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("read response: %v", err) + } + + if resp.StatusCode != http.StatusOK { + return "", moerr.NewInternalErrorNoCtxf("LLM API %d: %s", resp.StatusCode, string(body)) + } + + var chatResp chatResponse + if err := json.Unmarshal(body, &chatResp); err != nil { + return "", moerr.NewInternalErrorNoCtxf("parse response: %v", err) + } + + if len(chatResp.Choices) == 0 { + return "", moerr.NewInternalErrorNoCtx("LLM returned no choices") + } + + return chatResp.Choices[0].Message.Content, nil +} diff --git a/pkg/testinfra/llm/client_test.go b/pkg/testinfra/llm/client_test.go new file mode 100644 index 0000000000000..1651a1b1d0007 --- /dev/null +++ b/pkg/testinfra/llm/client_test.go @@ -0,0 +1,118 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package llm + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestNewClient(t *testing.T) { + c := NewClient("http://example.com/v1", "key123", "gpt-4") + if c.apiURL != "http://example.com/v1" { + t.Errorf("apiURL = %q", c.apiURL) + } + if c.apiKey != "key123" { + t.Errorf("apiKey = %q", c.apiKey) + } + if c.model != "gpt-4" { + t.Errorf("model = %q", c.model) + } +} + +func TestChat_Success(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify request + if r.Method != "POST" { + t.Errorf("method = %s, want POST", r.Method) + } + if r.Header.Get("Authorization") != "Bearer test-key" { + t.Errorf("auth header = %q", r.Header.Get("Authorization")) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Errorf("content-type = %q", r.Header.Get("Content-Type")) + } + + var req chatRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + if req.Model != "test-model" { + t.Errorf("model = %q, want test-model", req.Model) + } + if len(req.Messages) != 1 { + t.Fatalf("messages len = %d, want 1", len(req.Messages)) + } + if req.Messages[0].Content != "hello" { + t.Errorf("message content = %q", req.Messages[0].Content) + } + + resp := chatResponse{ + Choices: []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + }{ + {Message: struct { + Content string `json:"content"` + }{Content: "world"}}, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + c := NewClient(srv.URL, "test-key", "test-model") + result, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hello"}}) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if result != "world" { + t.Errorf("result = %q, want %q", result, "world") + } +} + +func TestChat_HTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte("rate limited")) + })) + defer srv.Close() + + c := NewClient(srv.URL, "key", "model") + _, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}) + if err == nil { + t.Fatal("expected error for 429") + } +} + +func TestChat_EmptyChoices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := chatResponse{Choices: nil} + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + c := NewClient(srv.URL, "key", "model") + _, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}) + if err == nil { + t.Fatal("expected error for empty choices") + } +} diff --git a/pkg/testinfra/scanner/scanner.go b/pkg/testinfra/scanner/scanner.go new file mode 100644 index 0000000000000..c80304066f49e --- /dev/null +++ b/pkg/testinfra/scanner/scanner.go @@ -0,0 +1,172 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "io/fs" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// CaseInfo holds the test files for a single BVT category. +type CaseInfo struct { + Category string + Files []string +} + +// ScanBVTCases scans test/distributed/cases/ and returns case info per category. +func ScanBVTCases(repoRoot string) ([]CaseInfo, error) { + casesDir := filepath.Join(repoRoot, "test", "distributed", "cases") + entries, err := os.ReadDir(casesDir) + if err != nil { + return nil, moerr.NewInternalErrorNoCtxf("read cases dir: %v", err) + } + + result := make([]CaseInfo, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + info := CaseInfo{Category: entry.Name()} + catDir := filepath.Join(casesDir, entry.Name()) + _ = filepath.WalkDir(catDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil || d.IsDir() { + return nil + } + if strings.HasSuffix(d.Name(), ".test") || strings.HasSuffix(d.Name(), ".sql") { + info.Files = append(info.Files, d.Name()) + } + return nil + }) + sort.Strings(info.Files) + result = append(result, info) + } + return result, nil +} + +// FormatCaseSummary returns a compact text listing of BVT cases for the LLM prompt. +// Uses compact mode by default to save tokens. +func FormatCaseSummary(cases []CaseInfo) string { + var b strings.Builder + for _, c := range cases { + b.WriteString(c.Category) + b.WriteString("/") + b.WriteString(strconv.Itoa(len(c.Files))) + b.WriteString(" ") + } + return b.String() +} + +// ReadSampleCases reads the first N lines from BVT .test/.sql files +// in categories relevant to the given changed paths. +// This gives the LLM concrete examples of existing case format. +func ReadSampleCases(repoRoot string, changedPaths []string, maxSamples int, maxLinesPerFile int) string { + if maxSamples <= 0 { + maxSamples = 3 + } + if maxLinesPerFile <= 0 { + maxLinesPerFile = 40 + } + + // Determine relevant categories from changed paths + categories := inferCategories(changedPaths) + if len(categories) == 0 { + return "" + } + + casesDir := filepath.Join(repoRoot, "test", "distributed", "cases") + var b strings.Builder + sampled := 0 + + for _, cat := range categories { + if sampled >= maxSamples { + break + } + catDir := filepath.Join(casesDir, cat) + entries, err := os.ReadDir(catDir) + if err != nil { + continue + } + for _, e := range entries { + if sampled >= maxSamples { + break + } + name := e.Name() + if e.IsDir() || !(strings.HasSuffix(name, ".test") || strings.HasSuffix(name, ".sql")) { + continue + } + data, err := os.ReadFile(filepath.Join(catDir, name)) + if err != nil { + continue + } + lines := strings.Split(string(data), "\n") + if len(lines) > maxLinesPerFile { + lines = lines[:maxLinesPerFile] + } + b.WriteString("### ") + b.WriteString(cat) + b.WriteString("/") + b.WriteString(name) + b.WriteString("\n```sql\n") + b.WriteString(strings.Join(lines, "\n")) + b.WriteString("\n```\n\n") + sampled++ + } + } + return b.String() +} + +// inferCategories maps changed file paths to likely BVT categories. +func inferCategories(paths []string) []string { + seen := make(map[string]bool) + mappings := map[string][]string{ + "pkg/sql/plan": {"function", "optimizer", "subquery", "join", "window", "cte"}, + "pkg/sql/compile": {"function", "expression", "dml"}, + "pkg/sql/parsers": {"ddl", "dml", "expression"}, + "pkg/vm/engine/tae": {"disttae", "dml"}, + "pkg/vm/engine/dist": {"disttae"}, + "pkg/txn": {"pessimistic_transaction", "optimistic"}, + "pkg/lockservice": {"pessimistic_transaction"}, + "pkg/frontend": {"snapshot", "pitr", "tenant"}, + "pkg/cdc": {"dml"}, + "pkg/fulltext": {"fulltext"}, + "pkg/vectorindex": {"vector"}, + "pkg/vectorize": {"function", "expression"}, + "pkg/partition": {"ddl"}, + "pkg/backup": {"snapshot", "pitr"}, + } + + for _, p := range paths { + for prefix, cats := range mappings { + if strings.HasPrefix(p, prefix) { + for _, c := range cats { + seen[c] = true + } + } + } + } + + result := make([]string, 0, len(seen)) + for c := range seen { + result = append(result, c) + } + sort.Strings(result) + return result +} diff --git a/pkg/testinfra/scanner/scanner_test.go b/pkg/testinfra/scanner/scanner_test.go new file mode 100644 index 0000000000000..ab176e90ef6c8 --- /dev/null +++ b/pkg/testinfra/scanner/scanner_test.go @@ -0,0 +1,169 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestScanBVTCases(t *testing.T) { + // Create a temporary directory structure + tmp := t.TempDir() + casesDir := filepath.Join(tmp, "test", "distributed", "cases") + funcDir := filepath.Join(casesDir, "function") + ddlDir := filepath.Join(casesDir, "ddl") + + if err := os.MkdirAll(funcDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(ddlDir, 0o755); err != nil { + t.Fatal(err) + } + + // Create test files + for _, f := range []string{"test1.test", "test2.test", "other.txt"} { + if err := os.WriteFile(filepath.Join(funcDir, f), []byte("SELECT 1;"), 0o644); err != nil { + t.Fatal(err) + } + } + for _, f := range []string{"create.sql", "drop.test"} { + if err := os.WriteFile(filepath.Join(ddlDir, f), []byte("CREATE TABLE t(a INT);"), 0o644); err != nil { + t.Fatal(err) + } + } + + cases, err := ScanBVTCases(tmp) + if err != nil { + t.Fatalf("ScanBVTCases: %v", err) + } + + if len(cases) != 2 { + t.Fatalf("got %d categories, want 2", len(cases)) + } + + // Find each category + caseMap := make(map[string]CaseInfo) + for _, c := range cases { + caseMap[c.Category] = c + } + + funcInfo, ok := caseMap["function"] + if !ok { + t.Fatal("missing 'function' category") + } + // Should have 2 .test files, not the .txt + if len(funcInfo.Files) != 2 { + t.Errorf("function files = %d, want 2: %v", len(funcInfo.Files), funcInfo.Files) + } + + ddlInfo, ok := caseMap["ddl"] + if !ok { + t.Fatal("missing 'ddl' category") + } + // Should have 1 .sql + 1 .test = 2 + if len(ddlInfo.Files) != 2 { + t.Errorf("ddl files = %d, want 2: %v", len(ddlInfo.Files), ddlInfo.Files) + } +} + +func TestScanBVTCases_MissingDir(t *testing.T) { + _, err := ScanBVTCases("/nonexistent/path") + if err == nil { + t.Fatal("expected error for missing dir") + } +} + +func TestFormatCaseSummary(t *testing.T) { + cases := []CaseInfo{ + {Category: "function", Files: []string{"a.test", "b.test", "c.test"}}, + {Category: "ddl", Files: []string{"x.sql"}}, + } + + summary := FormatCaseSummary(cases) + + if !strings.Contains(summary, "function/3") { + t.Errorf("summary should contain 'function/3', got: %q", summary) + } + if !strings.Contains(summary, "ddl/1") { + t.Errorf("summary should contain 'ddl/1', got: %q", summary) + } +} + +func TestFormatCaseSummary_Empty(t *testing.T) { + summary := FormatCaseSummary(nil) + if summary != "" { + t.Errorf("expected empty summary, got: %q", summary) + } +} + +func TestReadSampleCases(t *testing.T) { + tmp := t.TempDir() + funcDir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(funcDir, 0o755); err != nil { + t.Fatal(err) + } + + content := "-- test sample\nSELECT 1;\nSELECT 2;\ndrop table if exists t1;\n" + if err := os.WriteFile(filepath.Join(funcDir, "sample.test"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + // Changed path in pkg/sql/plan should map to "function" category + result := ReadSampleCases(tmp, []string{"pkg/sql/plan/some_file.go"}, 3, 40) + if !strings.Contains(result, "function/sample.test") { + t.Errorf("should include function/sample.test, got: %q", result) + } + if !strings.Contains(result, "SELECT 1;") { + t.Errorf("should include file content, got: %q", result) + } +} + +func TestReadSampleCases_NoMatch(t *testing.T) { + tmp := t.TempDir() + result := ReadSampleCases(tmp, []string{"unknown/path.go"}, 3, 40) + if result != "" { + t.Errorf("expected empty result for unknown path, got: %q", result) + } +} + +func TestInferCategories(t *testing.T) { + cases := []struct { + path string + want []string + }{ + {"pkg/sql/plan/foo.go", []string{"cte", "function", "join", "optimizer", "subquery", "window"}}, + {"pkg/txn/foo.go", []string{"optimistic", "pessimistic_transaction"}}, + {"pkg/frontend/foo.go", []string{"pitr", "snapshot", "tenant"}}, + {"pkg/fulltext/foo.go", []string{"fulltext"}}, + {"pkg/cdc/foo.go", []string{"dml"}}, + {"unknown/foo.go", nil}, + } + + for _, tc := range cases { + got := inferCategories([]string{tc.path}) + if len(got) != len(tc.want) { + t.Errorf("inferCategories(%q) = %v, want %v", tc.path, got, tc.want) + continue + } + for i, g := range got { + if g != tc.want[i] { + t.Errorf("inferCategories(%q)[%d] = %q, want %q", tc.path, i, g, tc.want[i]) + } + } + } +} diff --git a/pkg/testinfra/types/types.go b/pkg/testinfra/types/types.go new file mode 100644 index 0000000000000..949385b99bda3 --- /dev/null +++ b/pkg/testinfra/types/types.go @@ -0,0 +1,61 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +// CoverageStatus represents the test coverage state for a test category. +type CoverageStatus string + +const ( + StatusCovered CoverageStatus = "covered" + StatusNeedsAttention CoverageStatus = "needs_attention" + StatusNotRelated CoverageStatus = "not_related" +) + +// TestType represents the 6 test categories in MO's test infrastructure. +type TestType string + +const ( + TestBVT TestType = "bvt" + TestStability TestType = "stability" + TestChaos TestType = "chaos" + TestBigData TestType = "bigdata" + TestPITR TestType = "pitr" + TestSnapshot TestType = "snapshot" +) + +// CoverageItem describes coverage status for one test type. +type CoverageItem struct { + Type TestType `json:"type"` + Status CoverageStatus `json:"status"` + Description string `json:"description"` +} + +// SuggestedCase is a test case the AI suggests adding. +type SuggestedCase struct { + Type TestType `json:"type"` + Category string `json:"category"` + Filename string `json:"filename"` + Content string `json:"content"` + Reason string `json:"reason"` +} + +// CoverageReport is the structured output of the analysis. +type CoverageReport struct { + PRNumber int `json:"pr_number"` + Summary string `json:"summary"` + AffectedModules []string `json:"affected_modules"` + Coverage []CoverageItem `json:"coverage"` + SuggestedCases []SuggestedCase `json:"suggested_cases"` +} diff --git a/pkg/testinfra/types/types_test.go b/pkg/testinfra/types/types_test.go new file mode 100644 index 0000000000000..5e21e85fe3926 --- /dev/null +++ b/pkg/testinfra/types/types_test.go @@ -0,0 +1,101 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "encoding/json" + "testing" +) + +func TestCoverageStatusValues(t *testing.T) { + if StatusCovered != "covered" { + t.Errorf("StatusCovered = %q, want %q", StatusCovered, "covered") + } + if StatusNeedsAttention != "needs_attention" { + t.Errorf("StatusNeedsAttention = %q, want %q", StatusNeedsAttention, "needs_attention") + } + if StatusNotRelated != "not_related" { + t.Errorf("StatusNotRelated = %q, want %q", StatusNotRelated, "not_related") + } +} + +func TestTestTypeValues(t *testing.T) { + expected := map[TestType]string{ + TestBVT: "bvt", + TestStability: "stability", + TestChaos: "chaos", + TestBigData: "bigdata", + TestPITR: "pitr", + TestSnapshot: "snapshot", + } + for tt, want := range expected { + if string(tt) != want { + t.Errorf("TestType %v = %q, want %q", tt, string(tt), want) + } + } +} + +func TestCoverageReportJSON(t *testing.T) { + report := CoverageReport{ + PRNumber: 123, + Summary: "test summary", + AffectedModules: []string{"pkg/sql"}, + Coverage: []CoverageItem{ + {Type: TestBVT, Status: StatusCovered, Description: "ok"}, + {Type: TestStability, Status: StatusNotRelated, Description: "n/a"}, + }, + SuggestedCases: []SuggestedCase{ + { + Type: TestBVT, + Category: "function", + Filename: "test1.test", + Content: "SELECT 1;", + Reason: "missing coverage", + }, + }, + } + + data, err := json.Marshal(report) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + var got CoverageReport + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + + if got.PRNumber != 123 { + t.Errorf("PRNumber = %d, want 123", got.PRNumber) + } + if got.Summary != "test summary" { + t.Errorf("Summary = %q, want %q", got.Summary, "test summary") + } + if len(got.Coverage) != 2 { + t.Fatalf("Coverage len = %d, want 2", len(got.Coverage)) + } + if got.Coverage[0].Type != TestBVT { + t.Errorf("Coverage[0].Type = %q, want %q", got.Coverage[0].Type, TestBVT) + } + if got.Coverage[0].Status != StatusCovered { + t.Errorf("Coverage[0].Status = %q, want %q", got.Coverage[0].Status, StatusCovered) + } + if len(got.SuggestedCases) != 1 { + t.Fatalf("SuggestedCases len = %d, want 1", len(got.SuggestedCases)) + } + if got.SuggestedCases[0].Content != "SELECT 1;" { + t.Errorf("SuggestedCases[0].Content = %q", got.SuggestedCases[0].Content) + } +} diff --git a/pkg/testinfra/writer/writer.go b/pkg/testinfra/writer/writer.go new file mode 100644 index 0000000000000..0e9a73b54a912 --- /dev/null +++ b/pkg/testinfra/writer/writer.go @@ -0,0 +1,120 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package writer + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +// WriteCases writes suggested BVT cases to disk under test/distributed/cases/. +// Returns the list of written file paths (relative to repoRoot). +// WriteCases writes suggested test cases to disk. +// BVT/PITR/Snapshot → test/distributed/cases/{category}/ +// Stability/Chaos/BigData → output/{type}/{category}/ (for manual copy to nightly repo) +func WriteCases(repoRoot string, cases []types.SuggestedCase) ([]string, error) { + if len(cases) == 0 { + return nil, nil + } + + var written []string + for _, sc := range cases { + if sc.Category == "" || sc.Filename == "" || sc.Content == "" { + continue + } + + var dir, relPath string + switch sc.Type { + case types.TestBVT, types.TestPITR, types.TestSnapshot: + // These go directly into the matrixone repo + dir = filepath.Join(repoRoot, "test", "distributed", "cases", sc.Category) + relPath = filepath.Join("test", "distributed", "cases", sc.Category, sc.Filename) + case types.TestStability, types.TestChaos, types.TestBigData: + // These are staged in output/ for manual copy to mo-nightly-regression + dir = filepath.Join(repoRoot, "output", string(sc.Type), sc.Category) + relPath = filepath.Join("output", string(sc.Type), sc.Category, sc.Filename) + default: + continue + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return written, moerr.NewInternalErrorNoCtxf("mkdir %s: %v", dir, err) + } + + fpath := filepath.Join(dir, sc.Filename) + + // Don't overwrite existing files + if _, err := os.Stat(fpath); err == nil { + fmt.Fprintf(os.Stderr, "skip: %s already exists\n", relPath) + continue + } + + if err := os.WriteFile(fpath, []byte(sc.Content), 0o644); err != nil { + return written, moerr.NewInternalErrorNoCtxf("write %s: %v", relPath, err) + } + written = append(written, relPath) + } + return written, nil +} + +// CreatePR creates a new branch, commits the written case files, and opens a PR. +// Returns the PR URL. +func CreatePR(repoRoot string, repo string, prNumber int, files []string) (string, error) { + if len(files) == 0 { + return "", moerr.NewInternalErrorNoCtx("no files to commit") + } + + branch := "testinfra/pr-" + strconv.Itoa(prNumber) + "-cases" + title := fmt.Sprintf("test: add BVT cases for PR #%d", prNumber) + body := fmt.Sprintf("Auto-generated BVT test cases for PR #%d.\n\nGenerated by `mo-testplan`.", prNumber) + + type step struct { + name string + args []string + } + + steps := []step{ + {"git", []string{"-C", repoRoot, "checkout", "-b", branch}}, + } + for _, f := range files { + steps = append(steps, step{"git", []string{"-C", repoRoot, "add", f}}) + } + steps = append(steps, + step{"git", []string{"-C", repoRoot, "commit", "-m", title}}, + step{"git", []string{"-C", repoRoot, "push", "origin", branch}}, + step{"gh", []string{"pr", "create", "--repo", repo, "--head", branch, + "--title", title, "--body", body}}, + ) + + for _, s := range steps { + cmd := exec.Command(s.name, s.args...) + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + return "", moerr.NewInternalErrorNoCtxf("%s %v failed: %v", s.name, s.args, err) + } + // The last command (gh pr create) outputs the PR URL + if s.name == "gh" { + return string(out), nil + } + } + return "", nil +} diff --git a/pkg/testinfra/writer/writer_test.go b/pkg/testinfra/writer/writer_test.go new file mode 100644 index 0000000000000..3e543e039d687 --- /dev/null +++ b/pkg/testinfra/writer/writer_test.go @@ -0,0 +1,235 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package writer + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/matrixorigin/matrixone/pkg/testinfra/types" +) + +func TestWriteCases_Basic(t *testing.T) { + tmp := t.TempDir() + + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "decimal_compare.test", + Content: "-- test decimal vs integer compare\nSELECT 1.0 = 1;\nSELECT 2.5 > 2;\n", + Reason: "missing coverage", + }, + { + Type: types.TestBVT, + Category: "window", + Filename: "window_decimal.test", + Content: "-- test window function with decimal\nSELECT SUM(1.0) OVER();\n", + Reason: "missing coverage", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + + if len(written) != 2 { + t.Fatalf("written = %d, want 2", len(written)) + } + + // Verify file content + data, err := os.ReadFile(filepath.Join(tmp, "test", "distributed", "cases", "function", "decimal_compare.test")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != cases[0].Content { + t.Errorf("content mismatch: %q", string(data)) + } + + // Verify second file + data2, err := os.ReadFile(filepath.Join(tmp, "test", "distributed", "cases", "window", "window_decimal.test")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data2) != cases[1].Content { + t.Errorf("content mismatch: %q", string(data2)) + } +} + +func TestWriteCases_SkipExisting(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "test", "distributed", "cases", "function") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + // Pre-create the file + if err := os.WriteFile(filepath.Join(dir, "existing.test"), []byte("original"), 0o644); err != nil { + t.Fatal(err) + } + + cases := []types.SuggestedCase{ + { + Type: types.TestBVT, + Category: "function", + Filename: "existing.test", + Content: "new content", + Reason: "test", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + + if len(written) != 0 { + t.Errorf("should skip existing file, written = %v", written) + } + + // Verify original content preserved + data, err := os.ReadFile(filepath.Join(dir, "existing.test")) + if err != nil { + t.Fatal(err) + } + if string(data) != "original" { + t.Errorf("existing file was overwritten: %q", string(data)) + } +} + +func TestWriteCases_SkipNonBVT(t *testing.T) { + tmp := t.TempDir() + + // Stability cases should go to output/ directory, not be skipped + cases := []types.SuggestedCase{ + { + Type: types.TestStability, + Category: "sysbench/mixed", + Filename: "run.yml", + Content: "duration: 10\ntransaction:\n - name: test\n", + Reason: "test", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if len(written) != 1 { + t.Fatalf("should write stability case to output/, written = %v", written) + } + if !strings.Contains(written[0], "output") { + t.Errorf("stability case should be in output/, got: %s", written[0]) + } + + // Verify file exists + data, err := os.ReadFile(filepath.Join(tmp, written[0])) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !strings.Contains(string(data), "duration") { + t.Errorf("content mismatch: %q", string(data)) + } +} + +func TestWriteCases_PITRAndSnapshot(t *testing.T) { + tmp := t.TempDir() + + cases := []types.SuggestedCase{ + { + Type: types.TestPITR, + Category: "pitr", + Filename: "pitr_new.sql", + Content: "CREATE PITR p1;\nSELECT 1;\ndrop pitr p1;\n", + Reason: "test", + }, + { + Type: types.TestSnapshot, + Category: "snapshot", + Filename: "snap_new.sql", + Content: "CREATE SNAPSHOT s1;\nSELECT 1;\n", + Reason: "test", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if len(written) != 2 { + t.Fatalf("written = %d, want 2", len(written)) + } + // Both should be in test/distributed/cases/ + for _, w := range written { + if !strings.HasPrefix(w, "test/distributed/cases/") { + t.Errorf("PITR/Snapshot should be in test/distributed/cases/, got: %s", w) + } + } +} + +func TestWriteCases_ChaosOutput(t *testing.T) { + tmp := t.TempDir() + + cases := []types.SuggestedCase{ + { + Type: types.TestChaos, + Category: "mo-chaos-config", + Filename: "chaos_new_scenario.yaml", + Content: "chaos:\n cm-chaos:\n - name: test\n", + Reason: "test", + }, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if len(written) != 1 { + t.Fatalf("written = %d, want 1", len(written)) + } + if !strings.Contains(written[0], "output/chaos") { + t.Errorf("chaos case should be in output/chaos/, got: %s", written[0]) + } +} + +func TestWriteCases_SkipEmpty(t *testing.T) { + tmp := t.TempDir() + + cases := []types.SuggestedCase{ + {Type: types.TestBVT, Category: "", Filename: "a.test", Content: "x"}, + {Type: types.TestBVT, Category: "f", Filename: "", Content: "x"}, + {Type: types.TestBVT, Category: "f", Filename: "a.test", Content: ""}, + } + + written, err := WriteCases(tmp, cases) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if len(written) != 0 { + t.Errorf("should skip cases with empty fields, written = %v", written) + } +} + +func TestWriteCases_EmptyList(t *testing.T) { + written, err := WriteCases(t.TempDir(), nil) + if err != nil { + t.Fatalf("WriteCases: %v", err) + } + if written != nil { + t.Errorf("expected nil, got %v", written) + } +}