From 47d846ddffcb2e14c39250b197bf9ba3bcb85c84 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:30:29 +0800 Subject: [PATCH 01/30] =?UTF-8?q?feat(config):=20=E6=8C=89=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E6=8B=86=E5=88=86=20embedding=20=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [models.embedding] 保留为默认配置，新增 [models.embedding.features.] （knowledge / cognitive / memes），支持 use_default 整体继承或按字段覆写； - 覆写支持连接信息、context_window_tokens、queue_interval_seconds、dimensions、 query_instruction / document_instruction 与 request_params 合并； - 生效配置相同的功能复用同一个 Embedder 与发车队列，重排器全局共享； - 修复指令前缀被 strip 的缺陷：指令与文本直接拼接，首尾空白与换行必须保留； - 认知重嵌入脚本改用 cognitive 功能实际生效的配置； - 嵌入/重排配置变更加入需重启提示，避免热重载静默无效。 --- config.toml.example | 128 +++++++- docs/cognitive-memory.md | 9 +- docs/configuration.md | 39 ++- docs/knowledge.md | 4 + scripts/reembed_cognitive.py | 12 +- src/Undefined/config/coercers.py | 12 + src/Undefined/config/config_class.py | 14 + src/Undefined/config/hot_reload.py | 4 + .../config/load_sections/knowledge.py | 3 + src/Undefined/config/model_parsers.py | 2 + src/Undefined/config/models.py | 67 +++- src/Undefined/config/parsers/__init__.py | 7 +- src/Undefined/config/parsers/embedding.py | 77 ++++- src/Undefined/knowledge/__init__.py | 10 +- src/Undefined/knowledge/runtime.py | 89 ++++++ src/Undefined/main.py | 43 +-- tests/test_embedding_feature_config.py | 298 ++++++++++++++++++ 17 files changed, 783 insertions(+), 35 deletions(-) create mode 100644 tests/test_embedding_feature_config.py diff --git a/config.toml.example b/config.toml.example index df7e9ff5..180da1d4 100644 --- a/config.toml.example +++ b/config.toml.example @@ -685,8 +685,8 @@ stream_enabled = false # en: Extra request-body params (optional), e.g. temperature or vendor-specific fields. [models.grok.request_params] -# zh: 嵌入模型配置（知识库语义检索使用）。 -# en: Embedding model config (used by knowledge semantic retrieval). +# zh: 嵌入模型默认配置（知识库 / 认知记忆 / 梗库默认共用；各功能可在下方 features 中单独覆写）。 +# en: Default embedding model config (shared by knowledge, cognitive memory and memes by default; override per feature below). [models.embedding] # zh: 是否让该模型请求使用 [proxy] 代理地址。默认关闭。 # en: Whether this model uses proxy addresses from [proxy]. Disabled by default. @@ -721,6 +721,130 @@ document_instruction = "" # en: Extra request-body params (optional) for embedding-provider-specific fields. [models.embedding.request_params] +# zh: 按功能覆写的 embedding 配置。每个功能可完全继承 [models.embedding] 默认配置， +# 也可单独设置并按字段覆写；生效配置完全相同的功能共用一个发车队列。 +# en: Per-feature embedding overrides. Each feature either inherits the +# [models.embedding] defaults or overrides individual fields; features with +# the same effective config share one dispatch queue. + +# zh: 按功能覆写示例：知识库（knowledge）。 +# en: Per-feature override example: knowledge base (knowledge). +[models.embedding.features.knowledge] +# zh: 是否完全使用 [models.embedding] 默认配置；true 时本表其余字段全部忽略。 +# en: Whether to fully inherit the [models.embedding] defaults; when true, all other fields here are ignored. +use_default = true +# zh: 以下字段仅在 use_default = false 时生效；只有出现在本表中的字段才会覆写默认配置。 +# en: The fields below apply only when use_default = false; only fields present here override the defaults. +# zh: 覆写 API 地址；空字符串表示继承默认配置。 +# en: Override API URL; an empty string inherits the default. +api_url = "" +# zh: 覆写 API Key；空字符串表示继承默认配置。 +# en: Override API key; an empty string inherits the default. +api_key = "" +# zh: 覆写模型名称；空字符串表示继承默认配置。 +# en: Override model name; an empty string inherits the default. +model_name = "" +# zh: 覆写代理开关；"inherit"（或 "default"）表示继承默认配置，也可写 true / false。 +# en: Override the proxy switch; "inherit" (or "default") inherits the default, true / false override it. +use_proxy = "inherit" +# zh: 覆写上下文窗口上限（token）；<=0 表示继承默认配置。 +# en: Override the context window cap (tokens); <=0 inherits the default. +context_window_tokens = 0 +# zh: 覆写队列发车间隔（秒）；<0 表示继承默认配置，0 表示请求到达立即发车。 +# en: Override the queue interval (seconds); <0 inherits the default, 0 dispatches on arrival. +queue_interval_seconds = -1.0 +# zh: 覆写向量维度；<0 表示继承默认配置，0 表示使用模型默认维度。 +# en: Override embedding dimensions; <0 inherits the default, 0 uses the model default. +dimensions = -1 +# zh: 覆写查询端指令前缀；空字符串表示继承默认配置。 +# en: Override the query instruction prefix; an empty string inherits the default. +query_instruction = "" +# zh: 覆写文档端指令前缀；空字符串表示继承默认配置。 +# en: Override the document instruction prefix; an empty string inherits the default. +document_instruction = "" + +# zh: 覆写额外请求体参数，按 key 合并到默认 request_params 之上（同名以本表为准）。 +# en: Override extra request-body params, merged over the default request_params (this table wins). +[models.embedding.features.knowledge.request_params] +# zh: 按功能覆写示例：认知记忆（cognitive）。 +# en: Per-feature override example: cognitive memory (cognitive). +[models.embedding.features.cognitive] +# zh: 是否完全使用 [models.embedding] 默认配置；true 时本表其余字段全部忽略。 +# en: Whether to fully inherit the [models.embedding] defaults; when true, all other fields here are ignored. +use_default = true +# zh: 以下字段仅在 use_default = false 时生效；只有出现在本表中的字段才会覆写默认配置。 +# en: The fields below apply only when use_default = false; only fields present here override the defaults. +# zh: 覆写 API 地址；空字符串表示继承默认配置。 +# en: Override API URL; an empty string inherits the default. +api_url = "" +# zh: 覆写 API Key；空字符串表示继承默认配置。 +# en: Override API key; an empty string inherits the default. +api_key = "" +# zh: 覆写模型名称；空字符串表示继承默认配置。 +# en: Override model name; an empty string inherits the default. +model_name = "" +# zh: 覆写代理开关；"inherit"（或 "default"）表示继承默认配置，也可写 true / false。 +# en: Override the proxy switch; "inherit" (or "default") inherits the default, true / false override it. +use_proxy = "inherit" +# zh: 覆写上下文窗口上限（token）；<=0 表示继承默认配置。 +# en: Override the context window cap (tokens); <=0 inherits the default. +context_window_tokens = 0 +# zh: 覆写队列发车间隔（秒）；<0 表示继承默认配置，0 表示请求到达立即发车。 +# en: Override the queue interval (seconds); <0 inherits the default, 0 dispatches on arrival. +queue_interval_seconds = -1.0 +# zh: 覆写向量维度；<0 表示继承默认配置，0 表示使用模型默认维度。 +# en: Override embedding dimensions; <0 inherits the default, 0 uses the model default. +dimensions = -1 +# zh: 覆写查询端指令前缀；空字符串表示继承默认配置。 +# en: Override the query instruction prefix; an empty string inherits the default. +query_instruction = "" +# zh: 覆写文档端指令前缀；空字符串表示继承默认配置。 +# en: Override the document instruction prefix; an empty string inherits the default. +document_instruction = "" + +# zh: 覆写额外请求体参数，按 key 合并到默认 request_params 之上（同名以本表为准）。 +# en: Override extra request-body params, merged over the default request_params (this table wins). +[models.embedding.features.cognitive.request_params] +# zh: 按功能覆写示例：梗库（memes）。 +# en: Per-feature override example: meme library (memes). +[models.embedding.features.memes] +# zh: 是否完全使用 [models.embedding] 默认配置；true 时本表其余字段全部忽略。 +# en: Whether to fully inherit the [models.embedding] defaults; when true, all other fields here are ignored. +use_default = true +# zh: 以下字段仅在 use_default = false 时生效；只有出现在本表中的字段才会覆写默认配置。 +# en: The fields below apply only when use_default = false; only fields present here override the defaults. +# zh: 覆写 API 地址；空字符串表示继承默认配置。 +# en: Override API URL; an empty string inherits the default. +api_url = "" +# zh: 覆写 API Key；空字符串表示继承默认配置。 +# en: Override API key; an empty string inherits the default. +api_key = "" +# zh: 覆写模型名称；空字符串表示继承默认配置。 +# en: Override model name; an empty string inherits the default. +model_name = "" +# zh: 覆写代理开关；"inherit"（或 "default"）表示继承默认配置，也可写 true / false。 +# en: Override the proxy switch; "inherit" (or "default") inherits the default, true / false override it. +use_proxy = "inherit" +# zh: 覆写上下文窗口上限（token）；<=0 表示继承默认配置。 +# en: Override the context window cap (tokens); <=0 inherits the default. +context_window_tokens = 0 +# zh: 覆写队列发车间隔（秒）；<0 表示继承默认配置，0 表示请求到达立即发车。 +# en: Override the queue interval (seconds); <0 inherits the default, 0 dispatches on arrival. +queue_interval_seconds = -1.0 +# zh: 覆写向量维度；<0 表示继承默认配置，0 表示使用模型默认维度。 +# en: Override embedding dimensions; <0 inherits the default, 0 uses the model default. +dimensions = -1 +# zh: 覆写查询端指令前缀；空字符串表示继承默认配置。 +# en: Override the query instruction prefix; an empty string inherits the default. +query_instruction = "" +# zh: 覆写文档端指令前缀；空字符串表示继承默认配置。 +# en: Override the document instruction prefix; an empty string inherits the default. +document_instruction = "" + +# zh: 覆写额外请求体参数，按 key 合并到默认 request_params 之上（同名以本表为准）。 +# en: Override extra request-body params, merged over the default request_params (this table wins). +[models.embedding.features.memes.request_params] + # zh: 重排模型配置（知识库二阶段检索使用）。 # en: Rerank model config (used in second-stage knowledge retrieval). [models.rerank] diff --git a/docs/cognitive-memory.md b/docs/cognitive-memory.md index f866bfda..20d3f0f9 100644 --- a/docs/cognitive-memory.md +++ b/docs/cognitive-memory.md @@ -42,6 +42,9 @@ queue_interval_seconds = 0.0 ``` > `models.embedding` 是必要前提。未配置时，即使 `cognitive.enabled = true`，启动时也会自动降级并打印警告。 +> 认知记忆默认复用 `[models.embedding]`；如需独立模型或参数，在 +> `[models.embedding.features.cognitive]` 中设置 `use_default = false` 后按字段覆写， +> 详见 [配置文档](configuration.md#44101-modelsembeddingfeaturesname-按功能覆写)。 启动后验证： @@ -354,7 +357,8 @@ data/cognitive/ ### [models.embedding]（必须配置） -复用知识库的 embedding 配置，无需重复配置： +默认复用知识库、梗库的 embedding 配置，无需重复配置；需要独立模型时用 +`[models.embedding.features.cognitive]` 覆写： | 字段 | 说明 | |------|------| @@ -364,6 +368,9 @@ data/cognitive/ | `queue_interval_seconds` | 发车间隔（默认 `0.0`；`<=0` 请求到达立即发车） | | `dimensions` | 向量维度（可选，模型默认值） | +向量库维度由首次写入确定；更换 `dimensions` 或嵌入模型会改变向量维度， +需要先按 [更换嵌入模型](#更换嵌入模型) 的说明重建向量库。 + ### 热更新说明 - **支持热更新**：`cognitive.query.*`、`cognitive.historian.poll_interval_seconds`、`cognitive.historian.rewrite_max_retry`、`cognitive.historian.recent_messages_inject_k`、`cognitive.historian.recent_message_line_max_len`、`cognitive.historian.source_message_max_len` diff --git a/docs/configuration.md b/docs/configuration.md index 43c90a43..47e2a267 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -494,6 +494,8 @@ Prompt caching 补充： ### 4.4.10 `[models.embedding]` 嵌入模型 +`[models.embedding]` 是所有功能共用的**默认**嵌入配置，也是唯一需要配置的嵌入表。 + | 字段 | 默认值 | 说明 | |---|---:|---| | `api_url` | `""` | 嵌入 API 地址 | @@ -502,10 +504,43 @@ Prompt caching 补充： | `use_proxy` | `false` | 是否使用 `[proxy]` 中的代理地址 | | `queue_interval_seconds` | `0.0` | 发车间隔；`<=0` 表示请求到达立即发车，`>0` 表示两次发车间隔 | | `dimensions` | `0` | 向量维度；`0`/空视为 `None`（模型默认） | -| `query_instruction` | `""` | 查询前缀 | -| `document_instruction` | `""` | 文档前缀 | +| `query_instruction` | `""` | 查询前缀；原样保留首尾空白，与查询文本直接拼接 | +| `document_instruction` | `""` | 文档前缀；原样保留首尾空白，与文档文本直接拼接 | | `request_params` | `{}` | 额外请求体参数；保留字段如 `model`/`input`/`dimensions` 会忽略 | +#### 4.4.10.1 `[models.embedding.features.]` 按功能覆写 + +支持单独设置的功能名：`knowledge`（知识库）、`cognitive`（认知记忆）、`memes`（梗库）。 +每个功能都可以选择完全继承默认配置，或按字段覆写： + +```toml +[models.embedding.features.cognitive] +use_default = false # 单独设置 +model_name = "qwen3-embedding-4b" +dimensions = 2560 +queue_interval_seconds = 1.0 +query_instruction = "Instruct: 检索相关记忆\nQuery: " +document_instruction = "passage: " +``` + +| 字段 | 哨兵值（= 继承默认） | 说明 | +|---|---:|---| +| `use_default` | `true` | `true` 时本表其余字段全部忽略 | +| `api_url` / `api_key` / `model_name` | `""` | 覆写连接与模型名 | +| `use_proxy` | `"inherit"` | 也可写 `"default"`；`true`/`false` 覆盖默认值 | +| `context_window_tokens` | `<=0` | 覆写上下文窗口上限 | +| `queue_interval_seconds` | `<0` | `0` 表示请求到达立即发车 | +| `dimensions` | `<0` | `0` 表示使用模型默认维度 | +| `query_instruction` / `document_instruction` | `""` | 覆写指令前缀；空字符串表示继承。如需“默认带前缀、个别功能不带”，请把默认前缀留空、只在需要的功能上单独设置 | +| `[.request_params]` | 空表 | 按 key 合并到默认 `request_params` 之上，同名以本表为准 | + +语义说明： + +- 未出现在本表中的字段，以及取哨兵值的字段，都表示继续继承 `[models.embedding]`； +- 生效配置完全相同（含指令前缀）的功能共用同一个 Embedder 与发车队列；任一字段不同则该功能拥有独立的 Embedder 与队列，重排器在所有功能间共享； +- 功能名拼写错误或写成未知功能名时会被忽略并记录警告，该功能回落到默认配置； +- 嵌入配置（含 `features` 子表）与 `[models.rerank]` 都在启动时构造运行时，热更新只提示“需要重启生效”，不会改变已运行实例。 + ### 4.4.11 `[models.rerank]` 重排模型 | 字段 | 默认值 | 说明 | diff --git a/docs/knowledge.md b/docs/knowledge.md index 2364de07..e212986d 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -58,6 +58,10 @@ enable_rerank = true # 是否启用重排（可被 tool 参数覆 rerank_top_k = 3 # 重排后返回数量（必须小于 default_top_k） ``` +> `[models.embedding]` 是知识库、认知记忆、梗库共用的默认嵌入配置。若知识库需要独立的模型或参数， +> 在 `[models.embedding.features.knowledge]` 中设置 `use_default = false` 后按字段覆写； +> 详见 [配置文档](configuration.md#44101-modelsembeddingfeaturesname-按功能覆写)。 + **2. 准备知识库目录（含 `intro.md`）** ``` diff --git a/scripts/reembed_cognitive.py b/scripts/reembed_cognitive.py index d2218c1e..053b9d91 100755 --- a/scripts/reembed_cognitive.py +++ b/scripts/reembed_cognitive.py @@ -196,11 +196,12 @@ async def _reembed_collection( def _build_embedder(config: Config) -> Embedder: - """根据 config.toml 构建 Embedder 实例。""" - embedding_config: EmbeddingModelConfig = config.embedding_model + """根据 config.toml 构建 Embedder 实例（使用 cognitive 功能实际生效的配置）。""" + embedding_config: EmbeddingModelConfig = config.resolve_embedding_model("cognitive") if not embedding_config.api_url or not embedding_config.model_name: logger.error( - "config.toml 中 [models.embedding] 未配置 api_url 或 model_name，无法继续。" + "config.toml 中 [models.embedding]（或 " + "[models.embedding.features.cognitive]）未配置 api_url 或 model_name，无法继续。" ) sys.exit(1) @@ -216,10 +217,11 @@ async def _main(args: argparse.Namespace) -> None: db_path = args.db_path or config.cognitive.vector_store_path logger.info("ChromaDB 路径: %s", db_path) + cognitive_embedding = config.resolve_embedding_model("cognitive") logger.info( "嵌入模型: %s (dimensions=%s)", - config.embedding_model.model_name, - config.embedding_model.dimensions or "auto", + cognitive_embedding.model_name, + cognitive_embedding.dimensions or "auto", ) if not Path(db_path).exists(): diff --git a/src/Undefined/config/coercers.py b/src/Undefined/config/coercers.py index 1d2ce56d..e54848fe 100644 --- a/src/Undefined/config/coercers.py +++ b/src/Undefined/config/coercers.py @@ -86,6 +86,18 @@ def _coerce_str(value: Any, default: str) -> str: return normalized if normalized is not None else default +def _coerce_instruction(value: Any, default: str) -> str: + """解析指令前缀，保留原始空白。 + + 指令会与文本直接拼接（如 ``f"{instruction}{text}"``），因此 + ``"passage: "`` 的尾部空格与多行指令的换行都有意义，不能 strip； + 仅空白字符串视为未设置。 + """ + if not isinstance(value, str): + return default + return value if value.strip() else default + + def _normalize_base_url(value: str, default: str) -> str: normalized = value.strip().rstrip("/") if normalized: diff --git a/src/Undefined/config/config_class.py b/src/Undefined/config/config_class.py index 47ffd6be..f7003e85 100644 --- a/src/Undefined/config/config_class.py +++ b/src/Undefined/config/config_class.py @@ -15,6 +15,7 @@ AutomationsConfig, ChatModelConfig, CognitiveConfig, + EmbeddingFeatureOverride, EmbeddingModelConfig, GrokModelConfig, ImageGenConfig, @@ -177,6 +178,8 @@ class Config: lxmusic2api_api_key: str # 嵌入模型 embedding_model: EmbeddingModelConfig + # 按功能覆写的嵌入配置（key 见 EMBEDDING_FEATURES） + embedding_features: dict[str, EmbeddingFeatureOverride] rerank_model: RerankModelConfig # 知识库 knowledge_enabled: bool @@ -572,6 +575,17 @@ def security_check_enabled(self) -> bool: return bool(self.security_model_enabled) + def resolve_embedding_model(self, feature: str) -> EmbeddingModelConfig: + """返回指定功能实际生效的 embedding 配置。 + + `feature` 取 `EMBEDDING_FEATURES` 之一；未单独设置（或缺省）的功能 + 使用 `[models.embedding]` 默认配置。 + """ + override = self.embedding_features.get(feature) + if override is None: + return self.embedding_model + return override.resolve(self.embedding_model) + # 热更新运行时参数 def update_from(self, new_config: "Config") -> dict[str, tuple[Any, Any]]: # 逐字段 diff；嵌套模型配置用 _update_dataclass 展开为 chat_model.api_url 等键 diff --git a/src/Undefined/config/hot_reload.py b/src/Undefined/config/hot_reload.py index af499326..04951fc7 100644 --- a/src/Undefined/config/hot_reload.py +++ b/src/Undefined/config/hot_reload.py @@ -40,6 +40,10 @@ "api.auth_key", "api.openapi_enabled", "naga", + # 嵌入/重排运行时在启动时构造，热更新仅提示需要重启 + "embedding_model", + "embedding_features", + "rerank_model", } _QUEUE_INTERVAL_KEYS: set[str] = { diff --git a/src/Undefined/config/load_sections/knowledge.py b/src/Undefined/config/load_sections/knowledge.py index 12e5887c..6d07c476 100644 --- a/src/Undefined/config/load_sections/knowledge.py +++ b/src/Undefined/config/load_sections/knowledge.py @@ -16,6 +16,7 @@ _get_value, ) from ..parsers import ( + _parse_embedding_feature_overrides, _parse_embedding_model_config, _parse_rerank_model_config, ) @@ -28,6 +29,7 @@ def load_knowledge( ) -> dict[str, Any]: # 知识库段多数项仅读 TOML（env_key=None），避免与 embedding 模型 env 混淆 embedding_model = _parse_embedding_model_config(data) + embedding_features = _parse_embedding_feature_overrides(data) rerank_model = _parse_rerank_model_config(data) knowledge_enabled = _coerce_bool( @@ -107,6 +109,7 @@ def load_knowledge( return { "embedding_model": embedding_model, + "embedding_features": embedding_features, "rerank_model": rerank_model, "knowledge_enabled": knowledge_enabled, "knowledge_base_dir": knowledge_base_dir, diff --git a/src/Undefined/config/model_parsers.py b/src/Undefined/config/model_parsers.py index c34014e2..ea7f0a94 100644 --- a/src/Undefined/config/model_parsers.py +++ b/src/Undefined/config/model_parsers.py @@ -7,6 +7,7 @@ _merge_admins, _parse_agent_model_config, _parse_chat_model_config, + _parse_embedding_feature_overrides, _parse_embedding_model_config, _parse_grok_model_config, _parse_historian_model_config, @@ -27,6 +28,7 @@ "_merge_admins", "_parse_agent_model_config", "_parse_chat_model_config", + "_parse_embedding_feature_overrides", "_parse_embedding_model_config", "_parse_grok_model_config", "_parse_historian_model_config", diff --git a/src/Undefined/config/models.py b/src/Undefined/config/models.py index a3bd687f..14a2f711 100644 --- a/src/Undefined/config/models.py +++ b/src/Undefined/config/models.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from ipaddress import ip_address from typing import Any @@ -11,6 +11,9 @@ HISTORIAN_MIN_POLL_INTERVAL_SECONDS: float = 0.1 PROMPT_FILE_INCLUDE_SLOTS: tuple[str, ...] = ("p0", "p1", "p2", "p3", "summary") +# 支持独立 embedding 配置的功能；顺序即文档与校验顺序 +EMBEDDING_FEATURES: tuple[str, ...] = ("knowledge", "cognitive", "memes") + def format_netloc(host: str, port: int) -> str: """格式化 host:port 为合法 netloc，IPv6 地址自动加方括号。""" @@ -200,6 +203,68 @@ class EmbeddingModelConfig: request_params: dict[str, Any] = field(default_factory=dict) +@dataclass +class EmbeddingFeatureOverride: + """`[models.embedding.features.]` 的按功能覆写配置。 + + 字段的哨兵值表示“继承 ``[models.embedding]`` 默认值”： + + - ``api_url`` / ``api_key`` / ``model_name`` / ``query_instruction`` / + ``document_instruction``：空字符串表示继承； + - ``use_proxy``：``None`` 表示继承； + - ``context_window_tokens``：``<=0`` 表示继承； + - ``queue_interval_seconds``：``<0`` 表示继承（``0`` 为请求到达立即发车）； + - ``dimensions``：``<0`` 表示继承（``0`` 为模型默认维度）； + - ``request_params``：按 key 合并到默认配置之上，同名以本表为准。 + + ``use_default=True`` 时整段忽略，直接使用默认配置。 + """ + + use_default: bool = True + api_url: str = "" + api_key: str = "" + model_name: str = "" + use_proxy: bool | None = None + context_window_tokens: int = 0 + queue_interval_seconds: float = -1.0 + dimensions: int = -1 + query_instruction: str = "" + document_instruction: str = "" + request_params: dict[str, Any] = field(default_factory=dict) + + def resolve(self, default: EmbeddingModelConfig) -> EmbeddingModelConfig: + """把覆写合并到默认配置上，返回该功能实际生效的 embedding 配置。""" + if self.use_default: + return default + request_params = dict(default.request_params) + request_params.update(self.request_params) + return replace( + default, + api_url=self.api_url or default.api_url, + api_key=self.api_key or default.api_key, + model_name=self.model_name or default.model_name, + use_proxy=default.use_proxy if self.use_proxy is None else self.use_proxy, + context_window_tokens=( + default.context_window_tokens + if self.context_window_tokens <= 0 + else self.context_window_tokens + ), + queue_interval_seconds=( + default.queue_interval_seconds + if self.queue_interval_seconds < 0 + else self.queue_interval_seconds + ), + dimensions=( + default.dimensions if self.dimensions < 0 else (self.dimensions or None) + ), + query_instruction=self.query_instruction or default.query_instruction, + document_instruction=( + self.document_instruction or default.document_instruction + ), + request_params=request_params, + ) + + @dataclass class RerankModelConfig: """重排模型配置""" diff --git a/src/Undefined/config/parsers/__init__.py b/src/Undefined/config/parsers/__init__.py index faf5adeb..b53613b9 100644 --- a/src/Undefined/config/parsers/__init__.py +++ b/src/Undefined/config/parsers/__init__.py @@ -3,7 +3,11 @@ # 模型配置解析：原始 dict → ChatModelConfig 等 dataclass from .agent import _parse_agent_model_config from .chat import _parse_chat_model_config -from .embedding import _parse_embedding_model_config, _parse_rerank_model_config +from .embedding import ( + _parse_embedding_feature_overrides, + _parse_embedding_model_config, + _parse_rerank_model_config, +) from .grok import _parse_grok_model_config from .helpers import _log_debug_info, _merge_admins, _verify_required_fields from .historian import _parse_historian_model_config @@ -23,6 +27,7 @@ "_merge_admins", "_parse_agent_model_config", "_parse_chat_model_config", + "_parse_embedding_feature_overrides", "_parse_embedding_model_config", "_parse_grok_model_config", "_parse_historian_model_config", diff --git a/src/Undefined/config/parsers/embedding.py b/src/Undefined/config/parsers/embedding.py index f51e523a..7014b5a4 100644 --- a/src/Undefined/config/parsers/embedding.py +++ b/src/Undefined/config/parsers/embedding.py @@ -11,13 +11,17 @@ from ..coercers import ( _coerce_bool, _coerce_float, + _coerce_instruction, _coerce_int, + _coerce_request_params, _coerce_str, _get_model_request_params, _get_value, _normalize_queue_interval, ) from ..models import ( + EMBEDDING_FEATURES, + EmbeddingFeatureOverride, EmbeddingModelConfig, RerankModelConfig, ) @@ -68,13 +72,13 @@ def _parse_embedding_model_config(data: dict[str, Any]) -> EmbeddingModelConfig: _get_value(data, ("models", "embedding", "dimensions"), None), 0 ) or None, - query_instruction=_coerce_str( + query_instruction=_coerce_instruction( _get_value(data, ("models", "embedding", "query_instruction"), None), "" ), context_window_tokens=_resolve_context_window_tokens( data, "embedding", "EMBEDDING_MODEL_CONTEXT_WINDOW_TOKENS" ), - document_instruction=_coerce_str( + document_instruction=_coerce_instruction( _get_value(data, ("models", "embedding", "document_instruction"), None), "", ), @@ -82,6 +86,73 @@ def _parse_embedding_model_config(data: dict[str, Any]) -> EmbeddingModelConfig: ) +def _coerce_inheritable_bool(value: Any) -> bool | None: + """解析三态布尔：``None`` / "inherit" / "default" 表示继承默认配置。""" + if value is None: + return None + if isinstance(value, str) and value.strip().lower() in { + "inherit", + "default", + "unset", + "", + }: + return None + return _coerce_bool(value, False) + + +def _parse_embedding_feature_overrides( + data: dict[str, Any], +) -> dict[str, EmbeddingFeatureOverride]: + """解析 ``[models.embedding.features.]`` 按功能覆写配置。""" + raw = _get_value(data, ("models", "embedding", "features"), None) + if raw is None: + return {} + if not isinstance(raw, dict): + logger.warning( + "[配置] models.embedding.features 必须是表，实际类型=%s，已忽略", + type(raw).__name__, + ) + return {} + + unknown = sorted(str(name) for name in raw if name not in EMBEDDING_FEATURES) + if unknown: + logger.warning( + "[配置] models.embedding.features 中存在未知功能名，已忽略: %s", + ", ".join(unknown), + ) + + overrides: dict[str, EmbeddingFeatureOverride] = {} + for name in EMBEDDING_FEATURES: + entry = raw.get(name) + if entry is None: + continue + if not isinstance(entry, dict): + logger.warning( + "[配置] models.embedding.features.%s 必须是表，实际类型=%s，已忽略", + name, + type(entry).__name__, + ) + continue + overrides[name] = EmbeddingFeatureOverride( + use_default=_coerce_bool(entry.get("use_default", True), True), + api_url=_coerce_str(entry.get("api_url"), ""), + api_key=_coerce_str(entry.get("api_key"), ""), + model_name=_coerce_str(entry.get("model_name"), ""), + use_proxy=_coerce_inheritable_bool(entry.get("use_proxy")), + context_window_tokens=_coerce_int(entry.get("context_window_tokens"), 0), + queue_interval_seconds=_coerce_float( + entry.get("queue_interval_seconds"), -1.0 + ), + dimensions=_coerce_int(entry.get("dimensions"), -1), + query_instruction=_coerce_instruction(entry.get("query_instruction"), ""), + document_instruction=_coerce_instruction( + entry.get("document_instruction"), "" + ), + request_params=_coerce_request_params(entry.get("request_params")), + ) + return overrides + + def _parse_rerank_model_config(data: dict[str, Any]) -> RerankModelConfig: queue_interval_seconds = _normalize_queue_interval( _coerce_float( @@ -112,7 +183,7 @@ def _parse_rerank_model_config(data: dict[str, Any]) -> RerankModelConfig: context_window_tokens=_resolve_context_window_tokens( data, "rerank", "RERANK_MODEL_CONTEXT_WINDOW_TOKENS" ), - query_instruction=_coerce_str( + query_instruction=_coerce_instruction( _get_value(data, ("models", "rerank", "query_instruction"), None), "" ), request_params=_get_model_request_params(data, "rerank"), diff --git a/src/Undefined/knowledge/__init__.py b/src/Undefined/knowledge/__init__.py index 977d1e9a..9f344010 100644 --- a/src/Undefined/knowledge/__init__.py +++ b/src/Undefined/knowledge/__init__.py @@ -3,6 +3,12 @@ from Undefined.knowledge.embedder import Embedder from Undefined.knowledge.manager import KnowledgeManager from Undefined.knowledge.reranker import Reranker -from Undefined.knowledge.runtime import RetrievalRuntime +from Undefined.knowledge.runtime import RetrievalRuntime, RetrievalRuntimeRegistry -__all__ = ["Embedder", "Reranker", "KnowledgeManager", "RetrievalRuntime"] +__all__ = [ + "Embedder", + "Reranker", + "KnowledgeManager", + "RetrievalRuntime", + "RetrievalRuntimeRegistry", +] diff --git a/src/Undefined/knowledge/runtime.py b/src/Undefined/knowledge/runtime.py index 99b4a7a6..cfae277e 100644 --- a/src/Undefined/knowledge/runtime.py +++ b/src/Undefined/knowledge/runtime.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING from Undefined.knowledge.embedder import Embedder @@ -31,14 +32,21 @@ def __init__( rerank_model: RerankModelConfig, *, embed_batch_size: int = 64, + reranker_provider: Callable[[], Reranker | None] | None = None, ) -> None: self._requester = model_requester self._embedding_model = embedding_model self._rerank_model = rerank_model self._embed_batch_size = int(embed_batch_size) + # 多运行时（按功能拆分 embedding）时由注册表提供共享重排器 + self._reranker_provider = reranker_provider self._embedder: Embedder | None = None self._reranker: Reranker | None = None + @property + def embedding_model(self) -> EmbeddingModelConfig: + return self._embedding_model + @property def rerank_model_ready(self) -> bool: return bool(self._rerank_model.api_url and self._rerank_model.model_name) @@ -74,6 +82,8 @@ async def embed(self, texts: list[str]) -> list[list[float]]: return await embedder.embed(texts) def ensure_reranker(self) -> Reranker | None: + if self._reranker_provider is not None: + return self._reranker_provider() if not self.rerank_model_ready: return None reranker = self._reranker @@ -106,3 +116,82 @@ async def stop(self) -> None: if self._embedder is not None: await self._embedder.stop() self._embedder = None + + +class RetrievalRuntimeRegistry: + """按功能解析 embedding 配置并管理 `RetrievalRuntime` 生命周期。 + + - 每个功能（`EMBEDDING_FEATURES`，如 knowledge / cognitive / memes）取 + `Config.resolve_embedding_model(feature)` 的实际生效配置； + - 生效配置完全相同的功能共用同一个运行时（包括发车队列），避免重复建连 + 与重复限速；任一字段不同时该功能拥有独立的 Embedder 与队列； + - 重排模型在所有功能之间共享。 + """ + + def __init__( + self, + model_requester: ModelRequester, + *, + embedding_models: Mapping[str, EmbeddingModelConfig], + rerank_model: RerankModelConfig, + embed_batch_size: int = 64, + ) -> None: + self._requester = model_requester + self._embedding_models = dict(embedding_models) + self._rerank_model = rerank_model + self._embed_batch_size = int(embed_batch_size) + self._runtimes: list[RetrievalRuntime] = [] + self._reranker: Reranker | None = None + self._reranker_initialized = False + + def for_feature(self, feature: str) -> RetrievalRuntime: + """返回功能对应的检索运行时；相同生效配置复用同一实例。""" + model = self._embedding_models.get(feature) + if model is None: + raise KeyError(f"unknown embedding feature: {feature}") + for runtime in self._runtimes: + if runtime.embedding_model == model: + return runtime + runtime = RetrievalRuntime( + self._requester, + model, + self._rerank_model, + embed_batch_size=self._embed_batch_size, + reranker_provider=self.ensure_reranker, + ) + self._runtimes.append(runtime) + logger.info( + "[检索运行时] 功能已绑定 embedding 配置: feature=%s model=%s interval=%.2fs", + feature, + model.model_name, + model.queue_interval_seconds, + ) + return runtime + + @property + def runtimes(self) -> tuple[RetrievalRuntime, ...]: + return tuple(self._runtimes) + + def ensure_reranker(self) -> Reranker | None: + """共享重排器；未配置完整时返回 None。""" + if not self._reranker_initialized: + self._reranker_initialized = True + if self._rerank_model.api_url and self._rerank_model.model_name: + reranker = Reranker(self._requester, self._rerank_model) + reranker.start() + self._reranker = reranker + logger.info( + "[检索运行时] 重排发车器已启动: interval=%.2fs model=%s", + reranker.interval, + self._rerank_model.model_name, + ) + return self._reranker + + async def stop(self) -> None: + if self._reranker is not None: + await self._reranker.stop() + self._reranker = None + self._reranker_initialized = False + for runtime in self._runtimes: + await runtime.stop() + self._runtimes.clear() diff --git a/src/Undefined/main.py b/src/Undefined/main.py index 5d7e37c6..345d1748 100644 --- a/src/Undefined/main.py +++ b/src/Undefined/main.py @@ -177,7 +177,7 @@ async def main() -> None: meme_service = None meme_worker = None meme_job_queue = None - retrieval_runtime = None + retrieval_registry = None runtime_api_server: RuntimeAPIServer | None = None weixin_service: WeixinService | None = None _reranker: Any = None @@ -201,19 +201,27 @@ async def main() -> None: ) await ai.attachment_registry.load() faq_storage = FAQStorage() - from Undefined.knowledge import RetrievalRuntime + from Undefined.config.models import EMBEDDING_FEATURES + from Undefined.knowledge import RetrievalRuntimeRegistry - retrieval_runtime = RetrievalRuntime( + retrieval_registry = RetrievalRuntimeRegistry( ai._requester, - config.embedding_model, - config.rerank_model, + embedding_models={ + feature: config.resolve_embedding_model(feature) + for feature in EMBEDDING_FEATURES + }, + rerank_model=config.rerank_model, embed_batch_size=config.knowledge_embed_batch_size, ) + retrieval_runtime = retrieval_registry.for_feature("knowledge") + cognitive_retrieval_runtime = retrieval_registry.for_feature("cognitive") + meme_retrieval_runtime = retrieval_registry.for_feature("memes") # === Cognitive Memory === + cognitive_embedding = config.resolve_embedding_model("cognitive") cognitive_actually_enabled = config.cognitive.enabled if cognitive_actually_enabled and ( - not config.embedding_model.api_url or not config.embedding_model.model_name + not cognitive_embedding.api_url or not cognitive_embedding.model_name ): logger.warning( "[认知记忆] cognitive.enabled=true 但 models.embedding 未配置，自动降级禁用" @@ -231,7 +239,7 @@ async def main() -> None: need_reranker_for_knowledge or need_reranker_for_cognitive ) if need_shared_reranker: - _reranker = retrieval_runtime.ensure_reranker() + _reranker = retrieval_registry.ensure_reranker() if _reranker is None: if need_reranker_for_knowledge: logger.warning( @@ -245,12 +253,11 @@ async def main() -> None: if config.knowledge_enabled: from Undefined.knowledge import KnowledgeManager - if ( - not config.embedding_model.api_url - or not config.embedding_model.model_name - ): + knowledge_embedding = config.resolve_embedding_model("knowledge") + if not knowledge_embedding.api_url or not knowledge_embedding.model_name: raise ValueError( - "知识库已启用，但 models.embedding.api_url / model_name 未配置完整" + "知识库已启用，但 models.embedding.api_url / model_name " + "（或 models.embedding.features.knowledge 覆写）未配置完整" ) knowledge_manager = KnowledgeManager( @@ -295,7 +302,7 @@ async def main() -> None: vector_store = CognitiveVectorStore( str(_cog_chroma), - retrieval_runtime, + cognitive_retrieval_runtime, scheduler_foreground_burst=config.cognitive.vector_store_scheduler_foreground_burst, ) job_queue = JobQueue(str(_cog_queues)) @@ -308,7 +315,7 @@ async def main() -> None: vector_store=vector_store, job_queue=job_queue, profile_storage=profile_storage, - retrieval_runtime=retrieval_runtime, + retrieval_runtime=cognitive_retrieval_runtime, ) historian_worker = HistorianWorker( job_queue=job_queue, @@ -339,7 +346,7 @@ async def main() -> None: meme_store = MemeStore(config.memes.db_path) meme_vector_store = MemeVectorStore( config.memes.vector_store_path, - retrieval_runtime, + meme_retrieval_runtime, ) meme_job_queue = JobQueue(config.memes.queue_path) meme_service = MemeService( @@ -349,7 +356,7 @@ async def main() -> None: job_queue=meme_job_queue, ai_client=ai, attachment_registry=ai.attachment_registry, - retrieval_runtime=retrieval_runtime, + retrieval_runtime=meme_retrieval_runtime, ) meme_worker = MemeWorker( job_queue=meme_job_queue, @@ -526,8 +533,8 @@ def _apply_config_updates( await historian_worker.stop() await onebot.disconnect() await ai.close() - if retrieval_runtime is not None: - await retrieval_runtime.stop() + if retrieval_registry is not None: + await retrieval_registry.stop() await config_manager.stop_hot_reload() await close_render_browser() await close_render_cache() diff --git a/tests/test_embedding_feature_config.py b/tests/test_embedding_feature_config.py new file mode 100644 index 00000000..1ccd1089 --- /dev/null +++ b/tests/test_embedding_feature_config.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from Undefined.config.loader import Config +from Undefined.config.models import ( + EMBEDDING_FEATURES, + EmbeddingModelConfig, + RerankModelConfig, +) +from Undefined.knowledge.runtime import RetrievalRuntimeRegistry + +_BASE_TOML = """ +[onebot] +ws_url = "ws://127.0.0.1:3001" + +[models.embedding] +api_url = "https://embed.example.com/v1" +api_key = "embed-key" +model_name = "default-embed" +use_proxy = true +context_window_tokens = 4096 +queue_interval_seconds = 2.0 +dimensions = 1024 +query_instruction = "default-q: " +document_instruction = "default-d: " + +[models.embedding.request_params] +encoding_format = "float" +""" + + +def _load_config(tmp_path: Path, extra: str = "") -> Config: + path = tmp_path / "config.toml" + path.write_text(_BASE_TOML + extra, "utf-8") + return Config.load(path, strict=False) + + +class _DummyRequester: + async def embed( + self, + _model_config: EmbeddingModelConfig, + texts: list[str], + ) -> list[list[float]]: + return [[0.0] * 3 for _ in texts] + + async def rerank( + self, + _model_config: RerankModelConfig, + query: str, + documents: list[str], + top_n: int | None = None, + ) -> list[dict[str, Any]]: + return [] + + +def test_features_absent_means_every_feature_inherits_default(tmp_path: Path) -> None: + cfg = _load_config(tmp_path) + assert cfg.embedding_features == {} + for feature in EMBEDDING_FEATURES: + assert cfg.resolve_embedding_model(feature) == cfg.embedding_model + + +def test_use_default_true_ignores_other_fields(tmp_path: Path) -> None: + cfg = _load_config( + tmp_path, + """ +[models.embedding.features.cognitive] +use_default = true +model_name = "unused-embed" +dimensions = 3 +query_instruction = "unused-q: " +""", + ) + resolved = cfg.resolve_embedding_model("cognitive") + assert resolved == cfg.embedding_model + assert resolved.model_name == "default-embed" + assert resolved.dimensions == 1024 + + +def test_feature_overrides_individual_fields_and_keeps_rest( + tmp_path: Path, +) -> None: + cfg = _load_config( + tmp_path, + """ +[models.embedding.features.memes] +use_default = false +model_name = "meme-embed" +dimensions = 256 +queue_interval_seconds = 0.5 +query_instruction = "meme-q: " +""", + ) + resolved = cfg.resolve_embedding_model("memes") + + assert resolved.model_name == "meme-embed" + assert resolved.dimensions == 256 + assert resolved.queue_interval_seconds == 0.5 + assert resolved.query_instruction == "meme-q: " + # 未覆写字段继续继承默认配置 + assert resolved.api_url == "https://embed.example.com/v1" + assert resolved.api_key == "embed-key" + assert resolved.use_proxy is True + assert resolved.context_window_tokens == 4096 + assert resolved.document_instruction == "default-d: " + assert resolved.request_params == {"encoding_format": "float"} + + +def test_feature_request_params_merge_over_defaults(tmp_path: Path) -> None: + cfg = _load_config( + tmp_path, + """ +[models.embedding.features.knowledge] +use_default = false + +[models.embedding.features.knowledge.request_params] +encoding_format = "base64" +user = "kb" +""", + ) + resolved = cfg.resolve_embedding_model("knowledge") + assert resolved.request_params == { + "encoding_format": "base64", + "user": "kb", + } + # 默认配置本身不受影响 + assert cfg.embedding_model.request_params == {"encoding_format": "float"} + + +def test_feature_use_proxy_tristate(tmp_path: Path) -> None: + cfg = _load_config( + tmp_path, + """ +[models.embedding.features.knowledge] +use_default = false +use_proxy = false + +[models.embedding.features.memes] +use_default = false +use_proxy = "inherit" + +[models.embedding.features.cognitive] +use_default = false +use_proxy = true +""", + ) + assert cfg.resolve_embedding_model("knowledge").use_proxy is False + assert cfg.resolve_embedding_model("memes").use_proxy is True + assert cfg.resolve_embedding_model("cognitive").use_proxy is True + + +def test_feature_dimension_sentinels(tmp_path: Path) -> None: + cfg = _load_config( + tmp_path, + """ +[models.embedding.features.knowledge] +use_default = false +dimensions = 0 + +[models.embedding.features.cognitive] +use_default = false +dimensions = -1 +""", + ) + assert cfg.resolve_embedding_model("knowledge").dimensions is None + assert cfg.resolve_embedding_model("cognitive").dimensions == 1024 + + +def test_feature_context_window_and_interval_sentinels(tmp_path: Path) -> None: + cfg = _load_config( + tmp_path, + """ +[models.embedding.features.knowledge] +use_default = false +context_window_tokens = 512 +queue_interval_seconds = 0.0 +""", + ) + resolved = cfg.resolve_embedding_model("knowledge") + assert resolved.context_window_tokens == 512 + assert resolved.queue_interval_seconds == 0.0 + + +def test_instructions_preserve_whitespace(tmp_path: Path) -> None: + cfg = _load_config( + tmp_path, + """ +[models.embedding.features.knowledge] +use_default = false +document_instruction = "passage: " +query_instruction = "Instruct: task\\nQuery: " + +[models.rerank] +api_url = "https://embed.example.com/v1" +api_key = "rerank-key" +model_name = "rerank-model" +query_instruction = "Query: " +""", + ) + resolved = cfg.resolve_embedding_model("knowledge") + assert resolved.document_instruction == "passage: " + assert resolved.query_instruction == "Instruct: task\nQuery: " + assert cfg.rerank_model.query_instruction == "Query: " + # 仅空白视为未设置 + assert cfg.embedding_model.document_instruction == "default-d: " + + +def test_unknown_feature_is_ignored(tmp_path: Path) -> None: + cfg = _load_config( + tmp_path, + """ +[models.embedding.features.unknown_feature] +use_default = false +model_name = "nope" +""", + ) + assert cfg.embedding_features == {} + assert cfg.resolve_embedding_model("knowledge") == cfg.embedding_model + + +def test_runtime_registry_reuses_runtime_for_equal_configs() -> None: + default = EmbeddingModelConfig( + api_url="https://embed.example.com/v1", + api_key="embed-key", + model_name="default-embed", + ) + registry = RetrievalRuntimeRegistry( + _DummyRequester(), # type: ignore[arg-type] + embedding_models={feature: default for feature in EMBEDDING_FEATURES}, + rerank_model=RerankModelConfig(api_url="", api_key="", model_name=""), + ) + + knowledge_runtime = registry.for_feature("knowledge") + cognitive_runtime = registry.for_feature("cognitive") + + assert knowledge_runtime is cognitive_runtime + assert len(registry.runtimes) == 1 + assert registry.ensure_reranker() is None + + +@pytest.mark.asyncio +async def test_runtime_registry_splits_runtime_when_config_differs() -> None: + default = EmbeddingModelConfig( + api_url="https://embed.example.com/v1", + api_key="embed-key", + model_name="default-embed", + ) + memes = EmbeddingModelConfig( + api_url="https://embed.example.com/v1", + api_key="embed-key", + model_name="meme-embed", + dimensions=256, + ) + registry = RetrievalRuntimeRegistry( + _DummyRequester(), # type: ignore[arg-type] + embedding_models={ + "knowledge": default, + "cognitive": default, + "memes": memes, + }, + rerank_model=RerankModelConfig( + api_url="https://embed.example.com/v1", + api_key="rerank-key", + model_name="rerank-model", + ), + ) + + knowledge_runtime = registry.for_feature("knowledge") + cognitive_runtime = registry.for_feature("cognitive") + meme_runtime = registry.for_feature("memes") + + assert knowledge_runtime is cognitive_runtime + assert meme_runtime is not knowledge_runtime + assert len(registry.runtimes) == 2 + assert meme_runtime.embedding_model.model_name == "meme-embed" + + try: + reranker = registry.ensure_reranker() + assert reranker is not None + # 重排在所有功能间共享同一个实例 + assert knowledge_runtime.ensure_reranker() is reranker + assert meme_runtime.ensure_reranker() is reranker + finally: + await registry.stop() + + +def test_runtime_registry_unknown_feature() -> None: + registry = RetrievalRuntimeRegistry( + _DummyRequester(), # type: ignore[arg-type] + embedding_models={"knowledge": EmbeddingModelConfig("", "", "m")}, + rerank_model=RerankModelConfig(api_url="", api_key="", model_name=""), + ) + with pytest.raises(KeyError): + registry.for_feature("unknown") From 4814a08a7d04aa75e8c6f29031c3680ffd588584 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:30:57 +0800 Subject: [PATCH 02/30] =?UTF-8?q?docs(config):=20[prompt.file=5Fincludes]?= =?UTF-8?q?=20=E6=B3=A8=E9=87=8A=E6=8B=86=E5=88=86=E5=88=B0=E5=90=84?= =?UTF-8?q?=E6=8F=92=E6=A7=BD=E4=BD=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 聚合在 p0 上方的说明改为每个插槽各自的注释，明确写入的 Prompt 区块。 --- config.toml.example | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/config.toml.example b/config.toml.example index 180da1d4..c737e682 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1188,12 +1188,20 @@ tool_search_max_results = 5 # zh: 主 Prompt 固定插槽对应的本地 UTF-8 文件。路径为空时不注入。 # en: Local UTF-8 files mapped to stable main-Prompt slots. Empty paths disable injection. [prompt.file_includes] -# zh: P0 / P1 / P2 / P3 分别位于对应优先级区块开头；summary 位于总结区块开头。 -# en: P0-P3 are placed at the start of their priority sections; summary is placed at the start of the summary section. +# zh: 插入 开头（绝对优先级区块）。 +# en: Inserted at the start of the block. p0 = "" +# zh: 插入 开头（核心规则区块）。 +# en: Inserted at the start of the block. p1 = "" +# zh: 插入 开头（重要规则区块）。 +# en: Inserted at the start of the block. p2 = "" +# zh: 插入 开头（优化规则区块）。 +# en: Inserted at the start of the block. p3 = "" +# zh: 插入 开头（总结区块）。 +# en: Inserted at the start of the block. summary = "" # zh: Prompt 系统信息注入。总开关默认关闭；开启后各子项默认展示，可逐项关闭。 From e46bd59cf8a4164c073ef69469a60bacb1a407a6 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:36:52 +0800 Subject: [PATCH 03/30] =?UTF-8?q?feat(webui):=20=E9=80=82=E9=85=8D=20embed?= =?UTF-8?q?ding=20=E6=8C=89=E5=8A=9F=E8=83=BD=E8=A6=86=E5=86=99=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 覆写段说明移到区段提示，字段注释各自归位，避免被解析成同一段 hint； - use_proxy 在三态语义下改用下拉选择（inherit / true / false）。 --- config.toml.example | 21 +++++++++----------- src/Undefined/webui/static/js/config-form.js | 6 ++++++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/config.toml.example b/config.toml.example index c737e682..9f1bf05e 100644 --- a/config.toml.example +++ b/config.toml.example @@ -727,14 +727,12 @@ document_instruction = "" # [models.embedding] defaults or overrides individual fields; features with # the same effective config share one dispatch queue. -# zh: 按功能覆写示例：知识库（knowledge）。 -# en: Per-feature override example: knowledge base (knowledge). +# zh: 按功能覆写示例：知识库（knowledge）。仅 use_default = false 时下方字段生效，取哨兵值仍表示继承默认配置。 +# en: Per-feature override example: knowledge base (knowledge). Fields below apply only when use_default = false; sentinel values inherit the defaults. [models.embedding.features.knowledge] # zh: 是否完全使用 [models.embedding] 默认配置；true 时本表其余字段全部忽略。 # en: Whether to fully inherit the [models.embedding] defaults; when true, all other fields here are ignored. use_default = true -# zh: 以下字段仅在 use_default = false 时生效；只有出现在本表中的字段才会覆写默认配置。 -# en: The fields below apply only when use_default = false; only fields present here override the defaults. # zh: 覆写 API 地址；空字符串表示继承默认配置。 # en: Override API URL; an empty string inherits the default. api_url = "" @@ -766,14 +764,12 @@ document_instruction = "" # zh: 覆写额外请求体参数，按 key 合并到默认 request_params 之上（同名以本表为准）。 # en: Override extra request-body params, merged over the default request_params (this table wins). [models.embedding.features.knowledge.request_params] -# zh: 按功能覆写示例：认知记忆（cognitive）。 -# en: Per-feature override example: cognitive memory (cognitive). +# zh: 按功能覆写示例：认知记忆（cognitive）。仅 use_default = false 时下方字段生效，取哨兵值仍表示继承默认配置。 +# en: Per-feature override example: cognitive memory (cognitive). Fields below apply only when use_default = false; sentinel values inherit the defaults. [models.embedding.features.cognitive] # zh: 是否完全使用 [models.embedding] 默认配置；true 时本表其余字段全部忽略。 # en: Whether to fully inherit the [models.embedding] defaults; when true, all other fields here are ignored. use_default = true -# zh: 以下字段仅在 use_default = false 时生效；只有出现在本表中的字段才会覆写默认配置。 -# en: The fields below apply only when use_default = false; only fields present here override the defaults. # zh: 覆写 API 地址；空字符串表示继承默认配置。 # en: Override API URL; an empty string inherits the default. api_url = "" @@ -805,14 +801,12 @@ document_instruction = "" # zh: 覆写额外请求体参数，按 key 合并到默认 request_params 之上（同名以本表为准）。 # en: Override extra request-body params, merged over the default request_params (this table wins). [models.embedding.features.cognitive.request_params] -# zh: 按功能覆写示例：梗库（memes）。 -# en: Per-feature override example: meme library (memes). +# zh: 按功能覆写示例：梗库（memes）。仅 use_default = false 时下方字段生效，取哨兵值仍表示继承默认配置。 +# en: Per-feature override example: meme library (memes). Fields below apply only when use_default = false; sentinel values inherit the defaults. [models.embedding.features.memes] # zh: 是否完全使用 [models.embedding] 默认配置；true 时本表其余字段全部忽略。 # en: Whether to fully inherit the [models.embedding] defaults; when true, all other fields here are ignored. use_default = true -# zh: 以下字段仅在 use_default = false 时生效；只有出现在本表中的字段才会覆写默认配置。 -# en: The fields below apply only when use_default = false; only fields present here override the defaults. # zh: 覆写 API 地址；空字符串表示继承默认配置。 # en: Override API URL; an empty string inherits the default. api_url = "" @@ -1786,6 +1780,9 @@ poll_interval_seconds = 1.0 # zh: 任务超时时间（秒），超时后重新入队。 # en: Stale job timeout (seconds), re-queued after timeout. stale_job_timeout_seconds = 300.0 +# zh: 史官同时在途处理的任务上限（>=1）；超出后暂停取新任务，避免并发无上限。 +# en: Max in-flight historian jobs (>=1); new jobs wait until in-flight drops below it. +max_concurrency = 4 [cognitive.profile] # zh: 用户画像存储路径。 diff --git a/src/Undefined/webui/static/js/config-form.js b/src/Undefined/webui/static/js/config-form.js index 187e8284..1b00b5ed 100644 --- a/src/Undefined/webui/static/js/config-form.js +++ b/src/Undefined/webui/static/js/config-form.js @@ -428,6 +428,12 @@ const FIELD_SELECT_OPTION_RULES = [ match: (path) => path === "message_batcher.strategy", options: ["extend", "fixed"], }, + { + // 按功能覆写的 embedding 代理开关是三态的：inherit 表示继承默认配置 + match: (path) => + /^models\.embedding\.features\.[^.]+\.use_proxy$/.test(path), + options: ["inherit", "true", "false"], + }, ]; /** @type {Record>} */ From f60f01ed25926cca62f8362cf5fbd427f30e74dd Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:36:57 +0800 Subject: [PATCH 04/30] =?UTF-8?q?fix(cognitive):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=BE=A7=E5=86=99=E5=90=88=E5=B9=B6=E4=B8=A2=E5=A4=B1=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E5=B9=B6=E8=A1=A5=E9=BD=90=E6=81=A2=E5=A4=8D=E5=85=A5?= =?UTF-8?q?=E5=8F=A3=E4=B8=8E=E5=B9=B6=E5=8F=91=E4=B8=8A=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 同一实体的「读取 → LLM 改写 → 写入」整段互斥：并发合并不再互相覆盖， 后一个任务基于前一个任务已落盘的侧写继续合并（merge_guard 与写入锁分离， 避免同锁重入死锁）； - ProfileStorage 新增 read_revision / restore_revision，恢复前把当前内容 另存为新快照，恢复操作本身可回退；版本名做路径穿越校验； - 新增 scripts/restore_profile.py 作为历史版本的 list / show / restore 入口， 侧写回滚不再依赖手工 cp； - 史官 worker 增加 max_concurrency（默认 4，cognitive.historian.max_concurrency） 与在途计数 / 信号量双重约束，不再无上限并发；补 CLI、示例与文档。 --- docs/cognitive-memory.md | 24 ++- docs/configuration.md | 1 + scripts/README.md | 20 ++ scripts/restore_profile.py | 144 ++++++++++++++ src/Undefined/cognitive/historian/worker.py | 57 ++++-- src/Undefined/cognitive/profile_storage.py | 67 +++++++ src/Undefined/config/domain_parsers.py | 7 + src/Undefined/config/models.py | 2 + src/Undefined/main.py | 1 + tests/test_cognitive_profile_revision.py | 202 ++++++++++++++++++++ 10 files changed, 504 insertions(+), 21 deletions(-) create mode 100644 scripts/restore_profile.py create mode 100644 tests/test_cognitive_profile_revision.py diff --git a/docs/cognitive-memory.md b/docs/cognitive-memory.md index 20d3f0f9..d8f7b407 100644 --- a/docs/cognitive-memory.md +++ b/docs/cognitive-memory.md @@ -337,6 +337,7 @@ data/cognitive/ | `source_message_max_len` | int | `800` | 当前消息原文最大长度（支持热更新） | | `poll_interval_seconds` | float | `1.0` | 史官轮询间隔秒数，小于 `0.1` 时按 `0.1` 处理（支持热更新） | | `stale_job_timeout_seconds` | float | `300.0` | 启动时恢复 stale 任务的超时阈值 | +| `max_concurrency` | int | `4` | 史官同时在途任务上限（最小 `1`），超出后暂停取新任务；需重启生效 | ### [cognitive.profile] @@ -461,17 +462,24 @@ enabled = false **级别 2：侧写回滚** -若某用户侧写被错误更新，从快照目录恢复： +若某用户侧写被错误更新，用 [`scripts/restore_profile.py`](../scripts/restore_profile.py) 从快照目录恢复。 +恢复前会把当前内容另存为新快照，因此恢复操作本身也可再次回退： ```bash # 查看快照列表 -ls data/cognitive/profiles/history/users/{user_id}/ +uv run python scripts/restore_profile.py list --entity-type user --entity-id {user_id} -# 覆盖回正确版本 -cp data/cognitive/profiles/history/users/{user_id}/{timestamp}.md \ - data/cognitive/profiles/users/{user_id}.md +# 预览某个版本内容（不改动文件） +uv run python scripts/restore_profile.py show --entity-type user --entity-id {user_id} --revision {timestamp}.md + +# 恢复该版本（先 dry-run 确认，再实际恢复） +uv run python scripts/restore_profile.py restore --entity-type user --entity-id {user_id} --revision {timestamp}.md --dry-run +uv run python scripts/restore_profile.py restore --entity-type user --entity-id {user_id} --revision {timestamp}.md ``` +恢复只改侧写 Markdown 与历史快照，不会更新 ChromaDB 中的侧写向量；若同一实体在 +`cognitive_profiles` 里有旧向量，请按[更换嵌入模型](#更换嵌入模型)的方式重嵌入侧写。 + **级别 3：完整移除** ```bash @@ -531,4 +539,8 @@ failed 文件中包含原始 job 数据和 `error` 字段，记录失败原因 **Q: 史官处理速度跟不上怎么办？** -默认是单 worker 串行处理，每个任务需要 1-2 次 LLM 调用。高并发场景下 `pending/` 目录会积压，但不影响前台响应。可适当降低 `poll_interval_seconds` 或扩展多 worker 加快消费速度。 +单个 worker 按 `cognitive.historian.max_concurrency`（默认 4）并发处理任务，每个任务需要 1-2 次 LLM 调用。高并发场景下 `pending/` 目录会积压，但不影响前台响应；可提高 `max_concurrency`（需重启）或降低 `poll_interval_seconds` 加快消费速度。提高并发会同步放大 LLM 调用量与费用，请按模型配额评估。 + +**Q: 同一实体的两个任务同时改写侧写，会不会丢观察？** + +不会。侧写合并的「读取 → LLM 改写 → 写入」整段按实体互斥执行，同一用户/群聊的相邻任务会串行改写，后一个任务基于前一个任务已落盘的侧写继续合并。 diff --git a/docs/configuration.md b/docs/configuration.md index 47e2a267..bf1f93d1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1236,6 +1236,7 @@ api_key = "replace-with-your-key" | `source_message_max_len` | `800` | 当前触发消息最大字符数 | | `poll_interval_seconds` | `1.0` | 队列轮询间隔；小于 `0.1` 时按 `0.1` 秒处理，避免空队列忙循环 | | `stale_job_timeout_seconds` | `300.0` | processing 超时回收阈值 | +| `max_concurrency` | `4` | 史官同时在途任务上限（最小 `1`）；超出后暂停取新任务，需重启生效 | ### 4.26.5 `[cognitive.profile]` diff --git a/scripts/README.md b/scripts/README.md index c2d694d0..c82d7518 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -89,6 +89,26 @@ uv run python scripts/reembed_cognitive.py -v - 大量记录时注意 API 限速，可通过 `--batch-size` 降低并发 - 建议先用 `--dry-run` 确认记录数量和配置正确性 +### [`restore_profile.py`](restore_profile.py) — 认知记忆侧写历史版本恢复 + +侧写每次写入前会把旧内容存成历史快照（`cognitive.profile.revision_keep`，默认 5 份）。本脚本提供历史版本的列出、查看与恢复入口。 + +```bash +# 列出某用户/群聊的历史版本 +uv run python scripts/restore_profile.py list --entity-type user --entity-id 123456 + +# 查看某个历史版本内容 +uv run python scripts/restore_profile.py show --entity-type user --entity-id 123456 --revision 20260101000000000000.md + +# 恢复某个历史版本（恢复前会把当前内容另存为新快照，可再次回退） +uv run python scripts/restore_profile.py restore --entity-type user --entity-id 123456 --revision 20260101000000000000.md + +# 仅预览，不写盘 +uv run python scripts/restore_profile.py restore --entity-type user --entity-id 123456 --revision 20260101000000000000.md --dry-run +``` + +**注意**：恢复只改侧写 Markdown 与历史快照，不会更新 ChromaDB 中的侧写向量；需要同步检索结果时按 `docs/cognitive-memory.md` 的说明重嵌入。 + ### release_notes.py — 发布版本校验与 Release notes 生成 Release workflow 使用这个脚本在构建前校验版本一致性，并在发布阶段从 `CHANGELOG.md` 最新版本条目生成 GitHub Release 说明。Release notes 会先写入 changelog 自动提取内容，再用 `---` 分隔并追加 `Detailed Changes`，按上一个 tag 到当前 tag 的 commit 主题分类列出 features、bug fixes 和 maintenance/others。 diff --git a/scripts/restore_profile.py b/scripts/restore_profile.py new file mode 100644 index 00000000..c18993d4 --- /dev/null +++ b/scripts/restore_profile.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""认知记忆侧写历史版本的查看与恢复脚本。 + +侧写在每次写入前会把旧内容存成历史快照（默认保留 `cognitive.profile.revision_keep` +份），但历史快照此前只能手工 `ls` / `cp`，没有正式的读取与恢复入口。本脚本提供： + + # 列出某个用户/群聊的全部历史版本 + uv run python scripts/restore_profile.py list --entity-type user --entity-id 123456 + + # 查看某个历史版本内容 + uv run python scripts/restore_profile.py show --entity-type user --entity-id 123456 --revision 20260101000000000000.md + + # 恢复某个历史版本（恢复前会把当前内容另存为新快照，可再次回退） + uv run python scripts/restore_profile.py restore --entity-type user --entity-id 123456 --revision 20260101000000000000.md + + # 仅预览将要恢复的内容（不写盘） + uv run python scripts/restore_profile.py restore --entity-type user --entity-id 123456 --revision 20260101000000000000.md --dry-run + +恢复只改侧写 Markdown 文件与历史快照，不会重建 ChromaDB 中的侧写向量；如需同时刷新 +检索向量，请按 docs/cognitive-memory.md 的说明重嵌入。 +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys +from pathlib import Path + +# 将项目 src 加入 sys.path，使脚本可以直接 uv run 执行 +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +_SRC_DIR = _PROJECT_ROOT / "src" +if str(_SRC_DIR) not in sys.path: + sys.path.insert(0, str(_SRC_DIR)) + +from Undefined.cognitive.profile_storage import ProfileStorage # noqa: E402 +from Undefined.config.loader import Config # noqa: E402 + +logger = logging.getLogger("restore_profile") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="认知记忆侧写历史版本查看与恢复", + ) + parser.add_argument( + "action", + choices=("list", "show", "restore"), + help="list=列出历史版本；show=查看版本内容；restore=恢复版本", + ) + parser.add_argument( + "--entity-type", + required=True, + choices=("user", "group"), + help="实体类型", + ) + parser.add_argument("--entity-id", required=True, help="用户 ID 或群 ID") + parser.add_argument( + "--revision", + default="", + help="历史版本文件名（show / restore 必填，取值来自 list 输出）", + ) + parser.add_argument( + "--profiles-path", + default="", + help="侧写目录（默认读取 config.toml 中的 cognitive.profiles_path）", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="只打印将要执行的操作，不写盘", + ) + parser.add_argument("-v", "--verbose", action="store_true", help="输出调试日志") + args = parser.parse_args() + if args.action in {"show", "restore"} and not args.revision: + parser.error("show / restore 需要 --revision") + return args + + +def _print_revisions(revisions: list[str]) -> None: + if not revisions: + print("（暂无历史版本）") + return + for index, name in enumerate(revisions, start=1): + print(f"{index:>3}. {name}") + + +async def _main(args: argparse.Namespace) -> None: + config = Config.load(strict=False) + base_path = args.profiles_path or config.cognitive.profiles_path + storage = ProfileStorage( + base_path, + revision_keep=config.cognitive.profile_revision_keep, + ) + entity_type = args.entity_type + entity_id = args.entity_id + + if args.action == "list": + revisions = await storage.list_revisions(entity_type, entity_id) + print(f"侧写目录: {base_path}") + print(f"实体: {entity_type}:{entity_id}") + _print_revisions(revisions) + return + + if args.action == "show": + content = await storage.read_revision(entity_type, entity_id, args.revision) + if content is None: + logger.error( + "历史版本不存在: %s:%s/%s", entity_type, entity_id, args.revision + ) + sys.exit(1) + print(content) + return + + # restore + content = await storage.read_revision(entity_type, entity_id, args.revision) + if content is None: + logger.error("历史版本不存在: %s:%s/%s", entity_type, entity_id, args.revision) + sys.exit(1) + if args.dry_run: + print( + f"[dry-run] 将把 {entity_type}:{entity_id} 恢复为 {args.revision}，" + f"当前内容会先存为新快照。" + ) + print("---- 恢复后的内容 ----") + print(content) + return + restored = await storage.restore_revision(entity_type, entity_id, args.revision) + print(f"已恢复 {entity_type}:{entity_id} 到历史版本 {restored}") + print("当前内容已另存为新快照，可再次用 list / restore 回退。") + + +def main() -> None: + args = _parse_args() + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s %(name)s: %(message)s", + ) + asyncio.run(_main(args)) + + +if __name__ == "__main__": + main() diff --git a/src/Undefined/cognitive/historian/worker.py b/src/Undefined/cognitive/historian/worker.py index 16e3e349..3071b379 100644 --- a/src/Undefined/cognitive/historian/worker.py +++ b/src/Undefined/cognitive/historian/worker.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import json import logging from datetime import datetime, timezone, tzinfo @@ -49,6 +50,7 @@ def __init__( ai_client: Any, config_getter: Callable[[], Any], model_config: Any = None, + max_concurrency: int = 4, ) -> None: self._job_queue = job_queue self._vector_store = vector_store @@ -56,9 +58,11 @@ def __init__( self._ai_client = ai_client self._config_getter = config_getter self._model_config = model_config + self._max_concurrency = max(1, int(max_concurrency)) self._stop_event = asyncio.Event() self._task: asyncio.Task[None] | None = None self._inflight_tasks: set[asyncio.Task[None]] = set() + self._semaphore = asyncio.Semaphore(self._max_concurrency) async def _prepare_query_embedding(self, query_text: str) -> list[float] | None: embed_query = getattr(self._vector_store, "embed_query", None) @@ -97,8 +101,16 @@ async def _poll_loop(self) -> None: dispatch_count = 0 logger.info("[史官] 轮询循环已开始") while not self._stop_event.is_set(): - result = await self._job_queue.dequeue() config = self._config_getter() + poll_interval = max( + HISTORIAN_MIN_POLL_INTERVAL_SECONDS, + float(config.poll_interval_seconds), + ) + if len(self._inflight_tasks) >= self._max_concurrency: + # 在途任务达到上限时先不取新任务，避免无界并发与内存堆积 + await asyncio.sleep(poll_interval) + continue + result = await self._job_queue.dequeue() if result: job_id, job = result task = asyncio.create_task(self._process_job_with_retry(job_id, job)) @@ -128,12 +140,7 @@ async def _poll_loop(self) -> None: config.failed_max_files, ) - await asyncio.sleep( - max( - HISTORIAN_MIN_POLL_INTERVAL_SECONDS, - float(config.poll_interval_seconds), - ) - ) + await asyncio.sleep(poll_interval) if self._inflight_tasks: logger.info( @@ -143,6 +150,13 @@ async def _poll_loop(self) -> None: logger.info("[史官] 轮询循环已结束") async def _process_job_with_retry(self, job_id: str, job: dict[str, Any]) -> None: + # 并发上限由 _poll_loop 的在途计数与这里的信号量双重约束 + async with self._semaphore: + await self._process_job_with_retry_inner(job_id, job) + + async def _process_job_with_retry_inner( + self, job_id: str, job: dict[str, Any] + ) -> None: try: await self._process_job(job_id, job) except Exception as e: @@ -449,14 +463,16 @@ async def _merge_profiles( success_count = 0 for index, target in enumerate(targets, start=1): try: - merged = await self._merge_profile_target( - job=job, - canonical=canonical, - event_id=event_id, - target=target, - target_index=index, - target_count=len(targets), - ) + # 同一实体的「读 → LLM → 写」整段互斥，避免并发合并互相覆盖 + async with self._profile_merge_guard(target): + merged = await self._merge_profile_target( + job=job, + canonical=canonical, + event_id=event_id, + target=target, + target_index=index, + target_count=len(targets), + ) if merged: success_count += 1 except Exception as exc: @@ -475,6 +491,17 @@ async def _merge_profiles( len(targets), ) + def _profile_merge_guard(self, target: dict[str, str]) -> Any: + """返回目标实体的合并互斥锁；存储层未提供时退化为无锁上下文。""" + guard = getattr(self._profile_storage, "merge_guard", None) + if not callable(guard): + return contextlib.nullcontext() + entity_type = str(target.get("entity_type", "")) + entity_id = str(target.get("entity_id", "")) + if not entity_type or not entity_id: + return contextlib.nullcontext() + return guard(entity_type, entity_id) + async def _write_profile( self, *, diff --git a/src/Undefined/cognitive/profile_storage.py b/src/Undefined/cognitive/profile_storage.py index d1f61215..14f8e0ae 100644 --- a/src/Undefined/cognitive/profile_storage.py +++ b/src/Undefined/cognitive/profile_storage.py @@ -18,6 +18,7 @@ def __init__(self, base_path: str | Path, revision_keep: int = 5) -> None: self._base = Path(base_path) self._revision_keep = revision_keep self._locks: dict[str, asyncio.Lock] = {} + self._merge_locks: dict[str, asyncio.Lock] = {} logger.info( "[认知侧写] 初始化完成: base=%s revision_keep=%s", str(self._base), @@ -30,6 +31,18 @@ def _get_lock(self, entity_type: str, entity_id: str) -> asyncio.Lock: self._locks[key] = asyncio.Lock() return self._locks[key] + def merge_guard(self, entity_type: str, entity_id: str) -> asyncio.Lock: + """跨「读 → LLM → 写」整段侧写合并的互斥锁。 + + 只串行化同一实体的合并周期，避免两个 job 各自基于旧快照改写后互相覆盖 + （后写覆盖先写，先前的观察永久丢失）。与文件写入锁分开，避免与 + `write_profile` 的锁重入死锁。 + """ + key = f"{entity_type}:{entity_id}" + if key not in self._merge_locks: + self._merge_locks[key] = asyncio.Lock() + return self._merge_locks[key] + def _profile_path(self, entity_type: str, entity_id: str) -> Path: return self._base / f"{entity_type}s" / f"{entity_id}.md" @@ -133,6 +146,60 @@ def _list() -> list[str]: ) return result + @staticmethod + def _normalize_revision_name(revision: str) -> str: + name = str(revision).strip() + # 只接受历史目录下的单层文件名，阻断路径穿越 + if not name or name != Path(name).name or not name.endswith(".md"): + raise ValueError(f"非法的侧写历史版本名: {revision!r}") + return name + + async def read_revision( + self, entity_type: str, entity_id: str, revision: str + ) -> str | None: + """读取指定历史版本内容；版本名取 `list_revisions` 的返回值。""" + name = self._normalize_revision_name(revision) + path = self._history_dir(entity_type, entity_id) / name + + def _read() -> str | None: + if not path.exists(): + return None + return path.read_text(encoding="utf-8") + + content = await asyncio.to_thread(_read) + logger.info( + "[认知侧写] 读取历史版本: entity_type=%s entity_id=%s revision=%s found=%s", + entity_type, + entity_id, + name, + content is not None, + ) + return content + + async def restore_revision( + self, entity_type: str, entity_id: str, revision: str + ) -> str: + """把指定历史版本恢复为当前侧写。 + + 恢复前会把当前内容按常规流程存成新快照，因此恢复操作本身也可回退。 + 返回被恢复的版本名。 + """ + name = self._normalize_revision_name(revision) + content = await self.read_revision(entity_type, entity_id, name) + if content is None: + raise FileNotFoundError( + f"侧写历史版本不存在: {entity_type}:{entity_id}/{name}" + ) + async with self.merge_guard(entity_type, entity_id): + await self.write_profile(entity_type, entity_id, content) + logger.info( + "[认知侧写] 已恢复历史版本: entity_type=%s entity_id=%s revision=%s", + entity_type, + entity_id, + name, + ) + return name + @staticmethod def _sanitize_profile(content: str, entity_type: str, entity_id: str) -> str: import yaml diff --git a/src/Undefined/config/domain_parsers.py b/src/Undefined/config/domain_parsers.py index 17f24171..d6eaa153 100644 --- a/src/Undefined/config/domain_parsers.py +++ b/src/Undefined/config/domain_parsers.py @@ -136,6 +136,13 @@ def _parse_cognitive_config(data: dict[str, Any]) -> CognitiveConfig: hist.get("source_message_max_len") if isinstance(hist, dict) else None, 800, ), + historian_max_concurrency=max( + 1, + _coerce_int( + hist.get("max_concurrency") if isinstance(hist, dict) else None, + 4, + ), + ), poll_interval_seconds=max( HISTORIAN_MIN_POLL_INTERVAL_SECONDS, _coerce_float( diff --git a/src/Undefined/config/models.py b/src/Undefined/config/models.py index 14a2f711..07ccb6ee 100644 --- a/src/Undefined/config/models.py +++ b/src/Undefined/config/models.py @@ -554,6 +554,8 @@ class CognitiveConfig: historian_recent_message_line_max_len: int = 240 # Max characters for the current source message attached to historian jobs. historian_source_message_max_len: int = 800 + # Historian worker 同时在途处理的任务上限。 + historian_max_concurrency: int = 4 @dataclass diff --git a/src/Undefined/main.py b/src/Undefined/main.py index 345d1748..4995d115 100644 --- a/src/Undefined/main.py +++ b/src/Undefined/main.py @@ -324,6 +324,7 @@ async def main() -> None: ai_client=ai, config_getter=lambda: get_config(strict=False).cognitive, model_config=config.historian_model, + max_concurrency=config.cognitive.historian_max_concurrency, ) ai.set_cognitive_service(cognitive_service) logger.info( diff --git a/tests/test_cognitive_profile_revision.py b/tests/test_cognitive_profile_revision.py new file mode 100644 index 00000000..a6ab577b --- /dev/null +++ b/tests/test_cognitive_profile_revision.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from Undefined.cognitive.historian import HistorianWorker +from Undefined.cognitive.profile_storage import ProfileStorage + + +def _write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +@pytest.mark.asyncio +async def test_restore_revision_roundtrip_keeps_current_as_snapshot( + tmp_path: Path, +) -> None: + storage = ProfileStorage(tmp_path, revision_keep=5) + await storage.write_profile("user", "10001", "v1") + await storage.write_profile("user", "10001", "v2") + + revisions = await storage.list_revisions("user", "10001") + assert len(revisions) == 1 + assert await storage.read_revision("user", "10001", revisions[0]) == "v1" + + restored = await storage.restore_revision("user", "10001", revisions[0]) + assert restored == revisions[0] + assert await storage.read_profile("user", "10001") == "v1" + # 恢复前的 v2 被存成了新快照，恢复本身可再次回退 + after = await storage.list_revisions("user", "10001") + assert len(after) == 2 + contents = [await storage.read_revision("user", "10001", name) for name in after] + assert "v2" in contents + + +@pytest.mark.asyncio +async def test_read_revision_missing_returns_none(tmp_path: Path) -> None: + storage = ProfileStorage(tmp_path) + assert ( + await storage.read_revision("user", "10001", "20260101000000000000.md") is None + ) + + +@pytest.mark.asyncio +async def test_read_revision_rejects_path_traversal(tmp_path: Path) -> None: + storage = ProfileStorage(tmp_path) + for bad in ("../secret.md", "/etc/passwd", "sub/dir.md", "plain"): + with pytest.raises(ValueError): + await storage.read_revision("user", "10001", bad) + with pytest.raises(ValueError): + await storage.restore_revision("user", "10001", "../secret.md") + + +@pytest.mark.asyncio +async def test_restore_revision_missing_raises(tmp_path: Path) -> None: + storage = ProfileStorage(tmp_path) + with pytest.raises(FileNotFoundError): + await storage.restore_revision("user", "10001", "20260101000000000000.md") + + +@pytest.mark.asyncio +async def test_merge_guard_serializes_read_llm_write_cycles(tmp_path: Path) -> None: + storage = ProfileStorage(tmp_path) + order: list[str] = [] + + async def merge(tag: str) -> None: + async with storage.merge_guard("user", "10001"): + order.append(f"{tag}:enter") + content = await storage.read_profile("user", "10001") + await asyncio.sleep(0.05) # 模拟 LLM 改写耗时 + await storage.write_profile( + "user", "10001", f"{content or ''}+{tag}".strip("+") + ) + order.append(f"{tag}:exit") + + await asyncio.gather(merge("a"), merge("b")) + + assert order in ( + ["a:enter", "a:exit", "b:enter", "b:exit"], + ["b:enter", "b:exit", "a:enter", "a:exit"], + ) + # 后一个合并看到前一个的写入结果，不再互相覆盖 + assert await storage.read_profile("user", "10001") in {"a+b", "b+a"} + + +@pytest.mark.asyncio +async def test_merge_profiles_holds_entity_merge_guard() -> None: + events: list[str] = [] + + class _Guard: + def __init__(self, entity_type: str, entity_id: str) -> None: + self._key = f"{entity_type}:{entity_id}" + + async def __aenter__(self) -> None: + events.append(f"enter:{self._key}") + + async def __aexit__(self, *exc: Any) -> bool: + events.append(f"exit:{self._key}") + return False + + class _Storage: + def merge_guard(self, entity_type: str, entity_id: str) -> _Guard: + return _Guard(entity_type, entity_id) + + worker = HistorianWorker( + job_queue=None, + vector_store=None, + profile_storage=_Storage(), + ai_client=None, + config_getter=lambda: SimpleNamespace(), + ) + + async def _fake_merge_target(**kwargs: Any) -> bool: + events.append("merge") + return True + + worker._merge_profile_target = _fake_merge_target # type: ignore[method-assign] + + job: dict[str, Any] = { + "observations": ["某人在群里说了某事"], + "profile_targets": [ + {"entity_type": "user", "entity_id": "10001"}, + ], + } + await worker._merge_profiles(job, "canonical", "job-1") + + assert events == ["enter:user:10001", "merge", "exit:user:10001"] + + +@pytest.mark.asyncio +async def test_merge_profiles_without_storage_guard_still_runs() -> None: + worker = HistorianWorker( + job_queue=None, + vector_store=None, + profile_storage=SimpleNamespace(), + ai_client=None, + config_getter=lambda: SimpleNamespace(), + ) + + async def _fake_merge_target(**kwargs: Any) -> bool: + return True + + worker._merge_profile_target = _fake_merge_target # type: ignore[method-assign] + job: dict[str, Any] = { + "observations": ["x"], + "profile_targets": [{"entity_type": "user", "entity_id": "10001"}], + } + await worker._merge_profiles(job, "canonical", "job-1") + + +@pytest.mark.asyncio +async def test_poll_loop_respects_max_concurrency() -> None: + pending: list[dict[str, Any]] = [{"observations": []} for _ in range(6)] + concurrent = 0 + peak = 0 + processed: list[str] = [] + + class _Queue: + async def dequeue(self) -> tuple[str, dict[str, Any]] | None: + if not pending: + return None + index = len(pending) + pending.pop() + return f"job-{index}", {"observations": []} + + config = SimpleNamespace( + poll_interval_seconds=0.01, + failed_cleanup_interval=0, + failed_max_age_days=30, + failed_max_files=500, + job_max_retries=0, + ) + worker = HistorianWorker( + job_queue=_Queue(), + vector_store=None, + profile_storage=None, + ai_client=None, + config_getter=lambda: config, + max_concurrency=2, + ) + + async def _fake_process(job_id: str, job: dict[str, Any]) -> None: + nonlocal concurrent, peak + concurrent += 1 + peak = max(peak, concurrent) + await asyncio.sleep(0.05) + concurrent -= 1 + processed.append(job_id) + + worker._process_job = _fake_process # type: ignore[method-assign] + + await worker.start() + await asyncio.sleep(0.8) + await worker.stop() + + assert len(processed) == 6 + assert peak <= 2 From eb98e945cbd4a155bd855532756d468e75910629 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:44:39 +0800 Subject: [PATCH 05/30] =?UTF-8?q?fix(skills):=20=E4=BF=AE=E5=A4=8D=20agent?= =?UTF-8?q?=20handler=20=E6=97=A0=E6=B3=95=E5=8A=A0=E8=BD=BD=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E6=8A=8A=E5=8A=A0=E8=BD=BD=E5=A4=B1=E8=B4=A5=E6=9A=B4?= =?UTF-8?q?=E9=9C=B2=E5=87=BA=E6=9D=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - handler 模块名改用真实包路径（Undefined.skills.<...>.handler）， code_delivery_agent 的 from .docker_utils import 等相对导入终于可解析； - 按真实包导入而非按文件注册，常规 import 与注册表加载共用同一模块对象， 热重载用 importlib.reload 原地重执行，外部引用不再指向另一份状态； - SkillItem 新增 load_error：注册阶段即导入全部 handler，失败项打印错误、 记录 load_error 并从 schema 中排除，主 AI 不会再看到不可用的技能； - 补测试覆盖 agents/toolsets 的 handler 导入、相对导入与失败排除（原先只测 tools，CI 查不出 code_delivery_agent 挂掉）。 --- src/Undefined/skills/README.md | 3 +- src/Undefined/skills/agents/README.md | 2 +- src/Undefined/skills/registry.py | 165 ++++++++++++++++++---- src/Undefined/skills/tools/__init__.py | 5 +- src/Undefined/skills/toolsets/__init__.py | 4 +- tests/test_skill_handler_loading.py | 130 +++++++++++++++++ 6 files changed, 279 insertions(+), 30 deletions(-) create mode 100644 tests/test_skill_handler_loading.py diff --git a/src/Undefined/skills/README.md b/src/Undefined/skills/README.md index e62c1013..fcc9d37d 100644 --- a/src/Undefined/skills/README.md +++ b/src/Undefined/skills/README.md @@ -118,7 +118,8 @@ skills/ ## 运行机制（重要） -- **注册表 handler 延迟导入**: 启动时读取 `config.json` 建立完整本地 schema，仅在首次执行时才导入 `handler.py`，用于降低启动成本。 +- **注册表 handler 导入与校验**: 启动时读取 `config.json` 建立 schema，并立即导入每个 `handler.py`；导入失败的技能会记录 `load_error`、打印错误日志，并从对外 schema 中排除，主 AI 不会看到不可调用的技能。 +- **handler 模块名即真实包路径**: 随包技能的 handler 按 `Undefined.skills.<...>.handler` 导入，因此 `handler.py` 内可以使用同目录相对导入（`from .helper import ...`）；常规 `import` 与注册表加载得到同一个模块对象。 - **模型 schema 按需投影**: 可通过 `skills.tool_search_enabled`（即 `[skills]` 下的 `tool_search_enabled`）让主 AI 首轮只看到配置为始终加载的工具和 `tool_search` schema，其余工具以名称目录提示，检索后从下一模型轮开始可调用。它只降低模型上下文占用，不会卸载注册表或提前导入 handler；子 Agent 不使用该投影。 - **结构化日志 + 统计**: 统一输出 `event=execute`、`status=success/timeout/error` 等结构化字段，并记录执行耗时与成功/失败计数。 - **超时与取消**: 所有技能执行默认 120 秒超时，超时会返回提示并记录统计。 diff --git a/src/Undefined/skills/agents/README.md b/src/Undefined/skills/agents/README.md index c5a8bf9a..3c0e7381 100644 --- a/src/Undefined/skills/agents/README.md +++ b/src/Undefined/skills/agents/README.md @@ -182,7 +182,7 @@ Agent 的执行逻辑，负责： ## 运行特性 -- **延迟加载 (Lazy Load)**：Agent `handler.py` 首次调用时导入，减少启动耗时。 +- **加载即校验**：Agent `handler.py` 在注册阶段导入；导入失败（如相对导入错误、缺少依赖）会记录 `load_error` 并从 Agent schema 中排除，主 AI 不会被告知一个不可用的 Agent。 - **超时与取消**：Agent 调用默认 120s 超时，超时返回提示并记录统计。 - **结构化日志**：统一输出 `event=execute`、`status=success/timeout/error` 等字段。 - **热重载**：检测到 `skills/agents/` 变更后自动重载 Agent 注册表。 diff --git a/src/Undefined/skills/registry.py b/src/Undefined/skills/registry.py index 3c665b5c..1ac90d8e 100644 --- a/src/Undefined/skills/registry.py +++ b/src/Undefined/skills/registry.py @@ -13,6 +13,21 @@ logger = logging.getLogger(__name__) +# handler 模块名的前缀必须是真实可导入的包路径，否则 handler.py 内的 +# 相对导入（如 `from .docker_utils import ...`）会因为顶层包不存在而失败。 +_PACKAGE_PREFIX: str = __package__ or "Undefined.skills" +_REAL_PACKAGE_ROOT: Path = Path(__file__).resolve().parent +_SYNTHETIC_PREFIX = "_undefined_skill_modules" + + +def _is_under_real_package(path: Path) -> bool: + """判断目录是否属于随包发布的 skills 目录。""" + try: + path.resolve().relative_to(_REAL_PACKAGE_ROOT) + except (OSError, ValueError): + return False + return True + class RegistryExecutionTimeoutError(asyncio.TimeoutError): """由注册表超时包装器抛出的超时异常。""" @@ -63,6 +78,8 @@ class SkillItem: module_name: Optional[str] handler: Optional[Callable[[Dict[str, Any], Dict[str, Any]], Awaitable[Any]]] = None loaded: bool = False + # handler 导入失败的原因；不为空表示该项不可执行，也不会进入对外 schema + load_error: Optional[str] = None class BaseRegistry: @@ -86,7 +103,6 @@ def __init__( self.kind = kind self.timeout_seconds = timeout_seconds self._items: Dict[str, SkillItem] = {} - self._items_schema: List[Dict[str, Any]] = [] self._stats: Dict[str, SkillStats] = {} self._items_lock = asyncio.Lock() @@ -125,10 +141,9 @@ def _log_event(self, event: str, name: str = "", **fields: Any) -> None: def _reset_items(self) -> None: self._items = {} - self._items_schema = [] def load_items(self) -> None: - """从 base_dir 自动发现并加载技能定义（仅加载 config 配置文件，不导入 handler 代码）""" + """从 base_dir 自动发现技能定义，并导入 handler 暴露加载失败。""" self._reset_items() if not self.base_dir.exists(): @@ -136,6 +151,7 @@ def load_items(self) -> None: return self._discover_items_in_dir(self.base_dir, prefix="") + self.preload_handlers() active_names = set(self._items.keys()) self._stats = { @@ -144,7 +160,7 @@ def load_items(self) -> None: item_names = list(self._items.keys()) logger.info( - f"[{self.__class__.__name__}] 成功加载了 {len(self._items_schema)} 个项目: {', '.join(item_names)}" + f"[{self.__class__.__name__}] 成功加载了 {len(self.get_schema())} 个项目: {', '.join(item_names)}" ) def _discover_items_in_dir(self, parent_dir: Path, prefix: str) -> None: @@ -175,7 +191,6 @@ def _register_item_from_dir(self, item_dir: Path, prefix: str = "") -> None: item = self._build_skill_item(item_dir, config, handler_path, prefix) self._items[item.name] = item - self._items_schema.append(item.config) self._stats.setdefault(item.name, SkillStats()) if logger.isEnabledFor(logging.DEBUG): @@ -226,13 +241,21 @@ def _build_skill_item( ) def _build_module_name(self, item_dir: Path) -> str: + """合成 handler 的模块名。 + + 随包技能形如 ``Undefined.skills.agents.code_delivery_agent.handler``： + 父包真实存在，因此 handler.py 内的相对导入（``from .docker_utils import``） + 可以正常解析到同目录模块。不随包的目录（测试或外部注入）使用独立前缀， + 避免污染真实包命名空间。 + """ try: relative = item_dir.relative_to(self.skills_root) - parts = [self.skills_root.name] + list(relative.parts) - return ".".join(parts) except ValueError: - parts = list(item_dir.parts[-3:]) - return ".".join(parts) + relative = Path(*item_dir.parts[-3:]) + prefix = ( + _PACKAGE_PREFIX if _is_under_real_package(item_dir) else _SYNTHETIC_PREFIX + ) + return ".".join([*prefix.split("."), *relative.parts, "handler"]) def _load_handler_for_item( self, item: SkillItem, reload_module: bool = False @@ -244,30 +267,120 @@ def _load_handler_for_item( if not item.handler_path or not item.module_name: return - if reload_module and item.module_name in sys.modules: - del sys.modules[item.module_name] + module = self._resolve_handler_module(item) + if not hasattr(module, "execute"): + item.load_error = "RuntimeError: 处理器缺少 'execute' 函数" + raise RuntimeError(f"{item.handler_path} 的处理器缺少 'execute' 函数") - spec = importlib.util.spec_from_file_location( - item.module_name, item.handler_path - ) - if spec is None or spec.loader is None: - raise RuntimeError(f"加载处理器 spec 失败: {item.handler_path}") + item.handler = module.execute + item.loaded = True + item.load_error = None + + def _resolve_handler_module(self, item: SkillItem) -> Any: + """取得 handler 的模块对象；失败时给 item 记上 load_error。""" + assert item.module_name is not None and item.handler_path is not None + try: + if _is_under_real_package(item.handler_path.parent): + return self._import_packaged_handler( + item.module_name, item.handler_path + ) + return self._exec_handler_from_path(item.module_name, item.handler_path) + except Exception as exc: + item.load_error = f"{type(exc).__name__}: {exc}" + raise + + def _import_packaged_handler(self, module_name: str, handler_path: Path) -> Any: + """按真实包路径导入 handler。 + 走标准 import 机制（而不是按文件路径注册），这样同一份源码只有一份模块 + 对象：handler.py 内的相对导入能沿真实包链解析，其他位置的常规 import 也 + 不会拿到重复状态。 + """ + self._purge_submodules(module_name) + current = sys.modules.get(module_name) + if current is None: + return importlib.import_module(module_name) + # 已有模块对象时按文件重新执行：热重载与首次加载都以磁盘内容为准， + # 同时保留模块对象身份，外部持有的引用不会指向另一份模块状态。 + try: + return importlib.reload(current) + except Exception: + logger.warning( + "[%s] reload 处理器失败，改为按文件重新导入: %s", + self.__class__.__name__, + module_name, + exc_info=True, + ) + return self._exec_handler_from_path(module_name, handler_path) + + @staticmethod + def _exec_handler_from_path(module_name: str, handler_path: Path) -> Any: + """按文件路径导入不随包发布的 handler（测试或外部目录）。""" + spec = importlib.util.spec_from_file_location(module_name, handler_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"加载处理器 spec 失败: {handler_path}") module = importlib.util.module_from_spec(spec) - sys.modules[item.module_name] = module + sys.modules[module_name] = module try: spec.loader.exec_module(module) except Exception: - current = sys.modules.get(item.module_name) - if current is module: - del sys.modules[item.module_name] + if sys.modules.get(module_name) is module: + del sys.modules[module_name] raise + return module - if not hasattr(module, "execute"): - raise RuntimeError(f"{item.handler_path} 的处理器缺少 'execute' 函数") + @staticmethod + def _purge_submodules(module_name: str) -> None: + """清理 handler 同目录相对导入产生的子模块缓存。 - item.handler = module.execute - item.loaded = True + `handler.py` 内的 `from .xxx import ...` 会把同目录模块注册成 + `<父包>.<模块>`；热重载时要一并失效，否则仍会使用旧代码。 + """ + package_name = module_name.rpartition(".")[0] + parent_prefix = f"{package_name}." + stale = [ + name + for name in sys.modules + if not name.endswith(".handler") and name.startswith(parent_prefix) + ] + for name in stale: + sys.modules.pop(name, None) + + def preload_handlers(self) -> List[tuple[str, str]]: + """在注册阶段导入全部 handler，尽早暴露加载失败。 + + 失败的项会被记录 `load_error` 并从对外 schema 中排除，避免主 AI + 被告知一个实际不可用的技能。返回 `(技能名, 错误)` 列表。 + """ + failures: List[tuple[str, str]] = [] + for name, item in self._items.items(): + if item.handler is not None: + continue + try: + self._load_handler_for_item(item) + except Exception as exc: + if not item.load_error: + item.load_error = f"{type(exc).__name__}: {exc}" + if item.load_error: + failures.append((name, item.load_error)) + + if failures: + logger.error( + "[%s] %s 个技能加载失败，已从工具列表中排除:", + self.__class__.__name__, + len(failures), + ) + for name, error in failures: + logger.error(" - %s: %s", name, error) + return failures + + def get_load_failures(self) -> Dict[str, str]: + """返回 handler 加载失败的技能名与错误信息。""" + return { + name: item.load_error + for name, item in self._items.items() + if item.load_error + } def register_external_item( self, @@ -293,11 +406,11 @@ def register_external_item( loaded=True, ) self._items[name] = item - self._items_schema.append(schema) self._stats.setdefault(name, SkillStats()) def get_schema(self) -> List[Dict[str, Any]]: - return self._items_schema + """返回可用技能的 schema；handler 加载失败的项不对外暴露。""" + return [item.config for item in self._items.values() if not item.load_error] def get_stats(self) -> Dict[str, SkillStats]: return self._stats diff --git a/src/Undefined/skills/tools/__init__.py b/src/Undefined/skills/tools/__init__.py index 4685d0fe..c5c44f5b 100644 --- a/src/Undefined/skills/tools/__init__.py +++ b/src/Undefined/skills/tools/__init__.py @@ -43,12 +43,15 @@ def load_tools(self) -> None: # 3) MCP 工具集（创建注册表，但不初始化） self._load_mcp_toolsets() + # 4) 立即导入全部 handler：加载失败的项不会进入对外 schema + self.preload_handlers() + active_names = set(self._items.keys()) self._stats = { name: self._stats.get(name, SkillStats()) for name in active_names } - # 4) 输出工具列表（不包含 MCP 工具，因为 MCP 还未初始化） + # 5) 输出工具列表（不包含 MCP 工具，因为 MCP 还未初始化） self._log_tools_summary(include_mcp=False) def _categorize_tools( diff --git a/src/Undefined/skills/toolsets/__init__.py b/src/Undefined/skills/toolsets/__init__.py index dc8d3ab7..862703ec 100644 --- a/src/Undefined/skills/toolsets/__init__.py +++ b/src/Undefined/skills/toolsets/__init__.py @@ -44,6 +44,8 @@ def load_toolsets(self) -> None: category = category_dir.name self._discover_items_in_dir(category_dir, prefix=f"{category}.") + self.preload_handlers() + active_names = set(self._items.keys()) self._stats = { name: self._stats.get(name, SkillStats()) for name in active_names @@ -51,7 +53,7 @@ def load_toolsets(self) -> None: tool_names = list(self._items.keys()) logger.info( - f"成功加载了 {len(self._items_schema)} 个工具集工具: {', '.join(tool_names)}" + f"成功加载了 {len(self.get_schema())} 个工具集工具: {', '.join(tool_names)}" ) def get_tools_schema(self) -> List[Dict[str, Any]]: diff --git a/tests/test_skill_handler_loading.py b/tests/test_skill_handler_loading.py new file mode 100644 index 00000000..5ad70d14 --- /dev/null +++ b/tests/test_skill_handler_loading.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from Undefined.skills.agents import AgentRegistry +from Undefined.skills.tools import ToolRegistry +from Undefined.skills.toolsets import ToolSetRegistry + +PACKAGE_ROOT = Path(__file__).parents[1] / "src" / "Undefined" + + +def _write_skill( + item_dir: Path, + *, + name: str, + handler_source: str, +) -> None: + item_dir.mkdir(parents=True, exist_ok=True) + (item_dir / "config.json").write_text( + json.dumps( + { + "type": "function", + "function": { + "name": name, + "description": "test skill", + "parameters": {"type": "object", "properties": {}}, + }, + } + ), + encoding="utf-8", + ) + (item_dir / "handler.py").write_text(handler_source, encoding="utf-8") + + +def test_all_registered_agents_import_handlers() -> None: + """随包发布的 agent handler 都必须能导入（含使用相对导入的 agent）。""" + registry = AgentRegistry(PACKAGE_ROOT / "skills" / "agents") + + assert registry.get_load_failures() == {} + for name, item in registry._items.items(): + assert item.handler is not None, name + assert item.loaded is True, name + + +def test_agent_relative_import_handler_loads() -> None: + """handler.py 内的相对导入必须能解析到同目录模块。""" + registry = AgentRegistry(PACKAGE_ROOT / "skills" / "agents") + + item = registry._items["code_delivery_agent"] + assert item.module_name == "Undefined.skills.agents.code_delivery_agent.handler" + assert item.handler is not None + assert item.handler.__module__ == item.module_name + + +def test_handler_module_is_canonical_across_import_paths() -> None: + """按文件路径加载与常规 import 必须得到同一个模块对象。""" + registry = AgentRegistry(PACKAGE_ROOT / "skills" / "agents") + item = registry._items["summary_agent"] + + import Undefined.skills.agents.summary_agent.handler as module + + assert item.module_name is not None + assert sys.modules[item.module_name] is module + + +def test_toolset_handlers_load_with_canonical_names() -> None: + registry = ToolSetRegistry(PACKAGE_ROOT / "skills" / "toolsets") + + assert registry.get_load_failures() == {} + item = registry._items["render.render_latex"] + assert item.module_name == "Undefined.skills.toolsets.render.render_latex.handler" + assert item.handler is not None + + +def test_broken_handler_is_reported_and_hidden_from_schema(tmp_path: Path) -> None: + tools_dir = tmp_path / "skills" / "tools" + _write_skill( + tools_dir / "broken_tool", + name="broken_tool", + handler_source=( + "import definitely_not_installed_module\n\n" + "async def execute(args, context):\n" + " return 'ok'\n" + ), + ) + _write_skill( + tools_dir / "healthy_tool", + name="healthy_tool", + handler_source=("async def execute(args, context):\n return 'ok'\n"), + ) + + registry = ToolRegistry(tools_dir) + + failures = registry.get_load_failures() + assert "broken_tool" in failures + assert "definitely_not_installed_module" in failures["broken_tool"] + assert registry._items["healthy_tool"].load_error is None + + advertised = {schema["function"]["name"] for schema in registry.get_tools_schema()} + assert advertised == {"healthy_tool"} + + +def test_handler_without_execute_is_reported(tmp_path: Path) -> None: + tools_dir = tmp_path / "skills" / "tools" + _write_skill( + tools_dir / "no_execute", + name="no_execute", + handler_source="VALUE = 1\n", + ) + + registry = ToolRegistry(tools_dir) + + assert "no_execute" in registry.get_load_failures() + assert registry.get_tools_schema() == [] + + +def test_preload_handlers_returns_failures(tmp_path: Path) -> None: + tools_dir = tmp_path / "skills" / "tools" + _write_skill( + tools_dir / "boom", + name="boom", + handler_source="raise RuntimeError('boom at import')\n", + ) + + registry = ToolRegistry(tools_dir) + failures = registry.preload_handlers() + + assert any(name == "boom" and "boom at import" in error for name, error in failures) From 93cad888d7b4fe5121eb0b939a35c5fcf0634cb0 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:47:09 +0800 Subject: [PATCH 06/30] =?UTF-8?q?fix(config):=20=E7=83=AD=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E5=A4=B1=E8=B4=A5=E4=B8=8D=E5=86=8D=E9=9D=99=E9=BB=98?= =?UTF-8?q?=EF=BC=8C=E6=8C=89=E6=AD=A5=E9=AA=A4=E9=9A=94=E7=A6=BB=E5=B9=B6?= =?UTF-8?q?=E5=9B=9E=E6=94=B6=E5=90=8E=E5=8F=B0=E4=BB=BB=E5=8A=A1=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - apply_config_updates 改为步骤表逐个执行：单步异常不再中断其余步骤， 失败项与“未完全生效”汇总以 error 级日志输出； - 异步步骤（技能热重载 / 自动化并发 / 监听器重启）改为带强引用与 done 回调的任务，异常直接落 error 日志，不再无人知晓； - _apply_skills_hot_reload 按注册表隔离失败，避免一个注册表出错时其余 注册表既不停止也不启动； - Config.update_from 补充可见性说明并在 finally 中刷新派生集合； - ConfigManager._notify 回调失败记录名单后继续，其余回调不受影响； - 文档补充 §5.5 热更新失败的可见性与嵌入/重排的需重启说明。 --- docs/configuration.md | 9 ++ src/Undefined/config/config_class.py | 56 ++++---- src/Undefined/config/hot_reload.py | 186 +++++++++++++++++++++------ src/Undefined/config/manager.py | 15 ++- tests/test_config_hot_reload.py | 102 +++++++++++++++ 5 files changed, 303 insertions(+), 65 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bf1f93d1..e31fe394 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1378,6 +1378,7 @@ api_key = "replace-with-your-key" - `memes.vector_store_path` - `memes.queue_path` - `naga.*`（`enabled/api_url/api_key/use_proxy/moderation_enabled/mode/allowed_group_ids/blocked_group_ids/allowed_private_ids/blocked_private_ids`） +- `models.embedding` / `models.embedding.features.*` / `models.rerank`（嵌入与重排运行时在启动时构造，热更新只提示需重启） ### 5.3 明确“会执行热应用”的字段 - `onebot.file_send_mode` / `onebot.file_send_host`（新投递读取快照；进行中投递及旧 URL 生命周期不变） @@ -1408,6 +1409,14 @@ api_key = "replace-with-your-key" ### 5.4 其他字段 - `Config` 对象本身会更新。 + +### 5.5 热更新失败的可见性 + +热更新在共享的 `Config` 实例上逐字段就地生效，整个过程没有 `await`，同一事件循环内的读方不会在一次读取里看到“改了一半”的对象；跨 `await` 的多次读取仍可能分别落在变更前后，需要严格一致的快照时请在单次读取中取全所需字段。 + +应用阶段按步骤隔离：单步抛错不会中断后续步骤，失败步骤会以 `error` 级日志逐条打印（`热更新步骤失败`），并在末尾汇总一条“热更新未完全生效（运行时状态与 config.toml 不一致）”。异步步骤（技能热重载、自动化并发、配置监听器重启）使用被强引用跟踪的后台任务，任务异常同样以 `error` 级日志输出（`热更新后台任务失败`）。配置订阅者回调抛错时记录失败回调名单，其余回调继续执行。 + +因此若看到上述日志，说明 `config.toml` 已改但对应运行时未生效，需要修复报错原因后重新保存配置；必要时重启进程。 - 具体功能是否“立刻体现”，取决于模块是“每次读取配置”还是“启动时缓存”。 - 对于行为不确定项，建议改完观察日志；必要时重启进程确认。 diff --git a/src/Undefined/config/config_class.py b/src/Undefined/config/config_class.py index f7003e85..4c05a149 100644 --- a/src/Undefined/config/config_class.py +++ b/src/Undefined/config/config_class.py @@ -588,29 +588,41 @@ def resolve_embedding_model(self, feature: str) -> EmbeddingModelConfig: # 热更新运行时参数 def update_from(self, new_config: "Config") -> dict[str, tuple[Any, Any]]: - # 逐字段 diff；嵌套模型配置用 _update_dataclass 展开为 chat_model.api_url 等键 + """就地应用热更新，返回 `{字段路径: (旧值, 新值)}`。 + + 逐字段 diff；嵌套模型配置用 `_update_dataclass` 原地展开为 + `chat_model.api_url` 等键，保留对象身份，因此已经持有该对象的组件 + （队列间隔、HistorianWorker 等）能同步看到新值。 + + 可见性：整个应用过程没有 `await`，同一事件循环内的读方不会在一次读取中 + 看到“改了一半”的对象；跨 `await` 的多次读取仍可能分别落在变更前与变更后， + 需要严格一致的快照时请在单次读取中取全所需字段。派生集合在 finally 中刷新， + 即使某个字段解析异常也不会留下过期索引。 + """ changes: dict[str, tuple[Any, Any]] = {} - for field in fields(self): - name = field.name - old_value = getattr(self, name) - new_value = getattr(new_config, name) - if isinstance( - old_value, - ( - ChatModelConfig, - VisionModelConfig, - SecurityModelConfig, - AgentModelConfig, - GrokModelConfig, - ), - ): - changes.update(_update_dataclass(old_value, new_value, prefix=name)) - continue - if old_value != new_value: - setattr(self, name, new_value) - changes[name] = (old_value, new_value) - if changes: - self._refresh_runtime_sets() + try: + for field in fields(self): + name = field.name + old_value = getattr(self, name) + new_value = getattr(new_config, name) + if isinstance( + old_value, + ( + ChatModelConfig, + VisionModelConfig, + SecurityModelConfig, + AgentModelConfig, + GrokModelConfig, + ), + ): + changes.update(_update_dataclass(old_value, new_value, prefix=name)) + continue + if old_value != new_value: + setattr(self, name, new_value) + changes[name] = (old_value, new_value) + finally: + if changes: + self._refresh_runtime_sets() return changes def reload(self, strict: bool = False) -> dict[str, tuple[Any, Any]]: diff --git a/src/Undefined/config/hot_reload.py b/src/Undefined/config/hot_reload.py index 04951fc7..cd73a47b 100644 --- a/src/Undefined/config/hot_reload.py +++ b/src/Undefined/config/hot_reload.py @@ -2,6 +2,7 @@ import asyncio import logging +from collections.abc import Callable, Coroutine from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any @@ -142,83 +143,161 @@ class HotReloadContext: message_handler: MessageHandler | None = None +# 热更新创建的后台任务：持有强引用避免被 GC 回收，并统一回收异常 +_BACKGROUND_TASKS: set[asyncio.Task[None]] = set() + + +def _spawn_hot_reload_task( + coro: Coroutine[Any, Any, None], + description: str, +) -> None: + """创建热更新后台任务并回收异常，避免静默失败。""" + task = asyncio.create_task(coro, name=f"config-hot-reload:{description}") + _BACKGROUND_TASKS.add(task) + + def _on_done(finished: asyncio.Task[None]) -> None: + _BACKGROUND_TASKS.discard(finished) + if finished.cancelled(): + return + exc = finished.exception() + if exc is not None: + logger.error( + "[配置] 热更新后台任务失败: %s（该部分配置未生效）", + description, + exc_info=exc, + ) + + task.add_done_callback(_on_done) + + def apply_config_updates( updated: Config, changes: dict[str, tuple[object, object]], context: HotReloadContext, ) -> None: + """把热更新应用到运行时。 + + 每一步独立执行并捕获异常：单步失败不会中断其余步骤；失败项会以 error 级日志 + 列出，避免出现“配置已改、行为未改”却无人知晓的隐蔽不一致。 + """ if not changes: return changed_keys = set(changes.keys()) logger.debug("[配置] 热更新变更项: %s", ", ".join(sorted(changed_keys))) _log_restart_required(changed_keys) - context.security_service.apply_config(updated) - if "ai_request_max_retries" in changed_keys: + + handler = context.message_handler + + def _apply_security() -> None: + context.security_service.apply_config(updated) + + def _apply_retries() -> None: context.queue_manager.update_max_retries(updated.ai_request_max_retries) - if _needs_queue_interval_update(changed_keys): + def _apply_queue_intervals() -> None: context.queue_manager.update_model_intervals( build_model_queue_intervals(updated) ) - if _needs_intro_update(changed_keys): - intro_config = AgentIntroGenConfig( - enabled=updated.agent_intro_autogen_enabled, - queue_interval_seconds=updated.agent_intro_autogen_queue_interval, - max_tokens=updated.agent_intro_autogen_max_tokens, - cache_path=Path(updated.agent_intro_hash_path), + def _apply_intro() -> None: + context.ai_client.apply_intro_config( + AgentIntroGenConfig( + enabled=updated.agent_intro_autogen_enabled, + queue_interval_seconds=updated.agent_intro_autogen_queue_interval, + max_tokens=updated.agent_intro_autogen_max_tokens, + cache_path=Path(updated.agent_intro_hash_path), + ) ) - context.ai_client.apply_intro_config(intro_config) - if _needs_search_update(changed_keys): + def _apply_search() -> None: context.ai_client.apply_search_config(updated.searxng_url) - if _needs_attachment_update(changed_keys): + def _apply_attachments() -> None: context.ai_client.apply_attachment_config(updated) - if _needs_message_batcher_update(changed_keys): - handler = context.message_handler - if ( - handler is not None - and getattr(handler, "message_batcher", None) is not None - ): + def _apply_message_batcher() -> None: + if handler is None: + return + if getattr(handler, "message_batcher", None) is not None: handler.message_batcher.update_config(updated.message_batcher) - if _needs_automations_update(changed_keys): - asyncio.create_task( - _apply_message_handler_automations_hot_reload( - updated, - context.message_handler, - ) + def _apply_automations() -> None: + _spawn_hot_reload_task( + _apply_message_handler_automations_hot_reload(updated, handler), + "automations", ) - if _needs_core_ai_model_update(changed_keys): + def _apply_model_configs() -> None: context.ai_client.apply_model_configs( chat_config=updated.chat_model, vision_config=updated.vision_model, agent_config=updated.agent_model, runtime_config=updated, ) - elif _needs_runtime_ai_model_update(changed_keys): + + def _apply_runtime_config() -> None: context.ai_client.apply_runtime_config(updated) - if _needs_skills_hot_reload_update(changed_keys): - asyncio.create_task(_apply_skills_hot_reload(updated, context.ai_client)) - asyncio.create_task( - _apply_message_handler_skills_hot_reload( - updated, - context.message_handler, - ) + def _apply_skills_reload() -> None: + _spawn_hot_reload_task( + _apply_skills_hot_reload(updated, context.ai_client), + "skills", + ) + _spawn_hot_reload_task( + _apply_message_handler_skills_hot_reload(updated, handler), + "message-handler-skills", ) - if _needs_config_hot_reload_update(changed_keys): - asyncio.create_task( + def _apply_config_watcher() -> None: + _spawn_hot_reload_task( _restart_config_hot_reload( context.config_manager, updated.skills_hot_reload_interval, updated.skills_hot_reload_debounce, - ) + ), + "config-watcher", + ) + + steps: list[tuple[str, Callable[[], None]]] = [ + ("security", _apply_security), + ] + if "ai_request_max_retries" in changed_keys: + steps.append(("ai_request_max_retries", _apply_retries)) + if _needs_queue_interval_update(changed_keys): + steps.append(("queue_intervals", _apply_queue_intervals)) + if _needs_intro_update(changed_keys): + steps.append(("agent_intro", _apply_intro)) + if _needs_search_update(changed_keys): + steps.append(("search", _apply_search)) + if _needs_attachment_update(changed_keys): + steps.append(("attachments", _apply_attachments)) + if _needs_message_batcher_update(changed_keys): + steps.append(("message_batcher", _apply_message_batcher)) + if _needs_automations_update(changed_keys): + steps.append(("automations", _apply_automations)) + if _needs_core_ai_model_update(changed_keys): + steps.append(("core_ai_models", _apply_model_configs)) + elif _needs_runtime_ai_model_update(changed_keys): + steps.append(("runtime_ai_config", _apply_runtime_config)) + if _needs_skills_hot_reload_update(changed_keys): + steps.append(("skills_hot_reload", _apply_skills_reload)) + if _needs_config_hot_reload_update(changed_keys): + steps.append(("config_hot_reload", _apply_config_watcher)) + + failed: list[str] = [] + for name, step in steps: + try: + step() + except Exception: + failed.append(name) + logger.error("[配置] 热更新步骤失败: %s", name, exc_info=True) + + if failed: + logger.error( + "[配置] 热更新未完全生效（运行时状态与 config.toml 不一致）: %s；" + "请修复配置或代码后重新保存配置，必要时重启进程", + ", ".join(failed), ) @@ -287,18 +366,41 @@ async def _apply_skills_hot_reload(updated: Config, ai_client: AIClient) -> None if anthropic_skill_registry is not None: registries.append(anthropic_skill_registry) + def _registry_name(registry: Any) -> str: + return type(registry).__name__ + + failed: list[str] = [] if not updated.skills_hot_reload: for registry in registries: - await registry.stop_hot_reload() + try: + await registry.stop_hot_reload() + except Exception: + failed.append(_registry_name(registry)) + logger.error( + "[配置] 停止技能热重载失败: %s", + _registry_name(registry), + exc_info=True, + ) logger.info("[配置] 技能热重载已禁用") + if failed: + logger.error("[配置] 部分注册表热重载未停止: %s", ", ".join(failed)) return for registry in registries: - await registry.stop_hot_reload() - registry.start_hot_reload( - interval=updated.skills_hot_reload_interval, - debounce=updated.skills_hot_reload_debounce, - ) + try: + await registry.stop_hot_reload() + registry.start_hot_reload( + interval=updated.skills_hot_reload_interval, + debounce=updated.skills_hot_reload_debounce, + ) + except Exception: + failed.append(_registry_name(registry)) + logger.error( + "[配置] 重启技能热重载失败: %s", _registry_name(registry), exc_info=True + ) + if failed: + logger.error("[配置] 以下注册表热重载未生效: %s", ", ".join(failed)) + return logger.info( "[配置] 技能热重载已更新: interval=%.2fs debounce=%.2fs", updated.skills_hot_reload_interval, diff --git a/src/Undefined/config/manager.py b/src/Undefined/config/manager.py index f497f253..825d4b71 100644 --- a/src/Undefined/config/manager.py +++ b/src/Undefined/config/manager.py @@ -113,13 +113,26 @@ def _compute_snapshot(self) -> dict[str, tuple[int, int]]: return snapshot def _notify(self, changes: dict[str, tuple[Any, Any]]) -> None: + """把热更新变更分发给订阅者。 + + 单个回调失败不再静默丢弃：记录 error 级日志与失败回调名单，其余回调继续 + 执行。调用方据此可以在日志中看到“配置已改、行为未改”的不一致。 + """ if not self._callbacks: return config = self._config if config is None: return + failed: list[str] = [] for callback in list(self._callbacks): try: callback(config, changes) except Exception: - logger.debug("配置回调执行失败", exc_info=True) + name = getattr(callback, "__qualname__", repr(callback)) + failed.append(name) + logger.error("[配置] 热更新回调执行失败: %s", name, exc_info=True) + if failed: + logger.error( + "[配置] 以下回调未完成，相关运行时配置可能未生效，请修复后重新保存配置或重启: %s", + ", ".join(failed), + ) diff --git a/tests/test_config_hot_reload.py b/tests/test_config_hot_reload.py index 64ee94a3..2aae7c8f 100644 --- a/tests/test_config_hot_reload.py +++ b/tests/test_config_hot_reload.py @@ -644,3 +644,105 @@ async def test_apply_config_updates_refreshes_automation_concurrency() -> None: await asyncio.sleep(0) assert message_handler.automation_updates == [7] + + +def test_apply_config_updates_isolates_failed_step( + caplog: pytest.LogCaptureFixture, +) -> None: + """单个步骤失败不应中断其余步骤，且必须以 error 日志暴露。""" + updated = cast( + Any, + SimpleNamespace( + ai_request_max_retries=7, + chat_model=SimpleNamespace( + model_name="chat", + queue_interval_seconds=1.0, + pool=SimpleNamespace(enabled=False), + ), + agent_model=SimpleNamespace( + model_name="agent", + queue_interval_seconds=1.0, + pool=SimpleNamespace(enabled=False), + ), + vision_model=SimpleNamespace( + model_name="vision", queue_interval_seconds=1.0 + ), + security_model=SimpleNamespace( + model_name="security", queue_interval_seconds=1.0 + ), + naga_model=SimpleNamespace(model_name="naga", queue_interval_seconds=1.0), + grok_model=SimpleNamespace(model_name="grok", queue_interval_seconds=1.0), + historian_model=SimpleNamespace( + model_name="historian", queue_interval_seconds=1.0 + ), + ), + ) + + class _BrokenSecurity: + def apply_config(self, config: Any) -> None: + raise RuntimeError("security boom") + + queue_manager = _FakeQueueManager() + context = HotReloadContext( + ai_client=cast(Any, SimpleNamespace()), + queue_manager=cast(Any, queue_manager), + config_manager=cast(Any, SimpleNamespace()), + security_service=cast(Any, _BrokenSecurity()), + ) + + with caplog.at_level("ERROR"): + apply_config_updates( + updated, + {"naga_model.model_name": ("old", "new")}, + context, + ) + + # 后续步骤仍然执行 + assert len(queue_manager.intervals) == 1 + assert "security" in caplog.text + assert "热更新步骤失败" in caplog.text + assert "热更新未完全生效" in caplog.text + + +@pytest.mark.asyncio +async def test_spawned_hot_reload_task_logs_failure( + caplog: pytest.LogCaptureFixture, +) -> None: + from Undefined.config.hot_reload import _spawn_hot_reload_task + + async def _boom() -> None: + raise RuntimeError("background boom") + + with caplog.at_level("ERROR"): + _spawn_hot_reload_task(_boom(), "unit-test-task") + for _ in range(5): + await asyncio.sleep(0) + + assert "热更新后台任务失败" in caplog.text + assert "unit-test-task" in caplog.text + + +def test_config_manager_notify_survives_failing_callback( + caplog: pytest.LogCaptureFixture, +) -> None: + from Undefined.config.manager import ConfigManager + + manager = ConfigManager() + seen: list[dict[str, Any]] = [] + + def _boom(config: Any, changes: dict[str, Any]) -> None: + raise RuntimeError("callback boom") + + def _record(config: Any, changes: dict[str, Any]) -> None: + seen.append(changes) + + manager._config = cast(Any, SimpleNamespace()) + manager.subscribe(_boom) + manager.subscribe(_record) + + with caplog.at_level("ERROR"): + manager._notify({"core.bot_qq": (1, 2)}) + + assert seen == [{"core.bot_qq": (1, 2)}] + assert "热更新回调执行失败" in caplog.text + assert "回调未完成" in caplog.text From fca4e2bc35f6b4cc7d5310dbf4f3c0d23eee5cae Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:48:28 +0800 Subject: [PATCH 07/30] =?UTF-8?q?fix(main):=20=E5=A2=9E=E5=8A=A0=20SIGTERM?= =?UTF-8?q?=20=E4=BC=98=E9=9B=85=E5=81=9C=E6=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 容器 / systemd / supervisor 默认发送 SIGTERM，此前只捕获 KeyboardInterrupt， 会被直接终止并跳过落盘清理； - 新增 install_shutdown_signal_handlers 把 SIGTERM/SIGINT 收敛到同一事件， 非 POSIX 事件循环回退 signal.signal + call_soon_threadsafe； - _run_until_shutdown 在停止信号到达时取消连接任务并等待其收敛， 连接任务自身结束/报错仍按原路径处理； - 部署文档补充优雅停机说明。 --- docs/deployment.md | 4 ++ src/Undefined/main.py | 66 ++++++++++++++++++++++++++++++- tests/test_main_shutdown.py | 77 +++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 tests/test_main_shutdown.py diff --git a/docs/deployment.md b/docs/deployment.md index c01c2b44..20385bc8 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -116,6 +116,10 @@ uv run Undefined-webui > > WebUI 功能详见 [WebUI 使用指南](webui-guide.md)。 +#### 优雅停机 + +`SIGINT`（Ctrl+C）与 `SIGTERM` 都会触发同一套优雅停机流程：停止 OneBot 连接、关闭 Runtime API 与微信服务、停止史官 / memes worker、断开连接、关闭 AI 客户端与检索运行时、停止配置热重载并释放渲染资源。容器、systemd、supervisor 等默认以 `SIGTERM` 停止进程，无需额外配置；请避免使用 `SIGKILL`（`docker kill -s KILL`），否则会跳过全部落盘清理。 + #### 自动启动选项 若希望 WebUI 启动后自动拉起机器人进程，可在 `config.toml` 中设置： diff --git a/src/Undefined/main.py b/src/Undefined/main.py index 4995d115..55bbb1e2 100644 --- a/src/Undefined/main.py +++ b/src/Undefined/main.py @@ -2,6 +2,7 @@ import asyncio import logging +import signal import time import sys from typing import Any @@ -506,8 +507,9 @@ def _apply_config_updates( "/naga 命令和 /api/v1/naga/* 端点都不会可用" ) + shutdown_event = install_shutdown_signal_handlers(logger) try: - await onebot.run_with_reconnect() + await _run_until_shutdown(onebot, shutdown_event, logger) except KeyboardInterrupt: logger.info("[退出] 收到退出信号 (Ctrl+C)") except Exception as exc: @@ -542,6 +544,68 @@ def _apply_config_updates( logger.info("[退出] 机器人已停止运行") +def install_shutdown_signal_handlers(logger: logging.Logger) -> asyncio.Event: + """注册 SIGTERM / SIGINT 的优雅停机事件。 + + 容器与服务管理器默认发送 SIGTERM（而非 Ctrl+C 的 SIGINT），此前未处理会直接 + 终止进程并跳过后面的落盘清理。这里把两个信号都收敛到同一个事件，由主循环在 + 被唤醒后走正常关闭流程。 + """ + stop_event = asyncio.Event() + loop = asyncio.get_running_loop() + for signame in ("SIGTERM", "SIGINT"): + signum = getattr(signal, signame, None) + if signum is None: + continue + try: + loop.add_signal_handler(signum, stop_event.set) + except (NotImplementedError, RuntimeError, ValueError): + # Windows 的事件循环不支持 add_signal_handler，退回到 signal.signal + try: + signal.signal( + signum, + lambda *_args, _loop=loop, _event=stop_event: ( + _loop.call_soon_threadsafe(_event.set) + ), + ) + except (ValueError, OSError): + logger.warning("[退出] 无法注册 %s 处理器", signame) + return stop_event + + +async def _run_until_shutdown( + onebot: OneBotClient, + shutdown_event: asyncio.Event, + logger: logging.Logger, +) -> None: + """运行 OneBot 连接，收到停止信号或连接任务结束时返回。""" + run_task = asyncio.create_task(onebot.run_with_reconnect(), name="onebot-run") + stop_task = asyncio.create_task(shutdown_event.wait(), name="shutdown-wait") + try: + done, _pending = await asyncio.wait( + {run_task, stop_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + if run_task in done: + # 连接任务自行结束：把异常抛给上层处理 + run_task.result() + return + logger.info("[退出] 收到停止信号 (SIGTERM/SIGINT)，正在优雅停机...") + run_task.cancel() + try: + await run_task + except asyncio.CancelledError: + pass + except Exception as exc: + logger.warning("[退出] 停止 OneBot 连接时发生异常: %s", exc) + finally: + stop_task.cancel() + try: + await stop_task + except asyncio.CancelledError: + pass + + def run() -> None: """运行入口""" asyncio.run(main()) diff --git a/tests/test_main_shutdown.py b/tests/test_main_shutdown.py new file mode 100644 index 00000000..9577fdb2 --- /dev/null +++ b/tests/test_main_shutdown.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import asyncio +import logging +import os +import signal +import pytest + +from Undefined.main import _run_until_shutdown, install_shutdown_signal_handlers + +logger = logging.getLogger("test-main-shutdown") + + +class _SlowOneBot: + def __init__(self) -> None: + self.cancelled = False + + async def run_with_reconnect(self) -> None: + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + self.cancelled = True + raise + + +class _FailingOneBot: + async def run_with_reconnect(self) -> None: + raise RuntimeError("connection boom") + + +@pytest.mark.asyncio +async def test_run_until_shutdown_cancels_onebot_on_signal() -> None: + onebot = _SlowOneBot() + shutdown_event = asyncio.Event() + + task = asyncio.create_task( + _run_until_shutdown(onebot, shutdown_event, logger) # type: ignore[arg-type] + ) + await asyncio.sleep(0.05) + shutdown_event.set() + await asyncio.wait_for(task, timeout=2) + + assert onebot.cancelled is True + + +@pytest.mark.asyncio +async def test_run_until_shutdown_propagates_connection_error() -> None: + with pytest.raises(RuntimeError, match="connection boom"): + await _run_until_shutdown( + _FailingOneBot(), # type: ignore[arg-type] + asyncio.Event(), + logger, + ) + + +@pytest.mark.asyncio +async def test_install_shutdown_signal_handlers_reacts_to_sigterm() -> None: + if not hasattr(signal, "SIGTERM"): # pragma: no cover - 非 POSIX 平台 + pytest.skip("SIGTERM unavailable") + + original = signal.getsignal(signal.SIGTERM) + event = install_shutdown_signal_handlers(logger) + if signal.getsignal(signal.SIGTERM) is signal.SIG_DFL: + pytest.skip("当前事件循环不支持信号处理器") + try: + os.kill(os.getpid(), signal.SIGTERM) + await asyncio.wait_for(event.wait(), timeout=2) + assert event.is_set() + finally: + signal.signal(signal.SIGTERM, original) + + +@pytest.mark.asyncio +async def test_install_shutdown_signal_handlers_returns_fresh_event() -> None: + event = install_shutdown_signal_handlers(logger) + assert isinstance(event, asyncio.Event) + assert event.is_set() is False From d37ca897751e7047c400c98202b0311a0c2b82a2 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:50:30 +0800 Subject: [PATCH 08/30] =?UTF-8?q?fix(queue):=20=E7=BB=9F=E4=B8=80=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E4=B8=8A=E9=99=90=E6=9D=A5=E6=BA=90=EF=BC=8C=E9=81=BF?= =?UTF-8?q?=E5=85=8D=E7=AD=89=E5=BE=85=E6=96=B9=E6=8C=82=E5=88=B0=E8=B6=85?= =?UTF-8?q?=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit coordinator 判断是否已耗尽重试时用的是 config.ai_request_max_retries， 而真正的重试上限来自 QueueManager._max_retries；热更新后两者可分叉， 导致等待方在仍会重试时被唤醒为失败、或在重试已耗尽时干等到 480s 超时。 改为统一走 resolve_effective_retry_count（优先 QueueManager，缺失时回退 config），与等待超时预算的计算口径一致，并补覆盖分叉场景的测试。 --- .../services/coordinator/background.py | 11 ++- tests/test_coordinator_retry_alignment.py | 95 +++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 tests/test_coordinator_retry_alignment.py diff --git a/src/Undefined/services/coordinator/background.py b/src/Undefined/services/coordinator/background.py index 6f7d81a6..2aa993d7 100644 --- a/src/Undefined/services/coordinator/background.py +++ b/src/Undefined/services/coordinator/background.py @@ -6,6 +6,7 @@ import logging from typing import TYPE_CHECKING, Any, cast +from Undefined.ai.queue_budget import resolve_effective_retry_count from Undefined.services.queue_manager import QUEUE_LANE_BACKGROUND from Undefined.utils.resources import read_text_resource @@ -174,8 +175,14 @@ async def _execute_queued_llm_call(self, request: dict[str, Any]) -> None: retry_count, ) except Exception as exc: - retry_count = request.get("_retry_count", 0) - if retry_count >= self.config.ai_request_max_retries: + # 重试上限以 QueueManager 为准（与队列实际重试逻辑、等待超时同源）， + # 否则热更新后两边分叉：等待方可能在仍会重试时就收到失败，或在重试 + # 已耗尽时一直挂到 480s 超时。 + retry_count = int(request.get("_retry_count", 0) or 0) + max_retries = resolve_effective_retry_count( + self.config, getattr(self, "queue_manager", None) + ) + if retry_count >= max_retries: self.ai.set_llm_call_result(request_id, exc) raise diff --git a/tests/test_coordinator_retry_alignment.py b/tests/test_coordinator_retry_alignment.py new file mode 100644 index 00000000..ee5cc76f --- /dev/null +++ b/tests/test_coordinator_retry_alignment.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, Mock + +import pytest + +from Undefined.services.coordinator.background import BackgroundMixin +from Undefined.services.queue_manager import QUEUE_LANE_BACKGROUND + + +def _make_coordinator( + *, + queue_max_retries: int | None, + config_max_retries: int, +) -> Any: + coordinator: Any = object.__new__(BackgroundMixin) + coordinator.config = SimpleNamespace( + ai_request_max_retries=config_max_retries, + ) + if queue_max_retries is None: + queue_manager = SimpleNamespace() + else: + queue_manager = SimpleNamespace( + get_max_retries=Mock(return_value=queue_max_retries) + ) + coordinator.queue_manager = queue_manager + ai = SimpleNamespace() + ai.request_model = AsyncMock(side_effect=RuntimeError("llm boom")) + ai.set_llm_call_result = Mock() + coordinator.ai = ai + return coordinator + + +def _request(retry_count: int) -> dict[str, Any]: + return { + "type": "queued_llm_call", + "request_id": "req-1", + "model_config": SimpleNamespace(model_name="chat", max_tokens=64), + "messages": [{"role": "user", "content": "hi"}], + "call_type": "background", + "_retry_count": retry_count, + "_queue_lane": QUEUE_LANE_BACKGROUND, + } + + +@pytest.mark.asyncio +async def test_waiter_keeps_waiting_when_queue_can_still_retry() -> None: + """队列上限大于 config 时，不应过早把失败交给等待方。""" + coordinator = _make_coordinator(queue_max_retries=3, config_max_retries=0) + + with pytest.raises(RuntimeError, match="llm boom"): + await coordinator._execute_queued_llm_call(_request(retry_count=0)) + + coordinator.ai.set_llm_call_result.assert_not_called() + + +@pytest.mark.asyncio +async def test_waiter_released_when_queue_retries_exhausted() -> None: + """重试已耗尽时必须立刻唤醒等待方，而不是挂到 480s 超时。""" + coordinator = _make_coordinator(queue_max_retries=3, config_max_retries=9) + + with pytest.raises(RuntimeError, match="llm boom"): + await coordinator._execute_queued_llm_call(_request(retry_count=3)) + + coordinator.ai.set_llm_call_result.assert_called_once() + assert coordinator.ai.set_llm_call_result.call_args.args[0] == "req-1" + assert isinstance( + coordinator.ai.set_llm_call_result.call_args.args[1], RuntimeError + ) + + +@pytest.mark.asyncio +async def test_config_retries_used_when_queue_manager_missing_limit() -> None: + coordinator = _make_coordinator(queue_max_retries=None, config_max_retries=2) + + with pytest.raises(RuntimeError): + await coordinator._execute_queued_llm_call(_request(retry_count=1)) + coordinator.ai.set_llm_call_result.assert_not_called() + + coordinator = _make_coordinator(queue_max_retries=None, config_max_retries=2) + with pytest.raises(RuntimeError): + await coordinator._execute_queued_llm_call(_request(retry_count=2)) + coordinator.ai.set_llm_call_result.assert_called_once() + + +@pytest.mark.asyncio +async def test_zero_retry_limit_releases_immediately() -> None: + coordinator = _make_coordinator(queue_max_retries=0, config_max_retries=5) + + with pytest.raises(RuntimeError): + await coordinator._execute_queued_llm_call(_request(retry_count=0)) + + coordinator.ai.set_llm_call_result.assert_called_once() From a7d62d37b16df5cd11cae7a27324c1ec675cff75 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:54:32 +0800 Subject: [PATCH 09/30] =?UTF-8?q?chore:=20=E6=B8=85=E7=90=86=E7=A1=AE?= =?UTF-8?q?=E8=AE=A4=E6=97=A0=E5=BC=95=E7=94=A8=E7=9A=84=E6=AD=BB=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 删除 services/commands/{stats,bugfix}.py：CommandDispatcher 已不再通过 多重继承组合它们，全仓无引用（实际 /stats、/bugfix 走 skills/commands/*； 与 services/command.py 的 _handle_* 方法）； - 移除死配置 cognitive.historian.rewrite_max_retry（解析并写进文档但无调用点）， 同步清理示例与文档； - 移除死依赖 imgkit、croniter（源码零引用）并重新生成 uv.lock； - 删除 .githooks/pre-tag：git 没有 pre-tag 事件，该钩子永不执行； 文档改指向 Release workflow 的 release_notes.py validate； - 移除 SecurityService.check_rate_limit / record_rate_limit 两个无调用方法。 --- .githooks/pre-tag | 26 - config.toml.example | 3 - docs/app.md | 2 +- docs/build.md | 3 +- docs/cognitive-memory.md | 3 +- docs/configuration.md | 1 - docs/development.md | 4 +- pyproject.toml | 2 - src/Undefined/config/domain_parsers.py | 3 - src/Undefined/config/models.py | 1 - src/Undefined/services/commands/bugfix.py | 189 ----- src/Undefined/services/commands/stats.py | 822 ---------------------- src/Undefined/services/security.py | 8 - uv.lock | 28 - 14 files changed, 7 insertions(+), 1088 deletions(-) delete mode 100755 .githooks/pre-tag delete mode 100644 src/Undefined/services/commands/bugfix.py delete mode 100644 src/Undefined/services/commands/stats.py diff --git a/.githooks/pre-tag b/.githooks/pre-tag deleted file mode 100755 index 1f06d525..00000000 --- a/.githooks/pre-tag +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -TAG_NAME="${1:-}" - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -cd "$(git rev-parse --show-toplevel)" - -if [ -z "$TAG_NAME" ]; then - echo -e "${RED}❌ 缺少 tag 名称${NC}" - exit 1 -fi - -if ! command -v uv >/dev/null 2>&1; then - echo -e "${RED}❌ 未找到 uv，请先安装 uv${NC}" - exit 1 -fi - -echo -e "${YELLOW}🔖 校验 release 版本一致性: $TAG_NAME${NC}" -uv run python scripts/release_notes.py validate --tag "$TAG_NAME" - -echo -e "${GREEN}✅ 版本检查通过: ${TAG_NAME#v}${NC}" diff --git a/config.toml.example b/config.toml.example index 9f1bf05e..53d3c064 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1762,9 +1762,6 @@ profile_top_k = 8 rerank_candidate_multiplier = 3 [cognitive.historian] -# zh: 记忆改写最大重试次数。 -# en: Max retries for memory rewrite. -rewrite_max_retry = 2 # zh: 提供给史官的最近消息参考条数（用于实体消歧）；0=禁用。 # en: Number of recent message references for historian disambiguation; 0 disables. recent_messages_inject_k = 12 diff --git a/docs/app.md b/docs/app.md index cfd03a07..bba8b028 100644 --- a/docs/app.md +++ b/docs/app.md @@ -133,7 +133,7 @@ Android 端仍然走同一套连接模型，但 UI 目标是： - Console：`Undefined-Console-*` 桌面端和 Android 产物 - Chat：`Undefined-Chat-*` 桌面端和 Android 产物 -两个 App 的版本都必须与 `pyproject.toml` 主版本一致。使用 `uv run python scripts/bump_version.py ` 统一更新版本；pre-tag 和 Release workflow 会校验 Console / Chat 的 `package.json`、`package-lock.json`、`Cargo.toml`、`tauri.conf.json` 和 `Cargo.lock`。 +两个 App 的版本都必须与 `pyproject.toml` 主版本一致。使用 `uv run python scripts/bump_version.py ` 统一更新版本；Release workflow 会校验 Console / Chat 的 `package.json`、`package-lock.json`、`Cargo.toml`、`tauri.conf.json` 和 `Cargo.lock`。 ## 9. 本地开发 diff --git a/docs/build.md b/docs/build.md index 5067cfd7..e11e5abd 100644 --- a/docs/build.md +++ b/docs/build.md @@ -272,9 +272,10 @@ Biome v2 通过 `files.includes` 表达检查范围和排除规则。两个 App ```text .githooks/pre-commit -.githooks/pre-tag ``` +> 打 tag 前的版本一致性校验不在本地钩子中执行（git 没有 `pre-tag` 事件），由 Release workflow 调用 `scripts/release_notes.py validate` 完成。 + 安装方式： ```bash diff --git a/docs/cognitive-memory.md b/docs/cognitive-memory.md index d8f7b407..2f74a911 100644 --- a/docs/cognitive-memory.md +++ b/docs/cognitive-memory.md @@ -331,7 +331,6 @@ data/cognitive/ | 字段 | 类型 | 默认值 | 说明 | |------|------|--------|------| -| `rewrite_max_retry` | int | `2` | 绝对化改写最大重试次数（支持热更新） | | `recent_messages_inject_k` | int | `12` | 提供给史官的最近消息参考条数（0=禁用，支持热更新） | | `recent_message_line_max_len` | int | `240` | 最近消息参考中每条文本最大长度（支持热更新） | | `source_message_max_len` | int | `800` | 当前消息原文最大长度（支持热更新） | @@ -374,7 +373,7 @@ data/cognitive/ ### 热更新说明 -- **支持热更新**：`cognitive.query.*`、`cognitive.historian.poll_interval_seconds`、`cognitive.historian.rewrite_max_retry`、`cognitive.historian.recent_messages_inject_k`、`cognitive.historian.recent_message_line_max_len`、`cognitive.historian.source_message_max_len` +- **支持热更新**：`cognitive.query.*`、`cognitive.historian.poll_interval_seconds`、`cognitive.historian.recent_messages_inject_k`、`cognitive.historian.recent_message_line_max_len`、`cognitive.historian.source_message_max_len` - **需重启**：`cognitive.enabled`、`cognitive.vector_store.*`、`models.embedding.*`、`models.rerank.*` 说明： diff --git a/docs/configuration.md b/docs/configuration.md index e31fe394..834c046c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1230,7 +1230,6 @@ api_key = "replace-with-your-key" | 字段 | 默认值 | 说明 | |---|---:|---| -| `rewrite_max_retry` | `2` | 绝对化改写最大重试 | | `recent_messages_inject_k` | `12` | 注入给史官的近期消息条数 | | `recent_message_line_max_len` | `240` | 每条近期消息最大字符数 | | `source_message_max_len` | `800` | 当前触发消息最大字符数 | diff --git a/docs/development.md b/docs/development.md index 623fdca3..222cfdfc 100644 --- a/docs/development.md +++ b/docs/development.md @@ -94,9 +94,11 @@ npm run check ```text .githooks/pre-commit -.githooks/pre-tag ``` +> Git 没有 `pre-tag` 钩子事件，打 tag 前的版本校验由 Release workflow 调用 +> `uv run python scripts/release_notes.py validate --tag ` 完成。 + 启用方式： ```bash diff --git a/pyproject.toml b/pyproject.toml index 64a25c87..31a902bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ dependencies = [ "crawl4ai>=0.8.6", "matplotlib", "pillow", - "imgkit", "markdown>=3.10", "types-markdown>=3.10.0.20251106", "aiofiles>=25.1.0", @@ -31,7 +30,6 @@ dependencies = [ "aiohttp>=3.13.2", "types-aiofiles>=25.1.0.20251011", "APScheduler>=3.10.0", - "croniter>=6.2.2", "pymupdf>=1.27.2,!=1.27.2.2", "python-docx>=1.2.0", "python-pptx>=1.0.2", diff --git a/src/Undefined/config/domain_parsers.py b/src/Undefined/config/domain_parsers.py index d6eaa153..7d9e70df 100644 --- a/src/Undefined/config/domain_parsers.py +++ b/src/Undefined/config/domain_parsers.py @@ -121,9 +121,6 @@ def _parse_cognitive_config(data: dict[str, Any]) -> CognitiveConfig: rerank_candidate_multiplier=_coerce_int( q.get("rerank_candidate_multiplier") if isinstance(q, dict) else None, 3 ), - rewrite_max_retry=_coerce_int( - hist.get("rewrite_max_retry") if isinstance(hist, dict) else None, 2 - ), historian_recent_messages_inject_k=_coerce_int( hist.get("recent_messages_inject_k") if isinstance(hist, dict) else None, 12, diff --git a/src/Undefined/config/models.py b/src/Undefined/config/models.py index 07ccb6ee..b48d85c4 100644 --- a/src/Undefined/config/models.py +++ b/src/Undefined/config/models.py @@ -538,7 +538,6 @@ class CognitiveConfig: time_decay_min_similarity: float = 0.35 tool_default_top_k: int = 12 profile_top_k: int = 8 - rewrite_max_retry: int = 2 poll_interval_seconds: float = 1.0 stale_job_timeout_seconds: float = 300.0 profile_revision_keep: int = 5 diff --git a/src/Undefined/services/commands/bugfix.py b/src/Undefined/services/commands/bugfix.py deleted file mode 100644 index f4689a3e..00000000 --- a/src/Undefined/services/commands/bugfix.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Bug 修复归档命令（/bugfix）的实现逻辑。 - -本模块提供 ``BugfixCommandMixin``，供 ``CommandDispatcher`` 通过多重继承组合。 -通过回溯群聊记录并调用 AI 摘要，自动生成 FAQ 归档条目。 -""" - -from __future__ import annotations - -# 斜杠命令：目录扫描注册、权限/限流/子命令路由 - -import logging -from datetime import datetime -from typing import TYPE_CHECKING, Any -from uuid import uuid4 - -from Undefined.faq import extract_faq_title -from Undefined.onebot import ( - get_message_content, - get_message_sender_id, - parse_message_time, -) - -if TYPE_CHECKING: - from Undefined.config import Config - from Undefined.faq import FAQStorage - from Undefined.onebot import OneBotClient - from Undefined.utils.sender import MessageSender - -logger = logging.getLogger(__name__) - - -class BugfixCommandMixin: - """``/bugfix`` 命令相关方法集合，作为 ``CommandDispatcher`` 的 mixin 使用。""" - - if TYPE_CHECKING: - ai: Any - config: Config - faq_storage: FAQStorage - onebot: OneBotClient - sender: MessageSender - - async def _handle_bugfix( - self, group_id: int, admin_id: int, args: list[str] - ) -> None: - """处理 ``/bugfix`` 命令，通过分析聊天记录自动生成 FAQ 归档。""" - parsed = self._parse_bugfix_args(args) - if isinstance(parsed, str): - await self.sender.send_group_message(group_id, parsed) - return - - target_qqs, start_date, end_date, start_str, end_str = parsed - - await self.sender.send_group_message( - group_id, "🔍 正在获取对话记录进行回溯分析..." - ) - - try: - messages = await self._fetch_messages( - group_id, target_qqs, start_date, end_date - ) - if not messages: - await self.sender.send_group_message( - group_id, "❌ 未找到符合条件的对话记录。" - ) - return - - processed_text = await self._process_messages(messages) - summary = await self._obtain_bugfix_summary(group_id, processed_text) - - title = extract_faq_title(summary) - if not title or title == "未命名问题": - title = await self.ai.generate_title(summary) - - faq = await self.faq_storage.create( - group_id=group_id, - target_qq=target_qqs[0], - start_time=start_str, - end_time=end_str, - title=title, - content=summary, - ) - - result_msg = f"✅ Bug 修复分析完成！\n\n📌 FAQ ID: {faq.id}\n📋 标题: {title}\n\n{summary}" - await self.sender.send_group_message(group_id, result_msg) - - except Exception as e: - error_id = uuid4().hex[:8] - logger.exception("Bugfix 失败: error_id=%s err=%s", error_id, e) - await self.sender.send_group_message( - group_id, - f"❌ Bug 修复分析失败，请稍后重试（错误码: {error_id}）", - ) - - def _parse_bugfix_args( - self, args: list[str] - ) -> tuple[list[int], datetime, datetime, str, str] | str: - """解析 ``/bugfix`` 命令的参数。""" - if len(args) < 3: - return ( - "❌ 用法: /bugfix [QQ号|@用户2] ... <开始时间> <结束时间>\n" - "时间格式: YYYY/MM/DD/HH:MM，结束时间可用 now\n" - "示例: /bugfix 123456 2024/12/01/09:00 now" - ) - - try: - target_qqs = [int(arg) for arg in args[:-2]] - start_str, end_str_raw = args[-2], args[-1] - start_date = datetime.strptime(start_str, "%Y/%m/%d/%H:%M") - - if end_str_raw.lower() == "now": - end_date, end_str = datetime.now(), "now" - else: - end_date, end_str = ( - datetime.strptime(end_str_raw, "%Y/%m/%d/%H:%M"), - end_str_raw, - ) - - return target_qqs, start_date, end_date, start_str, end_str - except ValueError: - return "❌ 参数格式错误：QQ号应为数字或 @ 提及，时间格式应为 YYYY/MM/DD/HH:MM。" - - async def _obtain_bugfix_summary(self, group_id: int, processed_text: str) -> str: - """利用 AI 生成聊天记录的 Bug 分析摘要。""" - total_tokens = self.ai.count_tokens(processed_text) - max_tokens = self.config.chat_model.max_tokens - - if total_tokens <= max_tokens: - return str(await self.ai.summarize_chat(processed_text)) - - await self.sender.send_group_message( - group_id, f"📊 消息较长（{total_tokens} tokens），正在分段处理..." - ) - chunks = self.ai.split_messages_by_tokens(processed_text, max_tokens) - summaries = [await self.ai.summarize_chat(chunk) for chunk in chunks] - return str(await self.ai.merge_summaries(summaries)) - - async def _fetch_messages( - self, - group_id: int, - target_qqs: list[int], - start_date: datetime, - end_date: datetime, - ) -> list[dict[str, Any]]: - """从 OneBot 拉取指定时间范围内目标用户的消息。""" - batch = await self.onebot.get_group_msg_history(group_id, count=2500) - if not batch: - return [] - target_qqs_set = set(target_qqs) - results = [] - for msg in batch: - msg_time = parse_message_time(msg) - if ( - start_date <= msg_time <= end_date - and get_message_sender_id(msg) in target_qqs_set - ): - # 后台循环处理队列 - results.append(msg) - return sorted(results, key=lambda m: m.get("time", 0)) - - # 后台循环处理队列 - async def _process_messages(self, messages: list[dict[str, Any]]) -> str: - """将原始 OneBot 消息序列化为 AI 可读的纯文本。""" - lines = [] - for msg in messages: - sender_id = get_message_sender_id(msg) - msg_time = parse_message_time(msg).strftime("%Y-%m-%d %H:%M:%S") - content = get_message_content(msg) - text_parts = [] - for segment in content: - seg_type, seg_data = segment.get("type", ""), segment.get("data", {}) - if seg_type == "text": - text_parts.append(seg_data.get("text", "")) - elif seg_type == "image": - file = seg_data.get("file", "") or seg_data.get("url", "") - if file: - try: - url = await self.onebot.get_image(file) - if url: - res = await self.ai.analyze_multimodal(url, "image") - text_parts.append( - f"[pic]{res.get('description', '')}{res.get('ocr_text', '')}[/pic]" - ) - except Exception: - text_parts.append("[pic]图片处理失败[/pic]") - elif seg_type == "at": - text_parts.append(f"@{seg_data.get('qq', '')}") - if text_parts: - lines.append(f"[{msg_time}] {sender_id}: {''.join(text_parts)}") - return "\n".join(lines) diff --git a/src/Undefined/services/commands/stats.py b/src/Undefined/services/commands/stats.py deleted file mode 100644 index a992f760..00000000 --- a/src/Undefined/services/commands/stats.py +++ /dev/null @@ -1,822 +0,0 @@ -"""Token 使用统计命令（/stats）的实现逻辑。 - -本模块提供 ``StatsCommandMixin``，供 ``CommandDispatcher`` 通过多重继承组合。 -群聊与私聊统计、图表生成、AI 分析队列交互均在此实现。 -""" - -from __future__ import annotations - -# 斜杠命令：目录扫描注册、权限/限流/子命令路由 - -import asyncio -import base64 -import logging -import re -from pathlib import Path -from typing import TYPE_CHECKING, Any -from uuid import uuid4 - -from Undefined.ai.queue_budget import ( - compute_queued_llm_timeout_seconds, - resolve_effective_retry_count, -) -from Undefined.token_usage_storage import TokenUsageStorage - -if TYPE_CHECKING: - from Undefined.config import Config - from Undefined.onebot import OneBotClient - from Undefined.utils.history import MessageHistoryManager - from Undefined.utils.sender import MessageSender - -# 尝试导入 matplotlib（可选依赖） -plt: Any -try: - import matplotlib.pyplot as plt - - _MATPLOTLIB_AVAILABLE = True -except ImportError: - plt = None - _MATPLOTLIB_AVAILABLE = False - -logger = logging.getLogger(__name__) - -_STATS_DEFAULT_DAYS = 7 -_STATS_MIN_DAYS = 1 -_STATS_MAX_DAYS = 365 -_STATS_MODEL_TOP_N = 8 -_STATS_CALL_TYPE_TOP_N = 12 -_STATS_DATA_SUMMARY_MAX_CHARS = 12000 -_STATS_AI_FLAGS = {"--ai", "-a"} -_STATS_TIME_RANGE_RE = re.compile(r"^\d+[dwm]?$", re.IGNORECASE) - - -class StatsCommandMixin: - """``/stats`` 命令相关方法集合，作为 ``CommandDispatcher`` 的 mixin 使用。""" - - if TYPE_CHECKING: - ai: Any - config: Config - history_manager: MessageHistoryManager - onebot: OneBotClient - queue_manager: Any - sender: MessageSender - - _token_usage_storage: TokenUsageStorage - _stats_analysis_results: dict[str, str] - _stats_analysis_events: dict[str, asyncio.Event] - - def _parse_time_range(self, time_str: str) -> int: - """解析时间范围字符串，返回天数。 - - 参数: - time_str: 时间范围字符串（如 ``7d``、``1w``、``30d``）。 - - 返回: - clamp 在 ``[_STATS_MIN_DAYS, _STATS_MAX_DAYS]`` 内的天数。 - """ - if not time_str: - return _STATS_DEFAULT_DAYS - - def _clamp_days(value: int) -> int: - if value < _STATS_MIN_DAYS: - return _STATS_DEFAULT_DAYS - if value > _STATS_MAX_DAYS: - return _STATS_MAX_DAYS - return value - - time_str = time_str.lower().strip() - - if time_str.endswith("d"): - try: - return _clamp_days(int(time_str[:-1])) - except ValueError: - return _STATS_DEFAULT_DAYS - if time_str.endswith("w"): - try: - return _clamp_days(int(time_str[:-1]) * 7) - except ValueError: - return _STATS_DEFAULT_DAYS - if time_str.endswith("m"): - try: - return _clamp_days(int(time_str[:-1]) * 30) - except ValueError: - return _STATS_DEFAULT_DAYS - - try: - return _clamp_days(int(time_str)) - except ValueError: - return _STATS_DEFAULT_DAYS - - def _parse_stats_options(self, args: list[str]) -> tuple[int, bool]: - """解析 ``/stats`` 参数：时间范围 + AI 分析开关。""" - days = _STATS_DEFAULT_DAYS - enable_ai_analysis = False - picked_days = False - - for raw in args: - token = str(raw or "").strip() - if not token: - continue - lower = token.lower() - if lower in _STATS_AI_FLAGS: - enable_ai_analysis = True - continue - if not picked_days and _STATS_TIME_RANGE_RE.match(lower): - days = self._parse_time_range(lower) - picked_days = True - - return days, enable_ai_analysis - - async def _handle_stats( - self, group_id: int, sender_id: int, args: list[str] - ) -> None: - """处理群聊 ``/stats`` 命令，生成 token 使用统计图表（可选 AI 分析）。""" - if not _MATPLOTLIB_AVAILABLE: - await self.sender.send_group_message( - group_id, "❌ 缺少必要的库，无法生成图表。请安装 matplotlib。" - ) - return - - days, enable_ai_analysis = self._parse_stats_options(args) - - try: - summary = await self._token_usage_storage.get_summary(days=days) - if summary["total_calls"] == 0: - await self.sender.send_group_message( - group_id, f"📊 最近 {days} 天内无 Token 使用记录。" - ) - return - - from Undefined.utils.paths import RENDER_CACHE_DIR, ensure_dir - - img_dir = ensure_dir(RENDER_CACHE_DIR) - await self._generate_line_chart(summary, img_dir, days) - await self._generate_bar_chart(summary, img_dir) - await self._generate_pie_chart(summary, img_dir) - await self._generate_stats_table(summary, img_dir) - - ai_analysis = "" - if enable_ai_analysis: - ai_analysis = await self._run_stats_ai_analysis( - scope="group", - scope_id=group_id, - sender_id=sender_id, - summary=summary, - days=days, - ) - - forward_messages = self._build_stats_forward_nodes( - summary, img_dir, days, ai_analysis - ) - await self._send_group_forward_message( - group_id, - forward_messages, - history_message=self._build_stats_history_message( - summary, - days, - ai_analysis, - ), - ) - - from Undefined.utils.cache import cleanup_cache_dir - - cleanup_cache_dir(RENDER_CACHE_DIR) - - except Exception as e: - error_id = uuid4().hex[:8] - logger.exception( - "[Stats] 生成统计图表失败: error_id=%s err=%s", error_id, e - ) - await self.sender.send_group_message( - group_id, - f"❌ 生成统计图表失败，请稍后重试（错误码: {error_id}）", - ) - - async def _send_group_forward_message( - self, - group_id: int, - messages: list[dict[str, Any]], - *, - history_message: str, - ) -> None: - """发送群组合并转发消息，并在需要时写入历史记录。""" - send_forward = getattr(self.sender, "send_group_forward_message", None) - if callable(send_forward): - await send_forward(group_id, messages, history_message=history_message) - return - - await self.onebot.send_forward_msg(group_id, messages) - if self.history_manager is None: - return - text_content = history_message.strip() - if not text_content: - return - - await self.history_manager.add_group_message( - group_id=group_id, - sender_id=getattr(self.config, "bot_qq", 0), - text_content=text_content, - sender_nickname="Bot", - group_name="", - ) - - @staticmethod - def _build_stats_history_message( - summary: dict[str, Any], - days: int, - ai_analysis: str, - ) -> str: - """构建写入群聊历史的 ``/stats`` 输出摘要文本。""" - lines = [ - f"[命令输出] /stats 最近 {days} 天 Token 使用统计", - f"总调用: {summary.get('total_calls', 0)}", - f"总 Token: {summary.get('total_tokens', 0)}", - f"输入 Token: {summary.get('prompt_tokens', 0)}", - f"输出 Token: {summary.get('completion_tokens', 0)}", - ] - if ai_analysis.strip(): - lines.extend(["", "AI 分析:", ai_analysis.strip()]) - return "\n".join(lines) - - async def _handle_stats_private( - self, - user_id: int, - sender_id: int, - args: list[str], - send_message: Any = None, - *, - is_webui_session: bool = False, - ) -> None: - """处理私聊 ``/stats``（含 WebUI 虚拟私聊适配）。""" - - async def _send_private(message: str) -> None: - if send_message is not None: - await send_message(message) - else: - await self.sender.send_private_message(user_id, message) - - days, enable_ai_analysis = self._parse_stats_options(args) - try: - summary = await self._token_usage_storage.get_summary(days=days) - if summary["total_calls"] == 0: - await _send_private(f"📊 最近 {days} 天内无 Token 使用记录。") - return - - ai_analysis = "" - if enable_ai_analysis: - ai_analysis = await self._run_stats_ai_analysis( - scope="private", - scope_id=0, - sender_id=sender_id, - summary=summary, - days=days, - ) - - if not _MATPLOTLIB_AVAILABLE: - message = "❌ 缺少必要的库，无法生成图表。请安装 matplotlib。" - if is_webui_session: - message += "\n\n" + self._build_stats_summary_text(summary) - if ai_analysis: - message += f"\n\n🤖 AI 智能分析\n{ai_analysis}" - await _send_private(message) - return - - from Undefined.utils.cache import cleanup_cache_dir - from Undefined.utils.paths import RENDER_CACHE_DIR, ensure_dir - - img_dir = ensure_dir(RENDER_CACHE_DIR) - await self._generate_line_chart(summary, img_dir, days) - await self._generate_bar_chart(summary, img_dir) - await self._generate_pie_chart(summary, img_dir) - await self._generate_stats_table(summary, img_dir) - - await _send_private(f"📊 最近 {days} 天的 Token 使用统计：") - for img_name in ["line_chart", "bar_chart", "pie_chart", "table"]: - img_path = img_dir / f"stats_{img_name}.png" - if img_path.exists(): - message = await self._build_private_stats_image_message( - img_path, - inline_base64=is_webui_session, - ) - await _send_private(message) - - await _send_private(self._build_stats_summary_text(summary)) - if ai_analysis: - await _send_private(f"🤖 AI 智能分析\n{ai_analysis}") - - cleanup_cache_dir(RENDER_CACHE_DIR) - except Exception as e: - error_id = uuid4().hex[:8] - logger.exception( - "[Stats] 私聊统计生成失败: error_id=%s user=%s err=%s", - error_id, - user_id, - e, - ) - await _send_private( - f"❌ 生成统计图表失败，请稍后重试（错误码: {error_id}）" - ) - - async def _build_private_stats_image_message( - self, - image_path: Path, - *, - inline_base64: bool, - ) -> str: - """构建私聊统计图片的 OneBot CQ 码消息。""" - file_uri = image_path.absolute().as_uri() - if not inline_base64: - return f"[CQ:image,file={file_uri}]" - - try: - encoded = await asyncio.to_thread( - lambda: base64.b64encode(image_path.read_bytes()).decode("ascii") - ) - except Exception as exc: - logger.warning( - "[Stats] 图像 base64 编码失败，回退文件路径: path=%s err=%s", - file_uri, - exc, - ) - return f"[CQ:image,file={file_uri}]" - - return f"[CQ:image,file=base64://{encoded}]" - - async def _run_stats_ai_analysis( - self, - *, - scope: str, - scope_id: int, - sender_id: int, - summary: dict[str, Any], - days: int, - ) -> str: - """投递并等待 AI 对统计数据的分析结果。""" - if not self.queue_manager: - return "" - - data_summary = self._build_data_summary(summary, days) - request_id = uuid4().hex - analysis_event = asyncio.Event() - self._stats_analysis_events[request_id] = analysis_event - request_data = { - "type": "stats_analysis", - "group_id": scope_id, - "request_id": request_id, - "sender_id": sender_id, - "data_summary": data_summary, - "summary": summary, - "days": days, - "scope": scope, - } - receipt = await self.queue_manager.add_group_mention_request( - request_data, model_name=self.config.chat_model.model_name - ) - logger.info("[Stats] 已投递 AI 分析请求: scope=%s target=%s", scope, scope_id) - - wait_timeout = compute_queued_llm_timeout_seconds( - self.ai.runtime_config, - self.config.chat_model, - retry_count=resolve_effective_retry_count( - self.ai.runtime_config, self.queue_manager - ), - initial_wait_seconds=float( - getattr(receipt, "estimated_wait_seconds", 0.0) or 0.0 - ), - ) - try: - await asyncio.wait_for(analysis_event.wait(), timeout=wait_timeout) - ai_analysis = self._stats_analysis_results.pop(request_id, "") - logger.info( - "[Stats] 已获取 AI 分析结果: scope=%s len=%s", scope, len(ai_analysis) - ) - return ai_analysis - except asyncio.TimeoutError: - logger.warning( - "[Stats] AI 分析超时: scope=%s target=%s timeout=%.1fs", - scope, - scope_id, - wait_timeout, - ) - return "AI 分析超时，已先发送图表与汇总数据。" - finally: - self._stats_analysis_events.pop(request_id, None) - self._stats_analysis_results.pop(request_id, None) - - def _build_data_summary(self, summary: dict[str, Any], days: int) -> str: - """构建用于 AI 分析的统计数据摘要。""" - lines = [] - lines.append("📊 Token 使用综合分析数据：") - lines.append("") - - lines.append("【整体概况】") - lines.append(f"统计周期: {days} 天") - lines.append(f"总调用次数: {summary['total_calls']}") - lines.append(f"总 Token 消耗: {summary['total_tokens']:,}") - lines.append(f"平均响应时间: {summary['avg_duration']:.2f}s") - lines.append(f"涉及模型数: {len(summary['models'])}") - lines.append("") - - daily_stats = summary.get("daily_stats", {}) - if daily_stats: - dates = sorted(daily_stats.keys()) - total_daily_calls = sum(daily_stats[d]["calls"] for d in dates) - total_daily_tokens = sum(daily_stats[d]["tokens"] for d in dates) - avg_daily_calls = total_daily_calls / len(dates) if dates else 0 - avg_daily_tokens = total_daily_tokens / len(dates) if dates else 0 - - peak_day = ( - max(dates, key=lambda d: daily_stats[d]["tokens"]) if dates else "" - ) - peak_day_tokens = daily_stats[peak_day]["tokens"] if peak_day else 0 - - lines.append("【时间维度】") - lines.append(f"统计天数: {len(dates)} 天") - lines.append(f"每日平均调用: {avg_daily_calls:.1f} 次") - lines.append(f"每日平均 Token: {avg_daily_tokens:,.0f} 个") - lines.append(f"高峰日期: {peak_day} ({peak_day_tokens:,} tokens)") - lines.append("") - - models = summary.get("models", {}) - if models: - lines.append("【模型维度】") - total_tokens_all = summary["total_tokens"] - sorted_models = sorted( - models.items(), key=lambda x: x[1]["tokens"], reverse=True - ) - for model_name, model_data in sorted_models[:_STATS_MODEL_TOP_N]: - calls = model_data["calls"] - tokens = model_data["tokens"] - prompt_tokens = model_data["prompt_tokens"] - completion_tokens = model_data["completion_tokens"] - token_pct = ( - (tokens / total_tokens_all * 100) if total_tokens_all > 0 else 0 - ) - avg_per_call = tokens / calls if calls > 0 else 0 - io_ratio = completion_tokens / prompt_tokens if prompt_tokens > 0 else 0 - - lines.append(f"模型: {model_name}") - lines.append( - f" - 调用次数: {calls} ({calls / summary['total_calls'] * 100:.1f}%)" - ) - lines.append(f" - Token 消耗: {tokens:,} ({token_pct:.1f}%)") - lines.append(f" - 平均每次调用: {avg_per_call:.0f} tokens") - lines.append( - f" - 输入: {prompt_tokens:,} / 输出: {completion_tokens:,}" - ) - lines.append(f" - 输入/输出比: 1:{io_ratio:.2f}") - lines.append("") - - if len(sorted_models) > _STATS_MODEL_TOP_N: - others = sorted_models[_STATS_MODEL_TOP_N:] - others_calls = sum(int(item[1].get("calls", 0)) for item in others) - others_tokens = sum(int(item[1].get("tokens", 0)) for item in others) - others_pct = ( - (others_tokens / total_tokens_all * 100) - if total_tokens_all > 0 - else 0.0 - ) - lines.append( - f"其余 {len(others)} 个模型合计: 调用 {others_calls} 次, Token {others_tokens:,} ({others_pct:.1f}%)" - ) - lines.append("") - - call_types = summary.get("call_types", {}) - if call_types: - lines.append("【调用类型维度】") - sorted_types = sorted( - call_types.items(), key=lambda item: int(item[1]), reverse=True - ) - total_calls = max(1, int(summary.get("total_calls", 0))) - for call_type, count in sorted_types[:_STATS_CALL_TYPE_TOP_N]: - ratio = int(count) / total_calls * 100 - lines.append(f"- {call_type}: {count} 次 ({ratio:.1f}%)") - if len(sorted_types) > _STATS_CALL_TYPE_TOP_N: - rest_count = sum( - int(item[1]) for item in sorted_types[_STATS_CALL_TYPE_TOP_N:] - ) - ratio = rest_count / total_calls * 100 - lines.append( - f"- 其他 {len(sorted_types) - _STATS_CALL_TYPE_TOP_N} 类: {rest_count} 次 ({ratio:.1f}%)" - ) - lines.append("") - - prompt_tokens = summary.get("prompt_tokens", 0) - completion_tokens = summary.get("completion_tokens", 0) - total_tokens = summary.get("total_tokens", 0) - input_ratio = (prompt_tokens / total_tokens * 100) if total_tokens > 0 else 0 - output_ratio = ( - (completion_tokens / total_tokens * 100) if total_tokens > 0 else 0 - ) - output_per_input = completion_tokens / prompt_tokens if prompt_tokens > 0 else 0 - - lines.append("【效率指标】") - lines.append(f"输入 Token: {prompt_tokens:,} ({input_ratio:.1f}%)") - lines.append(f"输出 Token: {completion_tokens:,} ({output_ratio:.1f}%)") - lines.append(f"输入/输出比: 1:{output_per_input:.2f}") - lines.append("") - - if daily_stats and len(daily_stats) > 1: - lines.append("【趋势分析】") - dates = sorted(daily_stats.keys()) - first_day_tokens = daily_stats[dates[0]]["tokens"] - last_day_tokens = daily_stats[dates[-1]]["tokens"] - trend_change = ( - ((last_day_tokens - first_day_tokens) / first_day_tokens * 100) - if first_day_tokens > 0 - else 0 - ) - trend_desc = "增长" if trend_change > 0 else "下降" - lines.append( - f"总体趋势: {trend_desc} {abs(trend_change):.1f}% (从首日到末日)" - ) - lines.append("") - - summary_text = "\n".join(lines) - if len(summary_text) > _STATS_DATA_SUMMARY_MAX_CHARS: - trimmed = summary_text[: _STATS_DATA_SUMMARY_MAX_CHARS - 80].rstrip() - summary_text = ( - f"{trimmed}\n\n[数据摘要已截断，总长度 {len(summary_text)} 字符，" - f"仅保留前 {_STATS_DATA_SUMMARY_MAX_CHARS} 字符]" - ) - logger.info( - "[Stats] 数据摘要过长已截断: original_len=%s max_len=%s", - len("\n".join(lines)), - _STATS_DATA_SUMMARY_MAX_CHARS, - ) - return summary_text - - def _build_stats_summary_text(self, summary: dict[str, Any]) -> str: - """构建统计结果的纯文本摘要。""" - return f"""📈 摘要汇总: -• 总调用次数: {summary["total_calls"]} -• 总消耗 Tokens: {summary["total_tokens"]:,} - └─ 输入: {summary["prompt_tokens"]:,} - └─ 输出: {summary["completion_tokens"]:,} -• 平均耗时: {summary["avg_duration"]:.2f}s -• 涉及模型数: {len(summary["models"])}""" - - def set_stats_analysis_result( - self, group_id: int, request_id: str, analysis: str - ) -> None: - """设置 AI 分析结果（由队列处理器调用）。""" - event = self._stats_analysis_events.get(request_id) - if not event: - logger.warning( - "[StatsAnalysis] 未找到等待事件，群: %s, 请求: %s", - group_id, - request_id, - ) - return - self._stats_analysis_results[request_id] = analysis - event.set() - - def _build_stats_forward_nodes( - self, - summary: dict[str, Any], - img_dir: Path, - days: int, - ai_analysis: str = "", - ) -> list[dict[str, Any]]: - """构建用于合并转发的统计图表节点列表。""" - # 对外入队 API - nodes = [] - bot_qq = str(self.config.bot_qq) - - # 对外入队 API - def add_node(content: str) -> None: - nodes.append( - { - "type": "node", - "data": {"name": "Bot", "uin": bot_qq, "content": content}, - } - ) - - add_node(f"📊 最近 {days} 天的 Token 使用统计：") - - for img_name in ["line_chart", "bar_chart", "pie_chart", "table"]: - img_path = img_dir / f"stats_{img_name}.png" - if img_path.exists(): - add_node(f"[CQ:image,file={img_path.absolute().as_uri()}]") - - add_node(self._build_stats_summary_text(summary)) - - if ai_analysis: - add_node(f"🤖 AI 智能分析\n{ai_analysis}") - - return nodes - - async def _generate_line_chart( - self, summary: dict[str, Any], img_dir: Path, days: int - ) -> None: - """生成折线图：时间趋势。""" - daily_stats = summary["daily_stats"] - if not daily_stats: - return - - dates = sorted(daily_stats.keys()) - tokens = [daily_stats[d]["tokens"] for d in dates] - prompt_tokens = [daily_stats[d]["prompt_tokens"] for d in dates] - completion_tokens = [daily_stats[d]["completion_tokens"] for d in dates] - - fig, ax = plt.subplots(figsize=(12, 7)) - - ax.plot( - dates, tokens, marker="o", linewidth=2, label="Total Token", color="#2196F3" - ) - ax.plot( - dates, - prompt_tokens, - marker="s", - linewidth=2, - label="Input Token", - color="#4CAF50", - ) - ax.plot( - dates, - completion_tokens, - marker="^", - linewidth=2, - label="Output Token", - color="#FF9800", - ) - - ax.set_title( - f"Token Usage Trend for Last {days} Days", fontsize=16, fontweight="bold" - ) - ax.set_xlabel("Date", fontsize=12) - ax.set_ylabel("Token Count", fontsize=12) - ax.legend(loc="upper left", fontsize=10) - ax.grid(True, alpha=0.3) - - plt.xticks(rotation=45, ha="right") - plt.tight_layout() - - filepath = img_dir / "stats_line_chart.png" - plt.savefig(filepath, dpi=150, bbox_inches="tight") - plt.close(fig) - - async def _generate_bar_chart(self, summary: dict[str, Any], img_dir: Path) -> None: - """生成柱状图：模型对比。""" - models = summary["models"] - if not models: - return - - model_names = list(models.keys()) - tokens = [models[m]["tokens"] for m in model_names] - prompt_tokens = [models[m]["prompt_tokens"] for m in model_names] - completion_tokens = [models[m]["completion_tokens"] for m in model_names] - - fig, ax = plt.subplots(figsize=(14, 8)) - - x = range(len(model_names)) - width = 0.25 - - bars1 = ax.bar( - [i - width for i in x], - tokens, - width, - label="Total Token", - color="#2196F3", - alpha=0.8, - ) - bars2 = ax.bar( - x, - prompt_tokens, - width, - label="Input Token", - color="#4CAF50", - alpha=0.8, - ) - bars3 = ax.bar( - [i + width for i in x], - completion_tokens, - width, - label="Output Token", - color="#FF9800", - alpha=0.8, - ) - - ax.set_title("Token Usage Comparison by Model", fontsize=16, fontweight="bold") - ax.set_xlabel("Model", fontsize=12) - ax.set_ylabel("Token Count", fontsize=12) - ax.set_xticks(x) - ax.set_xticklabels(model_names, rotation=45, ha="right") - ax.legend(loc="upper right", fontsize=10) - ax.grid(True, alpha=0.3, axis="y") - - for bars in [bars1, bars2, bars3]: - for bar in bars: - height = bar.get_height() - if height > 0: - ax.text( - bar.get_x() + bar.get_width() / 2.0, - height, - f"{int(height):,}", - ha="center", - va="bottom", - fontsize=8, - ) - - plt.tight_layout() - - filepath = img_dir / "stats_bar_chart.png" - plt.savefig(filepath, dpi=150, bbox_inches="tight") - plt.close(fig) - - async def _generate_pie_chart(self, summary: dict[str, Any], img_dir: Path) -> None: - """生成饼图：输入/输出比例。""" - prompt_tokens = summary["prompt_tokens"] - completion_tokens = summary["completion_tokens"] - - if prompt_tokens == 0 and completion_tokens == 0: - return - - fig, ax = plt.subplots(figsize=(12, 8)) - - labels = ["Input Token", "Output Token"] - sizes = [prompt_tokens, completion_tokens] - colors = ["#4CAF50", "#FF9800"] - explode = (0.05, 0.05) - - wedges, *_ = ax.pie( - sizes, - explode=explode, - labels=labels, - colors=colors, - autopct="%1.1f%%", - startangle=90, - textprops={"fontsize": 12}, - ) - - ax.set_title("Input/Output Token Ratio", fontsize=16, fontweight="bold", pad=20) - - ax.legend( - wedges, - [f"{labels[i]}: {sizes[i]:,}" for i in range(len(labels))], - loc="center left", - bbox_to_anchor=(1, 0, 0.5, 1), - fontsize=10, - ) - - plt.tight_layout() - - filepath = img_dir / "stats_pie_chart.png" - plt.savefig(filepath, dpi=150, bbox_inches="tight") - plt.close(fig) - - async def _generate_stats_table( - self, summary: dict[str, Any], img_dir: Path - ) -> None: - """生成统计表格图片。""" - models = summary["models"] - if not models: - return - - model_names = list(models.keys()) - data = [] - for model in model_names: - m = models[model] - data.append( - [ - model, - m["calls"], - f"{m['tokens']:,}", - f"{m['prompt_tokens']:,}", - f"{m['completion_tokens']:,}", - ] - ) - - fig, ax = plt.subplots(figsize=(14, 9)) - ax.axis("tight") - ax.axis("off") - - table = ax.table( - cellText=data, - colLabels=["Model", "Calls", "Total Token", "Input Token", "Output Token"], - cellLoc="center", - loc="center", - ) - - table.auto_set_font_size(False) - table.set_fontsize(10) - table.scale(1.2, 1.5) - - for i in range(5): - table[(0, i)].set_facecolor("#2196F3") - table[(0, i)].set_text_props(weight="bold", color="white") - - for i in range(1, len(data) + 1): - for j in range(5): - if i % 2 == 0: - table[(i, j)].set_facecolor("#f0f0f0") - - ax.set_title( - "Model Usage Statistics Details", fontsize=16, fontweight="bold", pad=20 - ) - - plt.tight_layout() - - filepath = img_dir / "stats_table.png" - plt.savefig(filepath, dpi=150, bbox_inches="tight") - plt.close(fig) diff --git a/src/Undefined/services/security.py b/src/Undefined/services/security.py index 9d24f734..7696c87a 100644 --- a/src/Undefined/services/security.py +++ b/src/Undefined/services/security.py @@ -279,14 +279,6 @@ async def detect_injection( logger.exception("[安全] 注入检测失败: %s 耗时=%.2fs", exc, duration) return True # 安全起见默认检测到 - def check_rate_limit(self, user_id: int) -> tuple[bool, int]: - """检查速率限制""" - return self.rate_limiter.check(user_id) - - def record_rate_limit(self, user_id: int) -> None: - """记录速率限制""" - self.rate_limiter.record(user_id) - async def generate_injection_response(self, original_message: str) -> str: """生成注入攻击响应""" return await self.injection_response_agent.generate_response(original_message) diff --git a/uv.lock b/uv.lock index cb8f0255..679e372f 100644 --- a/uv.lock +++ b/uv.lock @@ -890,18 +890,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/e2/aa851e2f0248f0b19c63b0fe11f6bdc2ca64900924d399af368dc28f2a20/crawl4ai-0.8.6-py3-none-any.whl", hash = "sha256:57e127e8113640d8358705b231111d3cd9cc4ab05d44705e724cebc4a383e09c", size = 501931, upload-time = "2026-03-24T15:07:50.121Z" }, ] -[[package]] -name = "croniter" -version = "6.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762, upload-time = "2026-03-15T08:43:48.112Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/39/783980e78cb92c2d7bdb1fc7dbc86e94ccc6d58224d76a7f1f51b6c51e30/croniter-6.2.2-py3-none-any.whl", hash = "sha256:a5d17b1060974d36251ea4faf388233eca8acf0d09cbd92d35f4c4ac8f279960", size = 45422, upload-time = "2026-03-15T08:43:46.626Z" }, -] - [[package]] name = "cryptography" version = "46.0.6" @@ -1547,18 +1535,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, ] -[[package]] -name = "imgkit" -version = "1.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/96/66/49089f7ca929e854f4f110259de392ec8bbed5ac3f2f9e1ff493f6623b15/imgkit-1.2.3.tar.gz", hash = "sha256:c9ade93d0281277c898b983c959f11f0cc1cb4a5fed144560dfa4d273fb50cd8", size = 11690, upload-time = "2023-02-23T08:01:58.396Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/b0/e0c8aaafb20377eccb9aedbb9e00c21aeaac9cdc13d31d1c7532e1241bab/imgkit-1.2.3-py3-none-any.whl", hash = "sha256:cba8e3f67da6e0a5c87345a8b41605337851f9b9b75719500fa6810ecff84791", size = 10634, upload-time = "2023-02-23T08:01:57.223Z" }, -] - [[package]] name = "importlib-metadata" version = "8.7.1" @@ -4714,10 +4690,8 @@ dependencies = [ { name = "chardet" }, { name = "chromadb" }, { name = "crawl4ai" }, - { name = "croniter" }, { name = "fastmcp" }, { name = "httpx" }, - { name = "imgkit" }, { name = "langchain-community" }, { name = "linkify-it-py" }, { name = "lunar-python" }, @@ -4789,10 +4763,8 @@ requires-dist = [ { name = "chardet", specifier = ">=7.4.0.post1" }, { name = "chromadb", specifier = ">=1.5.5" }, { name = "crawl4ai", specifier = ">=0.8.6" }, - { name = "croniter", specifier = ">=6.2.2" }, { name = "fastmcp", specifier = ">=3.1.1" }, { name = "httpx", specifier = ">=0.27.0" }, - { name = "imgkit" }, { name = "langchain-community", specifier = ">=0.3.0" }, { name = "linkify-it-py", specifier = ">=2.0.3" }, { name = "lunar-python", specifier = ">=1.4.8" }, From af7d0ba210ae3c1db59ceb7d2ffd7e627606f1a0 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:56:18 +0800 Subject: [PATCH 10/30] =?UTF-8?q?fix(scripts):=20=E9=87=8D=E5=B5=8C?= =?UTF-8?q?=E5=85=A5=E8=84=9A=E6=9C=AC=E6=94=AF=E6=8C=81=E7=BB=B4=E5=BA=A6?= =?UTF-8?q?=E5=8F=98=E5=8C=96=E7=9A=84=E8=BF=81=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原实现只用 upsert 覆写，docstring 却声称覆盖“维度变化”；ChromaDB 的 collection 首次写入即定维，换嵌入模型后异维向量 upsert 会直接 InvalidArgumentError。 - 先读取现有向量维度并与新向量比较； - 维度不同则先读全量记录、删除并重建同名 collection（沿用原 hnsw 元数据） 后按新维度写回，记录不丢； - --dry-run 只提示会重建，不做任何写入；缺失 client 时明确报错而不是静默失败； - 补 chromadb 真实读写的维度变化 / 维度一致 / dry-run / 空库测试与文档说明。 --- docs/cognitive-memory.md | 2 + scripts/README.md | 2 + scripts/reembed_cognitive.py | 80 +++++++++++- tests/test_reembed_cognitive_script.py | 164 +++++++++++++++++++++++++ 4 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 tests/test_reembed_cognitive_script.py diff --git a/docs/cognitive-memory.md b/docs/cognitive-memory.md index 2f74a911..b17f92e9 100644 --- a/docs/cognitive-memory.md +++ b/docs/cognitive-memory.md @@ -438,6 +438,8 @@ data/cognitive/ 更换嵌入模型（维度变化或模型升级）后，需要对向量库进行全量重嵌入。详见 [`scripts/reembed_cognitive.py`](../scripts/reembed_cognitive.py)。 +向量维度发生变化时脚本会先读全量记录、再删除并重建 collection 后写回（ChromaDB 定维后无法原地改维，直接 upsert 异维向量会失败）；建议先 `--dry-run` 确认维度变化与记录数。 + ```bash # 1. 先在 config.toml 中更新 [models.embedding] 为新模型配置 # 2. 停止机器人 diff --git a/scripts/README.md b/scripts/README.md index c82d7518..03bceadb 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -62,6 +62,8 @@ python scripts/sync_config_template.py --stdout **原理**：ChromaDB 存储了完整的原文本（`documents`），脚本读取所有记录，用新模型重新计算向量后 upsert 覆写，metadata 保持不变。 +**维度变化**：collection 在首次写入时定维，异维 upsert 会直接报 `InvalidArgumentError`。脚本会先比较新旧向量维度，检测到变化时先读全量记录、再删除并重建 collection，最后按新维度写回；`--dry-run` 只提示会重建，不做任何写入。 + **前置条件**：先在 `config.toml` 中将 `[models.embedding]` 更新为新模型配置。 ```bash diff --git a/scripts/reembed_cognitive.py b/scripts/reembed_cognitive.py index 053b9d91..83851b5a 100755 --- a/scripts/reembed_cognitive.py +++ b/scripts/reembed_cognitive.py @@ -7,6 +7,10 @@ 原理：ChromaDB 存储了完整的原文本（documents），本脚本读取所有记录， 用新模型重新计算向量，然后通过 upsert 覆写回去。metadata 保持不变。 +维度变化：ChromaDB 的 collection 在首次写入时定维，异维向量 upsert 会直接失败 +（InvalidArgumentError）。脚本会先比较新旧向量维度，检测到变化时删除并重建 +collection（先读全量记录再重建，不会丢数据），然后按新维度全量写回。 + 用法： # 先在 config.toml 中更新 [models.embedding] 为新模型配置 uv run python scripts/reembed_cognitive.py @@ -128,12 +132,47 @@ def _get_all_records( return all_ids, all_docs, all_metas +def _collection_dimension(collection: Any) -> int: + """读取 collection 当前向量维度；空库或读取失败返回 0。""" + try: + sample = collection.get(limit=1, include=["embeddings"]) + except Exception as exc: # pragma: no cover - 依赖 ChromaDB 内部行为 + logger.warning("读取现有向量维度失败，将按新维度直接写入: %s", exc) + return 0 + embeddings = sample.get("embeddings") + if embeddings is None or len(embeddings) == 0: + return 0 + try: + return len(embeddings[0]) + except TypeError: + return 0 + + +def _recreate_collection( + client: Any, + collection_name: str, + metadata: dict[str, Any] | None, +) -> Any: + """删除并重建 collection，用于向量维度变化后的全量写回。""" + logger.warning( + "重建 collection %s（原维度与新模型不一致，ChromaDB 不支持原地改维）", + collection_name, + ) + client.delete_collection(collection_name) + return client.get_or_create_collection( + collection_name, + metadata=metadata or {"hnsw:space": "cosine"}, + ) + + async def _reembed_collection( collection: Any, collection_name: str, embedder: Embedder, batch_size: int, dry_run: bool, + *, + client: Any = None, ) -> int: """对单个 collection 执行全量重嵌入，返回处理的记录数。""" logger.info("正在读取 %s ...", collection_name) @@ -151,7 +190,12 @@ async def _reembed_collection( batch_size, ) + current_dimension = _collection_dimension(collection) + if current_dimension: + logger.info("%s 现有向量维度: %s", collection_name, current_dimension) + processed = 0 + dimension_checked = False start_time = time.perf_counter() for i in range(0, total, batch_size): @@ -162,6 +206,34 @@ async def _reembed_collection( # 计算新向量 new_embeddings = await embedder.embed(batch_docs) + if not dimension_checked: + dimension_checked = True + new_dimension = len(new_embeddings[0]) if new_embeddings else 0 + if ( + current_dimension + and new_dimension + and current_dimension != new_dimension + ): + logger.warning( + "%s 向量维度变化: %s -> %s", + collection_name, + current_dimension, + new_dimension, + ) + if dry_run: + logger.info( + "[dry-run] 实际执行时会重建 collection %s 后写入新维度向量", + collection_name, + ) + else: + if client is None: + raise RuntimeError( + "检测到向量维度变化，但缺少 ChromaDB client，无法重建 collection" + ) + collection = _recreate_collection( + client, collection_name, collection.metadata + ) + if not dry_run: # upsert 覆写：ID 不变，document 和 metadata 不变，仅更新 embedding collection.upsert( @@ -239,7 +311,12 @@ async def _main(args: argparse.Namespace) -> None: "cognitive_events", metadata={"hnsw:space": "cosine"} ) total_processed += await _reembed_collection( - events_col, "cognitive_events", embedder, args.batch_size, args.dry_run + events_col, + "cognitive_events", + embedder, + args.batch_size, + args.dry_run, + client=client, ) if not args.events_only: @@ -252,6 +329,7 @@ async def _main(args: argparse.Namespace) -> None: embedder, args.batch_size, args.dry_run, + client=client, ) finally: await embedder.stop() diff --git a/tests/test_reembed_cognitive_script.py b/tests/test_reembed_cognitive_script.py new file mode 100644 index 00000000..0f7931e0 --- /dev/null +++ b/tests/test_reembed_cognitive_script.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import ModuleType +from typing import Any, cast + +import chromadb +import pytest + + +def _load_script_module() -> ModuleType: + script_path = ( + Path(__file__).resolve().parent.parent / "scripts" / "reembed_cognitive.py" + ) + spec = importlib.util.spec_from_file_location( + "reembed_cognitive_script", script_path + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _FixedDimEmbedder: + """返回固定维度的假嵌入器。""" + + def __init__(self, dimension: int) -> None: + self._dimension = dimension + self.calls: list[list[str]] = [] + + async def embed(self, texts: list[str]) -> list[list[float]]: + self.calls.append(list(texts)) + return [[0.1] * self._dimension for _ in texts] + + +def _seed_collection(client: Any, name: str, dimension: int, count: int = 3) -> Any: + collection = client.get_or_create_collection( + name, metadata={"hnsw:space": "cosine"} + ) + collection.add( + ids=[f"id-{i}" for i in range(count)], + documents=[f"doc-{i}" for i in range(count)], + embeddings=[[0.2] * dimension for i in range(count)], + metadatas=[{"idx": i} for i in range(count)], + ) + return collection + + +@pytest.mark.asyncio +async def test_reembed_rebuilds_collection_on_dimension_change( + tmp_path: Path, +) -> None: + module = _load_script_module() + client = chromadb.PersistentClient(path=str(tmp_path)) + collection = _seed_collection(client, "cognitive_events", dimension=3) + + processed = await module._reembed_collection( + collection, + "cognitive_events", + _FixedDimEmbedder(5), + batch_size=2, + dry_run=False, + client=client, + ) + + assert processed == 3 + rebuilt = client.get_or_create_collection("cognitive_events") + assert rebuilt.count() == 3 + sample = cast( + Any, + rebuilt.get(limit=1, include=["embeddings", "documents", "metadatas"]), + ) + assert len(sample["embeddings"][0]) == 5 + assert sample["documents"][0].startswith("doc-") + assert sample["metadatas"][0]["idx"] in {0, 1, 2} + + +@pytest.mark.asyncio +async def test_reembed_keeps_collection_when_dimension_matches( + tmp_path: Path, +) -> None: + module = _load_script_module() + client = chromadb.PersistentClient(path=str(tmp_path)) + collection = _seed_collection(client, "cognitive_profiles", dimension=4) + + await module._reembed_collection( + collection, + "cognitive_profiles", + _FixedDimEmbedder(4), + batch_size=4, + dry_run=False, + client=client, + ) + + stored = client.get_collection("cognitive_profiles") + assert stored.count() == 3 + stored_sample = cast(Any, stored.get(limit=1, include=["embeddings"])) + assert len(stored_sample["embeddings"][0]) == 4 + + +@pytest.mark.asyncio +async def test_reembed_dry_run_reports_dimension_change_without_writing( + tmp_path: Path, +) -> None: + module = _load_script_module() + client = chromadb.PersistentClient(path=str(tmp_path)) + collection = _seed_collection(client, "cognitive_events", dimension=3) + + await module._reembed_collection( + collection, + "cognitive_events", + _FixedDimEmbedder(7), + batch_size=2, + dry_run=True, + client=client, + ) + + # 原 collection 未被删除，维度保持 3 + unchanged = client.get_collection("cognitive_events") + assert unchanged.count() == 3 + unchanged_sample = cast(Any, unchanged.get(limit=1, include=["embeddings"])) + assert len(unchanged_sample["embeddings"][0]) == 3 + + +@pytest.mark.asyncio +async def test_reembed_requires_client_when_dimension_changes( + tmp_path: Path, +) -> None: + module = _load_script_module() + client = chromadb.PersistentClient(path=str(tmp_path)) + collection = _seed_collection(client, "cognitive_events", dimension=3) + + with pytest.raises(RuntimeError, match="缺少 ChromaDB client"): + await module._reembed_collection( + collection, + "cognitive_events", + _FixedDimEmbedder(9), + batch_size=2, + dry_run=False, + ) + + +@pytest.mark.asyncio +async def test_reembed_empty_collection_is_skipped(tmp_path: Path) -> None: + module = _load_script_module() + client = chromadb.PersistentClient(path=str(tmp_path)) + collection = client.get_or_create_collection( + "cognitive_events", metadata={"hnsw:space": "cosine"} + ) + embedder = _FixedDimEmbedder(3) + + processed = await module._reembed_collection( + collection, + "cognitive_events", + embedder, + batch_size=2, + dry_run=False, + client=client, + ) + + assert processed == 0 + assert embedder.calls == [] From ba2674dbdfa52f9db0a7020fa6d407f6190d8b3b Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 15:58:29 +0800 Subject: [PATCH 11/30] =?UTF-8?q?docs:=20=E4=BF=AE=E5=A4=8D=E9=9A=8F?= =?UTF-8?q?=E5=8C=85=E5=8C=96=E9=87=8D=E6=9E=84=E5=90=8E=E5=A4=B1=E6=95=88?= =?UTF-8?q?=E7=9A=84=E8=B7=AF=E5=BE=84=E4=B8=8E=E8=BD=A6=E9=81=93=E6=95=99?= =?UTF-8?q?=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CLAUDE.md 模块表改为真实包路径（ai/client/、ai/llm/、ai/prompts/、 services/coordinator/、cognitive/historian/、handlers/、onebot/、attachments/）， 消息流与多模型池分工里的同名旧路径一并修正； - ARCHITECTURE.md 去掉不存在的 '*_shim' 与门面说明，目录级事实来源指向 docs/development.md； - 「4 级优先级」统一改为 6 条车道（超管私聊 / 群聊超管 / 普通私聊 / 群聊@ / 群聊普通 / 后台请求），与 queue_manager 常量一致； - docs/message-batching.md 相关文件链接补 ../；docs/development.md 更新 services/commands 与 command.py 的职责（死 mixins 已删除）； - 文档互链自检：docs/ 与三个根文档相对链接全部可解析。 --- AGENTS.md | 4 +++- ARCHITECTURE.md | 24 ++++++++++++------------ CLAUDE.md | 22 +++++++++++----------- docs/development.md | 4 ++-- docs/message-batching.md | 14 +++++++------- 5 files changed, 35 insertions(+), 33 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 599e6826..8ee8ab30 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,9 @@ # Repository Guidelines ## Project Structure & Module Organization -`src/Undefined/` contains the main runtime package. Core areas include `ai/`, `services/`, `skills/`, `cognitive/`, `memes/`, `knowledge/`, `api/`, `webui/`, `config/`, `mcp/`, and `automations/` (`AutomationService` runtime plus JSON storage); media-facing integrations live in `arxiv/`, `bilibili/`, `github/`, and `attachments.py`. `tests/` holds the pytest suite. `apps/undefined-console/` is the Tauri + Vite management client and `apps/undefined-chat/` is the native-first Tauri + React 19 chat client (both connect to the same Management/Runtime services), while `code/NagaAgent/` remains a git submodule and should be updated deliberately, with upstream syncs kept separate from repo-local changes. Runtime and generated state primarily lives under `data/`, `logs/`, and `dist/`; the root `knowledge/` directory stores knowledge-base data rather than application code. Prefer editing source files and docs over generated outputs unless the task is explicitly about runtime state. +`src/Undefined/` contains the main runtime package. Core areas include `ai/`, `services/`, `skills/`, `cognitive/`, `memes/`, `knowledge/`, `api/`, `webui/`, `config/`, `mcp/`, and `automations/` (`AutomationService` runtime plus JSON storage); media-facing integrations live in `arxiv/`, `bilibili/`, `github/`, and `attachments/`. `tests/` holds the pytest suite. + +> Single source of truth for the module-level directory tree: [docs/development.md](docs/development.md). This file, `CLAUDE.md`, and `ARCHITECTURE.md` only keep overviews — update the tree first when the layout changes. `apps/undefined-console/` is the Tauri + Vite management client and `apps/undefined-chat/` is the native-first Tauri + React 19 chat client (both connect to the same Management/Runtime services), while `code/NagaAgent/` remains a git submodule and should be updated deliberately, with upstream syncs kept separate from repo-local changes. Runtime and generated state primarily lives under `data/`, `logs/`, and `dist/`; the root `knowledge/` directory stores knowledge-base data rather than application code. Prefer editing source files and docs over generated outputs unless the task is explicitly about runtime state. ## Build, Test, and Development Commands Use `uv` for the root project: diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7a0bc9a9..fa67f807 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -18,7 +18,7 @@ graph TB ConfigLoader["ConfigManager
配置管理器
[config/manager.py + loader.py]"] ConfigHotReload["ConfigHotReload
热更新应用器
[config/hot_reload.py]"] ConfigModels["配置模型
[config/models.py]
ChatModelConfig
VisionModelConfig
SecurityModelConfig
AgentModelConfig"] - OneBotClient["OneBotClient
WebSocket 客户端
[onebot/ + onebot.py shim]"] + OneBotClient["OneBotClient
WebSocket 客户端
[onebot/]"] Context["RequestContext
请求上下文
[context.py]"] WebUI["webui.py
配置控制台
[src/Undefined/webui.py]"] end @@ -47,10 +47,10 @@ graph TB CommandDispatcher["CommandDispatcher
命令分发器
• /help /stats /admin
• /bugfix /faq
[services/command.py]"] - MessageBatcher["MessageBatcher
同 sender 短时合并
• 按 (scope, sender_id) 分桶
• T1=window_seconds 结束 batch
• T2=pre_send_seconds 投机预发送
• 拍一拍/buffer 内 @bot 旁路
• 首条 @bot 整批走 mention 队列
[services/message_batcher/ + shim]"] + MessageBatcher["MessageBatcher
同 sender 短时合并
• 按 (scope, sender_id) 分桶
• T1=window_seconds 结束 batch
• T2=pre_send_seconds 投机预发送
• 拍一拍/buffer 内 @bot 旁路
• 首条 @bot 整批走 mention 队列
[services/message_batcher/]"] subgraph QueueSystem["车站-列车 队列系统 (services/)"] - AICoordinator["AICoordinator
AI 协调器
• Prompt 构建
• 队列管理
• 回复执行
[services/coordinator/ + ai_coordinator.py shim]"] + AICoordinator["AICoordinator
AI 协调器
• Prompt 构建
• 队列管理
• 回复执行
[services/coordinator/]"] QueueManager["QueueManager
队列管理器
[queue_manager.py]"] subgraph ModelQueues["ModelQueue 队列组 (按模型隔离)"] @@ -66,14 +66,14 @@ graph TB %% ==================== AI 核心能力层 ==================== subgraph AILayer["AI 核心能力层 (src/Undefined/ai/)"] - AIClient["AIClient
AI 客户端主入口
[ai/client/ + client.py shim]
• 技能热重载 • MCP 初始化
• Agent intro 生成"] + AIClient["AIClient
AI 客户端主入口
[ai/client/]
• 技能热重载 • MCP 初始化
• Agent intro 生成"] subgraph AIComponents["AI 组件"] - PromptBuilder["PromptBuilder
提示词构建器
[ai/prompts/ + prompts.py shim]"] + PromptBuilder["PromptBuilder
提示词构建器
[ai/prompts/]"] ToolSearchSession["ToolSearchSession
请求级工具按需投影
[ai/tool_search.py]
• 名称检索 • schema 逐轮扩展"] - ModelRequester["ModelRequester
模型请求器
[ai/llm/ + llm.py shim]
• OpenAI SDK • 工具清理
• Thinking 提取"] + ModelRequester["ModelRequester
模型请求器
[ai/llm/]
• OpenAI SDK • 工具清理
• Thinking 提取"] ToolManager["ToolManager
工具管理器
[tooling.py]
• 工具执行 • Agent 工具合并
• MCP 工具注入"] - MultimodalAnalyzer["MultimodalAnalyzer
多模态分析器
[ai/multimodal/ + multimodal.py shim]
• 图片/音频/视频"] + MultimodalAnalyzer["MultimodalAnalyzer
多模态分析器
[ai/multimodal/]
• 图片/音频/视频"] SummaryService["SummaryService
总结服务
[summaries.py]
• 聊天记录总结
• 标题生成"] TokenCounter["TokenCounter
Token 统计
[tokens.py]"] Parsing["Parsing
响应解析
[parsing.py]"] @@ -663,7 +663,7 @@ graph TB subgraph Features["特性"] F1["非阻塞: 即使前一个请求未完成,
新请求也会按时分发"] - F2["优先级: 四级优先级,
确保重要消息优先响应"] + F2["优先级: 六条车道,
确保重要消息优先响应"] F3["隔离性: 每个模型独立队列,
互不干扰"] F4["自动修剪: 普通队列超过10条时,
只保留最新2条"] F5["可配置节奏: 每个模型可独立设置
队列发车间隔"] @@ -860,10 +860,10 @@ description: 从 PDF 文件中提取文本和表格，填写表单。当用户 ### 8层架构分层 1. **外部实体层**：用户、管理员、OneBot 协议端 (NapCat/Lagrange.Core)、大模型 API 服务商 -2. **核心入口层**：main.py 启动入口、配置管理器 (config/loader.py + parsers/ + load_sections/)、热更新应用器 (config/hot_reload.py)、OneBotClient (onebot/ + onebot.py shim)、WeixinService (`weixin/` + `weixin-ilink-client`)、RequestContext (context.py)、Runtime API Server (api/app.py → api/routes/ 路由子模块，含 naga/ 子包) -3. **消息处理层**：MessageHandler (`handlers/`)、统一 DeliveryAddress 路由 (`utils/message_targets.py`)、SecurityService (security.py)、CommandDispatcher (services/command.py + commands/ mixins)、自动处理管线 (skills/pipelines/)、AutomationService (`automations/service.py`，pipeline 之后、对应 AI loop 之前 await，命中可拦截)、MessageBatcher (services/message_batcher/)、AICoordinator (services/coordinator/ + ai_coordinator.py 门面)、QueueManager (queue_manager.py)、Bilibili/arXiv/GitHub 解析与发送模块 +2. **核心入口层**（目录级事实来源见 [docs/development.md](docs/development.md)）：main.py 启动入口、配置管理器 (config/loader.py + parsers/ + load_sections/)、热更新应用器 (config/hot_reload.py)、OneBotClient (onebot/)、WeixinService (`weixin/` + `weixin-ilink-client`)、RequestContext (context.py)、Runtime API Server (api/app.py → api/routes/ 路由子模块，含 naga/ 子包) +3. **消息处理层**：MessageHandler (`handlers/`)、统一 DeliveryAddress 路由 (`utils/message_targets.py`)、SecurityService (security.py)、CommandDispatcher (services/command.py + services/commands/)、自动处理管线 (skills/pipelines/)、AutomationService (`automations/service.py`，pipeline 之后、对应 AI loop 之前 await，命中可拦截)、MessageBatcher (services/message_batcher/)、AICoordinator (services/coordinator/)、QueueManager (queue_manager.py)、Bilibili/arXiv/GitHub 解析与发送模块 自动提取由 `PipelineRegistry` 并行检测、并行处理全部命中的管线；随后 `await` 自动化工作流，未拦截时再进入 AI 自动回复。 -4. **AI 核心能力层**：AIClient (ai/client/ + client.py shim)、PromptBuilder (ai/prompts/ + prompts.py shim)、ModelRequester (ai/llm/ + llm.py shim)、ToolManager (tooling.py)、MultimodalAnalyzer (ai/multimodal/ + multimodal.py shim)、SummaryService (summaries.py)、TokenCounter (tokens.py)。OpenAI Chat Completions / Responses、Anthropic Messages SDK 归一化、CoT 续传与文本 Tool Call 容错见[模型 API 与兼容层](docs/model-compatibility.md)。 +4. **AI 核心能力层**：AIClient (ai/client/)、PromptBuilder (ai/prompts/)、ModelRequester (ai/llm/)、ToolManager (tooling.py)、MultimodalAnalyzer (ai/multimodal/)、SummaryService (summaries.py)、TokenCounter (tokens.py)。OpenAI Chat Completions / Responses、Anthropic Messages SDK 归一化、CoT 续传与文本 Tool Call 容错见[模型 API 与兼容层](docs/model-compatibility.md)。 5. **存储与上下文层**：MessageHistoryManager (utils/history.py, 10000条限制)、MemoryStorage (memory.py, 置顶备忘录, 500条上限)、EndSummaryStorage、CognitiveService + JobQueue + HistorianWorker + VectorStore + ProfileStorage、MemeService + MemeWorker + MemeStore + MemeVectorStore (表情包库)、FAQStorage、AutomationStorage (`data/automations.json`；旧 `scheduled_tasks.json` 启动时一次性转为新格式，不删旧文件、不双写)、TokenUsageStorage (自动归档) 6. **技能系统层**：ToolRegistry (registry.py)、AgentRegistry、7个 Agents、13类 Toolsets 7. **异步 IO 层**：统一 IO 工具 (utils/io.py)，包含 write_json、read_json、append_line、跨平台文件锁 (flock/msvcrt) @@ -880,7 +880,7 @@ description: 从 PDF 文件中提取文本和表格，填写表单。当用户 * **多模型隔离**：每个 AI 模型拥有独立的请求队列组（"站台"），互不干扰。 * **非阻塞发车**：实现了可配置节奏的非阻塞调度循环（默认 **1Hz**）。列车按节奏出发，带走一个请求到后台异步处理。 * **高可用性**：即使前一个请求仍在处理（如耗时的网络搜索），新的请求也会按时被分发，不会造成队列堵塞。 -* **优先级管理**：支持四级优先级（超级管理员 > 私聊 > 群聊@ > 群聊普通），确保重要消息优先响应。 +* **优先级管理**：支持六条车道（超级管理员私聊 > 群聊超级管理员 > 普通私聊 > 群聊@ > 群聊普通 > 后台请求），前两条为严格优先级、中间三条轮转发车，确保重要消息优先响应。 * **关停收敛**：`MessageHandler.close()` 会先 flush `MessageBatcher`，再调用 `QueueManager.drain()` 等待已入队请求和在途请求自然完成，最后才停止队列处理器，避免缓冲消息只入队未执行。 ### 7个智能体 Agent diff --git a/CLAUDE.md b/CLAUDE.md index cfc8d2d9..8f55da39 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,14 +57,14 @@ bash scripts/install_git_hooks.sh ## 架构分层 -核心源码位于 `src/Undefined/`，主要模块如下： +核心源码位于 `src/Undefined/`，主要模块如下（目录级事实来源是 [docs/development.md](docs/development.md) 的目录树，本表只做职责概览）： | 目录 / 文件 | 职责 | |---|---| -| `ai/` | AI 运行时核心：`client.py`(主入口)、`llm.py`(模型请求)、`prompts.py`(Prompt 构建)、`tooling.py`(工具管理)、`multimodal.py`(多模态)、`model_selector.py`(模型选择)、`summaries.py`(短期总结) | -| `services/` | 运行服务：`ai_coordinator.py`(协调器+队列投递)、`queue_manager.py`(车站-列车队列)、`message_batcher.py`(同 sender 短时合并)、`command.py`(命令分发)、`model_pool.py`(多模型池)、`security.py`(安全防护) | +| `ai/` | AI 运行时核心：`client/`(主入口与 ask 循环)、`llm/`(模型请求与重试)、`prompts/`(Prompt 构建)、`tooling.py`(工具管理)、`multimodal/`(多模态)、`model_selector.py`(模型选择)、`summaries.py`(短期总结)、`retrieval.py`(嵌入/重排请求) | +| `services/` | 运行服务：`coordinator/`(协调器 + 队列投递)、`queue_manager.py`(车站-列车队列)、`message_batcher/`(同 sender 短时合并)、`command.py`(命令分发)、`commands/`(命令注册表与目录)、`model_pool.py`(多模型池)、`security.py`(安全防护) | | `skills/` | 热重载技能系统：`tools/`(原子工具)、`toolsets/`(按域分组工具)、`agents/`(智能体)、`commands/`(斜杠指令)、`anthropic_skills/`(SKILL.md 知识注入) | -| `cognitive/` | 认知记忆：`service.py`(入口)、`vector_store.py`(ChromaDB)、`historian.py`(后台史官异步改写+侧写合并)、`job_queue.py`、`profile_storage.py` | +| `cognitive/` | 认知记忆：`service/`(入口)、`vector_store.py`(ChromaDB)、`historian/`(后台史官异步改写+侧写合并)、`job_queue.py`、`profile_storage.py` | | `memes/` | 表情包库：两阶段 AI 管线、异步处理队列、SQLite 元数据、ChromaDB 向量检索 | | `knowledge/` | 本地知识库：文本切分、嵌入、重排、ChromaDB 存储与运行时检索 | | `arxiv/` | arXiv 论文解析、元信息获取、PDF 下载与发送 | @@ -75,20 +75,20 @@ bash scripts/install_git_hooks.sh | `mcp/` | MCP 工具注册、连接与转换 | | `automations/` | 条件驱动的轻量工作流：`AutomationService` 运行时、start 匹配、@ 消费、DAG / 分支 / 循环、旧定时任务迁移 | | `config/` | 配置系统：`loader.py`(TOML 解析+类型化)、`models.py`(数据模型)、`hot_reload.py`(热更新) | -| `attachments.py` | 富媒体/附件注册、作用域隔离、`` 统一标签（`` 向后兼容）渲染 | +| `attachments/` | 富媒体/附件注册、作用域隔离、`` 统一标签（`` 向后兼容）渲染 | | `utils/` | `io.py`(异步 IO)、`history.py`(消息历史)、`paths.py`、`logging.py`、`sender.py` 等通用能力 | ### 消息处理流程 ```text -OneBot WebSocket → onebot.py → handlers.py +OneBot WebSocket → onebot/ → handlers/ → 附件登记 / 访问控制 / 表情包入库 → SecurityService(注入检测) → CommandDispatcher(斜杠指令，命中即结束后续处理) → skills/pipelines(Bilibili / arXiv / GitHub 并行自动提取) → Automations(pipeline 后接入；consume_ai_loop 时 await 并拦截对应 AI，否则后台执行并立刻放行；发生在 MessageBatcher 之前) → MessageBatcher(同 sender 短时合并；拍一拍/buffer 内 @bot 旁路) - → AICoordinator → QueueManager(按模型隔离, 4 级优先级) + → AICoordinator → QueueManager(按模型隔离, 6 条车道) → AIClient → LLM API / Skills / MCP Management / Runtime 请求 → webui/app.py 或 api/app.py → routes/* @@ -99,9 +99,9 @@ Management / Runtime 请求 → webui/app.py 或 api/app.py → routes/* - `ai/model_selector.py` — 纯选择逻辑（策略 / 偏好 / compare 状态），无 IO 副作用 - `services/model_pool.py` — 私聊交互服务，持有 ai/config/sender，处理 `/compare`、`选X`、`select_chat_config` -- `services/ai_coordinator.py` — 持有 `ModelPoolService`（`self.model_pool`），私聊队列投递时通过它选模型 -- `handlers.py` — 私聊消息只调 `await self.ai_coordinator.model_pool.handle_private_message(user_id, text)`，不直接感知选择细节 -- `skills/agents/runner.py` — Agent 直接调用 `ai_client.model_selector.select_agent_config(...)`，无 `hasattr` +- `services/coordinator/` — 持有 `ModelPoolService`（`self.model_pool`），私聊队列投递时通过它选模型 +- `handlers/` — 私聊消息只调 `await self.ai_coordinator.model_pool.handle_private_message(user_id, text)`，不直接感知选择细节 +- `skills/agents/runner/` — Agent 直接调用 `ai_client.model_selector.select_agent_config(...)`，无 `hasattr` - 默认关闭：`models.pool_enabled = false`；群聊不参与多模型，始终走主模型 ### Skills 系统 @@ -122,7 +122,7 @@ Management / Runtime 请求 → webui/app.py 或 api/app.py → routes/* ### 队列模型 -车站-列车模型（QueueManager）：按模型隔离队列组，4 级优先级（超管 > 私聊 > @提及 > 普通群聊），普通队列自动修剪保留最新 2 条，非阻塞按节奏发车（默认 1Hz）。 +车站-列车模型（QueueManager）：按模型隔离队列组，6 条车道按序发车（超管私聊 / 群聊超管 / 普通私聊 / 群聊@ / 群聊普通 / 后台请求；前两条为严格优先级，中间三条轮转发车），普通队列自动修剪保留最新 2 条，非阻塞按节奏发车（默认 1Hz）。 ### 同 sender 短时消息合并（MessageBatcher） diff --git a/docs/development.md b/docs/development.md index 222cfdfc..77544b72 100644 --- a/docs/development.md +++ b/docs/development.md @@ -38,9 +38,9 @@ src/Undefined/ ├── memes/ # 表情包库 (service + ingest/ + search/ + store + vector_store) ├── services/ # 核心运行服务 │ ├── coordinator/ # AICoordinator 唯一实现（群聊 / 私聊 / 批处理 / 后台任务 mixins） -│ ├── commands/ # CommandDispatcher mixins（stats / bugfix） +│ ├── commands/ # 命令注册表、命令元数据与目录（catalog / registry / context） │ ├── message_batcher/ # 同 sender 短时合并 -│ ├── command.py # 命令分发门面 + shim 组合 +│ ├── command.py # 命令分发主体（斜杠指令解析、权限与限流） │ ├── queue_manager.py # 车站-列车队列 │ └── security.py # 注入检测与速率限制 ├── utils/ # 通用支持工具组 (__init__.py 聚合 io/paths/resources；io.py 异步原子读写, history.py, coerce.py 类型强转) diff --git a/docs/message-batching.md b/docs/message-batching.md index 6c23305a..3393e245 100644 --- a/docs/message-batching.md +++ b/docs/message-batching.md @@ -97,13 +97,13 @@ allow_cancel_after_send = false ## 相关文件 -- 实现：[src/Undefined/services/message_batcher/](src/Undefined/services/message_batcher/) -- 接入：[src/Undefined/services/coordinator/](src/Undefined/services/coordinator/) 中 `handle_auto_reply` / `handle_private_reply` / `_dispatch_grouped_request` -- 创建/注入：[src/Undefined/handlers/message_flow.py](src/Undefined/handlers/message_flow.py) -- 关停 flush：[src/Undefined/main.py](src/Undefined/main.py) -- 热更新：[src/Undefined/config/hot_reload.py](src/Undefined/config/hot_reload.py) -- 提示词：[res/prompts/undefined.xml](res/prompts/undefined.xml)、[res/prompts/undefined_nagaagent.xml](res/prompts/undefined_nagaagent.xml) -- 测试：[tests/test_message_batcher.py](tests/test_message_batcher.py) +- 实现：[src/Undefined/services/message_batcher/](../src/Undefined/services/message_batcher/) +- 接入：[src/Undefined/services/coordinator/](../src/Undefined/services/coordinator/) 中 `handle_auto_reply` / `handle_private_reply` / `_dispatch_grouped_request` +- 创建/注入：[src/Undefined/handlers/message_flow.py](../src/Undefined/handlers/message_flow.py) +- 关停 flush：[src/Undefined/main.py](../src/Undefined/main.py) +- 热更新：[src/Undefined/config/hot_reload.py](../src/Undefined/config/hot_reload.py) +- 提示词：[res/prompts/undefined.xml](../res/prompts/undefined.xml)、[res/prompts/undefined_nagaagent.xml](../res/prompts/undefined_nagaagent.xml) +- 测试：[tests/test_message_batcher.py](../tests/test_message_batcher.py) ## 投机预发送（Speculative Pre-fire） From 6daa0fb0587afd79d8fb1b90a27a404a6ba69107 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 16:00:07 +0800 Subject: [PATCH 12/30] =?UTF-8?q?chore(biome):=20App=20=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=98=BE=E5=BC=8F=E4=B8=8D=E7=BB=A7=E6=89=BF?= =?UTF-8?q?=E6=A0=B9=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 两个 App 的 biome.json 由 root:false（实际仍参与父配置合并链）改为 extends: []，并各自声明 formatter.indentStyle = tab，App 格式与规则完全 由本目录决定，根目录 WebUI 规则不再渗入； - 文档同步：不再声称靠 files.includes 隐式隔离，明确 Console 的 lint:webui 通过 --config-path 显式使用根配置； - 实测：WebUI（17 文件）、Console（6 文件）、Chat（92 文件）三处检查均通过。 --- AGENTS.md | 2 +- apps/undefined-chat/biome.json | 5 ++++- apps/undefined-console/biome.json | 5 ++++- docs/build.md | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8ee8ab30..9e8718e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ For the console app, run `cd apps/undefined-console && npm ci && npm run check`. For the native chat client, run `cd apps/undefined-chat && npm ci && npm run check` (Biome + TypeScript + Vitest unit/e2e + `cargo fmt --check`/`cargo check`/`cargo test`). Use `npm run tauri:dev` for the desktop shell. ## Coding Style & Naming Conventions -Use 4-space indentation. Python code must be fully type-annotated and pass strict mypy checks. Disk I/O should go through `src/Undefined/utils/io.py` so writes stay async-safe and atomic. Follow `snake_case` for modules and functions, `PascalCase` for classes, and prefer extending existing services/helpers over introducing one-off abstractions. Skills handlers must not import repo-local modules outside `skills/`; pass dependencies through the execution context instead. WebUI JavaScript in `src/Undefined/webui/static/js/` and both native apps use Biome 2.5.10; keep the two package pins, lockfiles, and three configuration schemas in sync. Nested app configurations use `root: false` without inheriting the WebUI rules. App changes must satisfy Biome, TypeScript, and Cargo checks. +Use 4-space indentation. Python code must be fully type-annotated and pass strict mypy checks. Disk I/O should go through `src/Undefined/utils/io.py` so writes stay async-safe and atomic. Follow `snake_case` for modules and functions, `PascalCase` for classes, and prefer extending existing services/helpers over introducing one-off abstractions. Skills handlers must not import repo-local modules outside `skills/`; pass dependencies through the execution context instead. WebUI JavaScript in `src/Undefined/webui/static/js/` and both native apps use Biome 2.5.10; keep the two package pins, lockfiles, and three configuration schemas in sync. Nested app configurations use `extends: []` (explicit no-inherit) plus their own `formatter.indentStyle = "tab"`, so the root WebUI rules never leak into app checks. App changes must satisfy Biome, TypeScript, and Cargo checks. ## Tools & Features diff --git a/apps/undefined-chat/biome.json b/apps/undefined-chat/biome.json index 3ccb749e..962f7d50 100644 --- a/apps/undefined-chat/biome.json +++ b/apps/undefined-chat/biome.json @@ -1,6 +1,9 @@ { - "root": false, "$schema": "https://biomejs.dev/schemas/2.5.10/schema.json", + "extends": [], + "formatter": { + "indentStyle": "tab" + }, "files": { "includes": [ "**/src/**/*.ts", diff --git a/apps/undefined-console/biome.json b/apps/undefined-console/biome.json index 9e002ea2..9a04110b 100644 --- a/apps/undefined-console/biome.json +++ b/apps/undefined-console/biome.json @@ -1,6 +1,6 @@ { - "root": false, "$schema": "https://biomejs.dev/schemas/2.5.10/schema.json", + "extends": [], "files": { "includes": [ "**/src/**/*.ts", @@ -17,5 +17,8 @@ "noUnusedVariables": "warn" } } + }, + "formatter": { + "indentStyle": "tab" } } diff --git a/docs/build.md b/docs/build.md index e11e5abd..dd5c0860 100644 --- a/docs/build.md +++ b/docs/build.md @@ -266,7 +266,7 @@ Release workflow 会分别为 Console 和 Chat 构建 `arm64-v8a`、`armeabi-v7a WebUI、Console 和 Chat 统一使用 **Biome 2.5.10**。两个 App 的 `package.json` 固定该版本，锁文件与根目录及两个 App 的 `biome.json` schema 同步维护；使用 `npm ci` 安装锁定的工具版本。 -Biome v2 通过 `files.includes` 表达检查范围和排除规则。两个 App 配置设置 `root: false`，但不继承根目录的 WebUI 格式规则，保持各自原有的缩进风格和检查范围。后续升级应同时迁移三份配置并运行两个 App 的 `npm run check`，不能只修改 schema 版本号。迁移方式见 [Biome v2 官方指南](https://biomejs.dev/guides/upgrade-to-biome-v2/)。 +Biome v2 通过 `files.includes` 表达检查范围和排除规则。两个 App 的 `biome.json` 使用 `extends: []` **显式声明不继承根配置**，并各自声明 `formatter.indentStyle = "tab"`，因此 App 代码的格式与规则完全由本目录配置决定，根目录的 WebUI 规则（4 空格缩进、9 条关闭的 lint 规则）不会渗入 App 检查；反过来 WebUI 脚本由根配置的 `files.includes` 单独圈定，Console 的 `npm run lint:webui` 通过 `--config-path ../../biome.json` 显式使用根配置。后续升级应同时迁移三份配置并运行两个 App 的 `npm run check`，不能只修改 schema 版本号。迁移方式见 [Biome v2 官方指南](https://biomejs.dev/guides/upgrade-to-biome-v2/)。 仓库内已提供可版本化维护的 git hooks： From c826f9d0d8518318931ff9159b831a78b9adbfea Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 16:03:38 +0800 Subject: [PATCH 13/30] =?UTF-8?q?refactor(skills):=20=E6=8F=90=E4=BE=9B=20?= =?UTF-8?q?skills=20=E5=86=85=E5=85=B1=E4=BA=AB=E5=8A=A9=E6=89=8B=E5=B9=B6?= =?UTF-8?q?=E6=9C=BA=E6=A2=B0=E6=A0=A1=E9=AA=8C=E5=AF=BC=E5=85=A5=E8=BE=B9?= =?UTF-8?q?=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 src/Undefined/skills/shared.py：private_access_error、parse_positive_int； - 迁移 7 份 _private_access_error 与 5 份 _parse_positive_int 到共享实现， 消息文案统一（此前有的带“已被访问控制拦截”有的不带）； - 新增 tests/test_skills_import_boundary.py 棘轮：handler 只能依赖标准库、 第三方包、Undefined.skills.*、同目录相对导入与 context 注入；新增越界导入 直接失败，基线里已消失的条目也会失败（只减不增）； - skills/README.md 把“尽量不要”改成硬规则，并修正自身越界的官方示例 （改用 context 读取），CLAUDE.md / AGENTS.md 同步口径。 --- AGENTS.md | 2 +- CLAUDE.md | 2 +- src/Undefined/skills/README.md | 40 ++-- .../code_delivery_agent/tools/end/handler.py | 11 +- src/Undefined/skills/shared.py | 57 +++++ .../group/get_member_title/handler.py | 19 +- .../messages/react_message_emoji/handler.py | 49 ++--- .../toolsets/messages/send_message/handler.py | 17 +- .../toolsets/messages/send_poke/handler.py | 59 ++---- .../messages/send_private_message/handler.py | 17 +- .../messages/send_text_file/handler.py | 37 +--- .../messages/send_url_file/handler.py | 37 +--- tests/test_skills_import_boundary.py | 195 ++++++++++++++++++ 13 files changed, 327 insertions(+), 215 deletions(-) create mode 100644 src/Undefined/skills/shared.py create mode 100644 tests/test_skills_import_boundary.py diff --git a/AGENTS.md b/AGENTS.md index 9e8718e7..257cda0a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ For the console app, run `cd apps/undefined-console && npm ci && npm run check`. For the native chat client, run `cd apps/undefined-chat && npm ci && npm run check` (Biome + TypeScript + Vitest unit/e2e + `cargo fmt --check`/`cargo check`/`cargo test`). Use `npm run tauri:dev` for the desktop shell. ## Coding Style & Naming Conventions -Use 4-space indentation. Python code must be fully type-annotated and pass strict mypy checks. Disk I/O should go through `src/Undefined/utils/io.py` so writes stay async-safe and atomic. Follow `snake_case` for modules and functions, `PascalCase` for classes, and prefer extending existing services/helpers over introducing one-off abstractions. Skills handlers must not import repo-local modules outside `skills/`; pass dependencies through the execution context instead. WebUI JavaScript in `src/Undefined/webui/static/js/` and both native apps use Biome 2.5.10; keep the two package pins, lockfiles, and three configuration schemas in sync. Nested app configurations use `extends: []` (explicit no-inherit) plus their own `formatter.indentStyle = "tab"`, so the root WebUI rules never leak into app checks. App changes must satisfy Biome, TypeScript, and Cargo checks. +Use 4-space indentation. Python code must be fully type-annotated and pass strict mypy checks. Disk I/O should go through `src/Undefined/utils/io.py` so writes stay async-safe and atomic. Follow `snake_case` for modules and functions, `PascalCase` for classes, and prefer extending existing services/helpers over introducing one-off abstractions. Skills handlers must not import repo-local modules outside `skills/` (only `Undefined.skills.*` and same-directory relative imports are allowed); pass dependencies through the execution context, share helpers via `src/Undefined/skills/shared.py`, and keep `tests/test_skills_import_boundary.py`'s ratchet baseline shrinking. WebUI JavaScript in `src/Undefined/webui/static/js/` and both native apps use Biome 2.5.10; keep the two package pins, lockfiles, and three configuration schemas in sync. Nested app configurations use `extends: []` (explicit no-inherit) plus their own `formatter.indentStyle = "tab"`, so the root WebUI rules never leak into app checks. App changes must satisfy Biome, TypeScript, and Cargo checks. ## Tools & Features diff --git a/CLAUDE.md b/CLAUDE.md index 8f55da39..d3259d8a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -108,7 +108,7 @@ Management / Runtime 请求 → webui/app.py 或 api/app.py → routes/* - **热重载**：自动扫描 `skills/` 下 `config.json` / `handler.py` 变更并重载 - **自动处理管线**：`skills/pipelines//` 使用 `config.json + handler.py`，在斜杠命令之后、AI 自动回复之前并行检测/处理；命令输入和命令输出要写入历史，管线输出通过 `MessageSender` 自动写历史并登记本地媒体/文件附件 UID。 -- **Skills handler 不引用 `skills/` 外的本地模块**，依赖通过 context 注入 +- **Skills handler 不引用 `skills/` 外的本地模块**（`Undefined.skills.*` 与同目录相对导入除外），依赖通过 context 注入；跨技能共享的 helper 放 `skills/shared.py`，越界导入由 `tests/test_skills_import_boundary.py` 棘轮校验 - **Agent 标准结构**：`config.json` + `handler.py` + `prompt.md` + `intro.md` + `mcp.json`(可选) + `anthropic_skills/`(可选) - **共享授权**：通过 `callable.json` 将工具或 Agent 白名单暴露给其他 Agent - **Anthropic Skills**：支持 SKILL.md 目录结构与渐进式披露 diff --git a/src/Undefined/skills/README.md b/src/Undefined/skills/README.md index fcc9d37d..d4e05f41 100644 --- a/src/Undefined/skills/README.md +++ b/src/Undefined/skills/README.md @@ -246,32 +246,31 @@ tables = pdf.pages[0].extract_tables() 为了确保技能目录 (`skills/`) 的可移植性（例如直接移动到其他项目中使用），请遵循以下准则： -1. **避免外部依赖**: - - 尽量不要在 `handler.py` 中引用 `skills/` 目录之外的本地模块（如 `from Undefined.xxx import`）。 - - 如果是通用库（如 `httpx`, `pillow`），直接引用即可。 +1. **依赖边界（硬规则）**: + - `handler.py` 只允许依赖：Python 标准库、第三方包（如 `httpx`、`pillow`）、`skills/` 内部模块（`Undefined.skills.*` 或同目录相对导入），以及 `context` 注入的依赖。 + - **不允许**直接 `import` `skills/` 之外的仓库模块（如 `Undefined.services.*`、`Undefined.utils.*`、`Undefined.config`）；需要跨技能共享的 helper 放到 [`src/Undefined/skills/shared.py`](shared.py) 这类 skills 内模块。 + - 这条规则由 `tests/test_skills_import_boundary.py` 机械校验：新增越界导入会直接让测试失败。历史越界导入记录在该测试的基线里，重构时应顺带删除对应条目（棘轮只减不增）。 -2. **使用 RequestContext 获取请求信息**（推荐）: - - 使用 `RequestContext` 获取当前请求的 group_id、user_id 等信息，无需手动传递参数。 - - 这是获取请求上下文的首选方式，支持并发隔离。 +2. **从执行上下文获取请求信息**（推荐）: + - 运行时把 group_id、user_id、request_id 等放进 `context`，handler 直接读取即可，不要自己 import 仓库内部的上下文模块。 + - 若确实需要进程级请求隔离（并发下跨协程读取当前请求），使用 `context` 传入的客户端/服务对象，而不是引入全局状态。 ```python - from Undefined.context import get_group_id, get_user_id, get_request_id - async def execute(args, context): - # 优先从 args 获取（用户显式指定） - group_id = args.get("group_id") or get_group_id() - user_id = args.get("user_id") or get_user_id() - request_id = get_request_id() # 自动UUID追踪 - + # 优先从 args 获取（用户显式指定），否则回退到执行上下文 + group_id = args.get("group_id") or context.get("group_id") + user_id = args.get("user_id") or context.get("user_id") + request_id = context.get("request_id", "-") + if not group_id: return "无法确定群ID" - + # 使用 group_id 进行操作... ``` 3. **使用 Context 注入外部依赖**: - 如果需要使用外部项目的功能（如数据库连接、特殊的渲染函数），通过 `context` 参数传入。 -- 主程序（`handlers.py` 或 `ai/` 运行时）负责在调用时将这些依赖放入 `context`。 + - 主程序（`handlers/` 或 `ai/` 运行时）负责在调用时将这些依赖放入 `context`。 ```python # 错误的做法 @@ -288,15 +287,12 @@ tables = pdf.pages[0].extract_tables() await heavy_func() ``` -4. **向后兼容的获取方式**（仅在必要时使用）: - - 如果 `RequestContext` 不可用，可以回退到从 `context` 获取： - +4. **兼容旧写法的读取顺序**（仅在必要时使用）: + - 历史 handler 可能从多处取值，推荐优先级为：`args` > `context` > 旧字段（已废弃）。 + ```python - from Undefined.context import get_group_id - async def execute(args, context): - # 优先级：args > RequestContext > context > ai_client（已废弃） - group_id = args.get("group_id") or get_group_id() or context.get("group_id") + group_id = args.get("group_id") or context.get("group_id") ``` 5. **统一的加载机制**: diff --git a/src/Undefined/skills/agents/code_delivery_agent/tools/end/handler.py b/src/Undefined/skills/agents/code_delivery_agent/tools/end/handler.py index d279e423..465b35f1 100644 --- a/src/Undefined/skills/agents/code_delivery_agent/tools/end/handler.py +++ b/src/Undefined/skills/agents/code_delivery_agent/tools/end/handler.py @@ -8,6 +8,7 @@ from fnmatch import fnmatch from pathlib import Path from typing import Any +from Undefined.skills.shared import private_access_error logger = logging.getLogger(__name__) @@ -20,14 +21,6 @@ def _group_access_error(runtime_config: Any, group_id: int) -> str: return f"上传失败：目标群 {group_id} 不在允许列表内（access.allowed_group_ids）" -def _private_access_error(runtime_config: Any, user_id: int) -> str: - reason_getter = getattr(runtime_config, "private_access_denied_reason", None) - reason = reason_getter(user_id) if callable(reason_getter) else None - if reason == "blacklist": - return f"上传失败：目标用户 {user_id} 在黑名单内（access.blocked_private_ids）" - return f"上传失败：目标用户 {user_id} 不在允许列表内（access.allowed_private_ids）" - - def _should_exclude(rel_path: str, patterns: list[str]) -> bool: """检查路径是否匹配任一排除模式。""" for pattern in patterns: @@ -139,7 +132,7 @@ async def execute(args: dict[str, Any], context: dict[str, Any]) -> str: if target_type == "private" and not runtime_config.is_private_allowed( target_id ): - access_error = _private_access_error(runtime_config, target_id) + access_error = private_access_error(runtime_config, target_id) if access_error is not None: upload_status = access_error diff --git a/src/Undefined/skills/shared.py b/src/Undefined/skills/shared.py new file mode 100644 index 00000000..a2df55e0 --- /dev/null +++ b/src/Undefined/skills/shared.py @@ -0,0 +1,57 @@ +"""Skills 内部共享助手。 + +`skills/` 下的 handler 允许依赖本模块（以及同目录的相对导入），用于收敛那些 +在每个工具里各写一份的小函数。跨 `skills/` 的公共能力应优先放到这里，而不是 +在多个 handler 间复制实现。 +""" + +from __future__ import annotations + +from typing import Any + + +def private_access_error( + runtime_config: Any, + target_id: int, + *, + prefix: str = "发送失败：", +) -> str: + """按访问控制拒绝原因生成统一的用户可见说明。 + + 读取 `runtime_config.private_access_denied_reason(target_id)`： + - `blacklist` 表示命中 `access.blocked_private_ids`； + - 其余情况（含 `allowlist` / 未配置）统一提示不在允许列表内。 + """ + reason_getter = getattr(runtime_config, "private_access_denied_reason", None) + reason = reason_getter(target_id) if callable(reason_getter) else None + if reason == "blacklist": + return ( + f"{prefix}目标用户 {target_id} 在黑名单内（access.blocked_private_ids），" + "已被访问控制拦截" + ) + return ( + f"{prefix}目标用户 {target_id} 不在允许列表内（access.allowed_private_ids），" + "已被访问控制拦截" + ) + + +def parse_positive_int( + value: Any, + field_name: str, +) -> tuple[int | None, str | None]: + """解析可选正整数字段，返回 `(值, 错误说明)`。 + + `None` 表示未提供（返回 `(None, None)`）；非法值返回 `(None, 错误说明)`。 + """ + if value is None: + return None, None + try: + parsed = int(value) + except (TypeError, ValueError): + return None, f"{field_name} 必须是整数" + if parsed <= 0: + return None, f"{field_name} 必须是正整数" + return parsed, None + + +__all__ = ["parse_positive_int", "private_access_error"] diff --git a/src/Undefined/skills/toolsets/group/get_member_title/handler.py b/src/Undefined/skills/toolsets/group/get_member_title/handler.py index c2063fd1..657a6a0d 100644 --- a/src/Undefined/skills/toolsets/group/get_member_title/handler.py +++ b/src/Undefined/skills/toolsets/group/get_member_title/handler.py @@ -5,22 +5,11 @@ from typing import Any from Undefined.context import RequestContext +from Undefined.skills.shared import parse_positive_int logger = logging.getLogger(__name__) -def _parse_positive_int(value: Any, field_name: str) -> tuple[int | None, str | None]: - if value is None: - return None, None - try: - parsed = int(value) - except (TypeError, ValueError): - return None, f"{field_name} 必须是整数" - if parsed <= 0: - return None, f"{field_name} 必须是正整数" - return parsed, None - - def _snapshot_context(context: dict[str, Any]) -> dict[str, Any]: """冻结关键上下文，避免异步等待期间读取到漂移数据。""" ctx = RequestContext.current() @@ -66,7 +55,7 @@ def _resolve_group_id( if group_id_raw is None: return None, "请提供群号（group_id 参数），或者在群聊中调用" - group_id, group_err = _parse_positive_int(group_id_raw, "group_id") + group_id, group_err = parse_positive_int(group_id_raw, "group_id") if group_err or group_id is None: return None, group_err or "group_id 非法" return group_id, None @@ -79,8 +68,8 @@ def _resolve_user_id(args: dict[str, Any]) -> tuple[int | None, str | None]: if user_id_raw is None and qq_raw is None: return None, "请提供要查询的群成员 QQ 号（user_id 或 qq 参数）" - user_id, user_id_err = _parse_positive_int(user_id_raw, "user_id") - qq, qq_err = _parse_positive_int(qq_raw, "qq") + user_id, user_id_err = parse_positive_int(user_id_raw, "user_id") + qq, qq_err = parse_positive_int(qq_raw, "qq") if user_id_raw is not None and user_id_err: return None, user_id_err diff --git a/src/Undefined/skills/toolsets/messages/react_message_emoji/handler.py b/src/Undefined/skills/toolsets/messages/react_message_emoji/handler.py index 5d1bef0a..3447deda 100644 --- a/src/Undefined/skills/toolsets/messages/react_message_emoji/handler.py +++ b/src/Undefined/skills/toolsets/messages/react_message_emoji/handler.py @@ -8,6 +8,7 @@ from Undefined.context import RequestContext from Undefined.utils.qq_emoji import resolve_emoji_id_by_alias, search_emoji_aliases from Undefined.skills.toolsets.messages.context_utils import mark_message_sent +from Undefined.skills.shared import parse_positive_int, private_access_error logger = logging.getLogger(__name__) @@ -20,18 +21,6 @@ _MESSAGE_LOCKS_MAX = 1000 -def _parse_positive_int(value: Any, field_name: str) -> tuple[int | None, str | None]: - if value is None: - return None, None - try: - parsed = int(value) - except (TypeError, ValueError): - return None, f"{field_name} 必须是整数" - if parsed <= 0: - return None, f"{field_name} 必须是正整数" - return parsed, None - - def _parse_bool(value: Any, default: bool) -> bool: if value is None: return default @@ -89,13 +78,13 @@ def _resolve_action(args: Dict[str, Any]) -> tuple[ActionType | None, str | None def _resolve_message_id( args: Dict[str, Any], snapshot: dict[str, Any] ) -> tuple[int | None, str | None]: - direct_id, direct_error = _parse_positive_int(args.get("message_id"), "message_id") + direct_id, direct_error = parse_positive_int(args.get("message_id"), "message_id") if direct_error: return None, direct_error if direct_id is not None: return direct_id, None - trigger_id, trigger_error = _parse_positive_int( + trigger_id, trigger_error = parse_positive_int( snapshot.get("trigger_message_id"), "trigger_message_id" ) if trigger_error: @@ -107,7 +96,7 @@ def _resolve_message_id( def _resolve_emoji_id(args: Dict[str, Any]) -> tuple[int | None, str | None]: - emoji_id, emoji_id_error = _parse_positive_int(args.get("emoji_id"), "emoji_id") + emoji_id, emoji_id_error = parse_positive_int(args.get("emoji_id"), "emoji_id") if emoji_id_error: return None, emoji_id_error if emoji_id is not None: @@ -210,7 +199,7 @@ def _resolve_target_constraint( if normalized_type not in ("group", "private"): return None, "target_type 只能是 group 或 private" - parsed_target_id, target_id_error = _parse_positive_int(target_id_raw, "target_id") + parsed_target_id, target_id_error = parse_positive_int(target_id_raw, "target_id") if target_id_error or parsed_target_id is None: return None, target_id_error or "target_id 非法" @@ -224,20 +213,20 @@ def _extract_message_location( message_type_raw = str(message_detail.get("message_type", "")).strip().lower() if message_type_raw == "group": - group_id, _ = _parse_positive_int(message_detail.get("group_id"), "group_id") + group_id, _ = parse_positive_int(message_detail.get("group_id"), "group_id") if group_id is not None: return ("group", group_id) elif message_type_raw == "private": - user_id, _ = _parse_positive_int(message_detail.get("user_id"), "user_id") + user_id, _ = parse_positive_int(message_detail.get("user_id"), "user_id") if user_id is not None: return ("private", user_id) # 兜底推断 - group_id, _ = _parse_positive_int(message_detail.get("group_id"), "group_id") + group_id, _ = parse_positive_int(message_detail.get("group_id"), "group_id") if group_id is not None: return ("group", group_id) - user_id, _ = _parse_positive_int(message_detail.get("user_id"), "user_id") + user_id, _ = parse_positive_int(message_detail.get("user_id"), "user_id") if user_id is not None: return ("private", user_id) return None @@ -248,11 +237,11 @@ def _resolve_current_session_target( ) -> tuple[TargetType, int] | None: request_type = snapshot.get("request_type") if request_type == "group": - group_id, _ = _parse_positive_int(snapshot.get("group_id"), "group_id") + group_id, _ = parse_positive_int(snapshot.get("group_id"), "group_id") if group_id is not None: return ("group", group_id) if request_type == "private": - user_id, _ = _parse_positive_int(snapshot.get("user_id"), "user_id") + user_id, _ = parse_positive_int(snapshot.get("user_id"), "user_id") if user_id is not None: return ("private", user_id) return None @@ -272,20 +261,6 @@ def _group_access_error(runtime_config: Any, group_id: int) -> str: ) -def _private_access_error(runtime_config: Any, user_id: int) -> str: - reason_getter = getattr(runtime_config, "private_access_denied_reason", None) - reason = reason_getter(user_id) if callable(reason_getter) else None - if reason == "blacklist": - return ( - f"目标用户 {user_id} 在黑名单内（access.blocked_private_ids），" - "已被访问控制拦截" - ) - return ( - f"目标用户 {user_id} 不在允许列表内（access.allowed_private_ids），" - "已被访问控制拦截" - ) - - def _validate_target_and_allowlist( *, snapshot: dict[str, Any], @@ -324,7 +299,7 @@ def _validate_target_and_allowlist( if target_type == "group" and not runtime_config.is_group_allowed(target_id): return _group_access_error(runtime_config, target_id) if target_type == "private" and not runtime_config.is_private_allowed(target_id): - return _private_access_error(runtime_config, target_id) + return private_access_error(runtime_config, target_id) return None diff --git a/src/Undefined/skills/toolsets/messages/send_message/handler.py b/src/Undefined/skills/toolsets/messages/send_message/handler.py index d49200a2..e914325d 100644 --- a/src/Undefined/skills/toolsets/messages/send_message/handler.py +++ b/src/Undefined/skills/toolsets/messages/send_message/handler.py @@ -21,6 +21,7 @@ normalize_sent_message_id, parse_reply_to, ) +from Undefined.skills.shared import private_access_error logger = logging.getLogger(__name__) @@ -60,20 +61,6 @@ def _group_access_error(runtime_config: Any, target_id: int) -> str: ) -def _private_access_error(runtime_config: Any, target_id: int) -> str: - reason_getter = getattr(runtime_config, "private_access_denied_reason", None) - reason = reason_getter(target_id) if callable(reason_getter) else None - if reason == "blacklist": - return ( - f"发送失败：目标用户 {target_id} 在黑名单内（access.blocked_private_ids），" - "已被访问控制拦截" - ) - return ( - f"发送失败：目标用户 {target_id} 不在允许列表内（access.allowed_private_ids），" - "已被访问控制拦截" - ) - - def _format_send_success(message_id: Any) -> str: resolved_message_id = normalize_sent_message_id(message_id) if resolved_message_id is not None: @@ -143,7 +130,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if target_type == "private" and not runtime_config.is_private_allowed( target_id ): - return _private_access_error(runtime_config, target_id) + return private_access_error(runtime_config, target_id) if sender: try: diff --git a/src/Undefined/skills/toolsets/messages/send_poke/handler.py b/src/Undefined/skills/toolsets/messages/send_poke/handler.py index 9100c236..347e0504 100644 --- a/src/Undefined/skills/toolsets/messages/send_poke/handler.py +++ b/src/Undefined/skills/toolsets/messages/send_poke/handler.py @@ -6,24 +6,13 @@ from Undefined.context import RequestContext from Undefined.skills.toolsets.messages.context_utils import mark_message_sent +from Undefined.skills.shared import parse_positive_int, private_access_error logger = logging.getLogger(__name__) TargetType = Literal["group", "private"] -def _parse_positive_int(value: Any, field_name: str) -> tuple[int | None, str | None]: - if value is None: - return None, None - try: - parsed = int(value) - except (TypeError, ValueError): - return None, f"{field_name} 必须是整数" - if parsed <= 0: - return None, f"{field_name} 必须是正整数" - return parsed, None - - def _snapshot_context(context: Dict[str, Any]) -> dict[str, Any]: """冻结当前上下文关键字段，避免执行期间上下文变化导致竞态。""" ctx = RequestContext.current() @@ -79,7 +68,7 @@ def _resolve_send_target( ) if has_target_id: - target_id, target_error = _parse_positive_int(target_id_raw, "target_id") + target_id, target_error = parse_positive_int(target_id_raw, "target_id") if target_error or target_id is None: return None, target_error or "target_id 非法" return (normalized_target_type, target_id), None @@ -89,49 +78,47 @@ def _resolve_send_target( return None, "target_type 与当前会话类型不一致，无法推断 target_id" if normalized_target_type == "group": - group_id, group_error = _parse_positive_int( + group_id, group_error = parse_positive_int( snapshot.get("group_id"), "group_id" ) if group_error or group_id is None: return None, group_error or "无法根据 target_type 推断 target_id" return ("group", group_id), None - user_id, user_error = _parse_positive_int(snapshot.get("user_id"), "user_id") + user_id, user_error = parse_positive_int(snapshot.get("user_id"), "user_id") if user_error or user_id is None: return None, user_error or "无法根据 target_type 推断 target_id" return ("private", user_id), None legacy_group_id = args.get("group_id") if legacy_group_id is not None: - group_id, group_error = _parse_positive_int(legacy_group_id, "group_id") + group_id, group_error = parse_positive_int(legacy_group_id, "group_id") if group_error or group_id is None: return None, group_error or "group_id 非法" return ("group", group_id), None legacy_user_id = args.get("user_id") if legacy_user_id is not None: - user_id, user_error = _parse_positive_int(legacy_user_id, "user_id") + user_id, user_error = parse_positive_int(legacy_user_id, "user_id") if user_error or user_id is None: return None, user_error or "user_id 非法" return ("private", user_id), None request_type = snapshot.get("request_type") if request_type == "group": - group_id, group_error = _parse_positive_int( - snapshot.get("group_id"), "group_id" - ) + group_id, group_error = parse_positive_int(snapshot.get("group_id"), "group_id") if group_error: return None, group_error if group_id is not None: return ("group", group_id), None elif request_type == "private": - user_id, user_error = _parse_positive_int(snapshot.get("user_id"), "user_id") + user_id, user_error = parse_positive_int(snapshot.get("user_id"), "user_id") if user_error: return None, user_error if user_id is not None: return ("private", user_id), None - fallback_group_id, fallback_group_error = _parse_positive_int( + fallback_group_id, fallback_group_error = parse_positive_int( snapshot.get("group_id"), "group_id" ) if fallback_group_error: @@ -139,7 +126,7 @@ def _resolve_send_target( if fallback_group_id is not None: return ("group", fallback_group_id), None - fallback_user_id, fallback_user_error = _parse_positive_int( + fallback_user_id, fallback_user_error = parse_positive_int( snapshot.get("user_id"), "user_id" ) if fallback_user_error: @@ -158,7 +145,7 @@ def _resolve_target_user( target_user_raw = args.get("target_user_id") target_type, target_id = send_target if target_user_raw is not None: - target_user_id, target_user_error = _parse_positive_int( + target_user_id, target_user_error = parse_positive_int( target_user_raw, "target_user_id" ) if target_user_error or target_user_id is None: @@ -173,15 +160,13 @@ def _resolve_target_user( if target_type == "private": return target_id, None - sender_id, sender_error = _parse_positive_int( - snapshot.get("sender_id"), "sender_id" - ) + sender_id, sender_error = parse_positive_int(snapshot.get("sender_id"), "sender_id") if sender_error: return None, sender_error if sender_id is not None: return sender_id, None - fallback_user_id, fallback_user_error = _parse_positive_int( + fallback_user_id, fallback_user_error = parse_positive_int( snapshot.get("user_id"), "user_id" ) if fallback_user_error: @@ -237,20 +222,6 @@ def _group_access_error(runtime_config: Any, group_id: int) -> str: ) -def _private_access_error(runtime_config: Any, user_id: int) -> str: - reason_getter = getattr(runtime_config, "private_access_denied_reason", None) - reason = reason_getter(user_id) if callable(reason_getter) else None - if reason == "blacklist": - return ( - f"发送失败：目标用户 {user_id} 在黑名单内（access.blocked_private_ids），" - "已被访问控制拦截" - ) - return ( - f"发送失败：目标用户 {user_id} 不在允许列表内（access.allowed_private_ids），" - "已被访问控制拦截" - ) - - async def _record_poke_history_if_needed( context: Dict[str, Any], target_type: TargetType, @@ -320,7 +291,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if target_type == "private" and not runtime_config.is_private_allowed( target_user_id ): - return _private_access_error(runtime_config, target_user_id) + return private_access_error(runtime_config, target_user_id) sender = context.get("sender") if sender is not None: @@ -357,7 +328,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: ) if target_type == "group": return _group_access_error(runtime_config, target_id) - return _private_access_error(runtime_config, target_user_id) + return private_access_error(runtime_config, target_user_id) except Exception as e: logger.exception( "[拍一拍] sender 发送失败: request_id=%s target_type=%s target_id=%s user=%s err=%s", diff --git a/src/Undefined/skills/toolsets/messages/send_private_message/handler.py b/src/Undefined/skills/toolsets/messages/send_private_message/handler.py index d7d086e3..a094adf0 100644 --- a/src/Undefined/skills/toolsets/messages/send_private_message/handler.py +++ b/src/Undefined/skills/toolsets/messages/send_private_message/handler.py @@ -15,24 +15,11 @@ normalize_sent_message_id, parse_reply_to, ) +from Undefined.skills.shared import private_access_error logger = logging.getLogger(__name__) -def _private_access_error(runtime_config: Any, user_id: int) -> str: - reason_getter = getattr(runtime_config, "private_access_denied_reason", None) - reason = reason_getter(user_id) if callable(reason_getter) else None - if reason == "blacklist": - return ( - f"发送失败：目标用户 {user_id} 在黑名单内（access.blocked_private_ids），" - "已被访问控制拦截" - ) - return ( - f"发送失败：目标用户 {user_id} 不在允许列表内（access.allowed_private_ids），" - "已被访问控制拦截" - ) - - def _format_send_success(user_id: int, message_id: Any) -> str: resolved_message_id = normalize_sent_message_id(message_id) if resolved_message_id is not None: @@ -91,7 +78,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: runtime_config = context.get("runtime_config") if runtime_config is not None: if not runtime_config.is_private_allowed(user_id): - return _private_access_error(runtime_config, user_id) + return private_access_error(runtime_config, user_id) send_private_message_callback = context.get("send_private_message_callback") sender = context.get("sender") diff --git a/src/Undefined/skills/toolsets/messages/send_text_file/handler.py b/src/Undefined/skills/toolsets/messages/send_text_file/handler.py index 591b7ee8..fe76f016 100644 --- a/src/Undefined/skills/toolsets/messages/send_text_file/handler.py +++ b/src/Undefined/skills/toolsets/messages/send_text_file/handler.py @@ -15,6 +15,7 @@ is_delivery_uncertain_error, ) from Undefined.utils.message_turn import mark_message_sent_this_turn +from Undefined.skills.shared import parse_positive_int, private_access_error logger = logging.getLogger(__name__) @@ -96,18 +97,6 @@ } -def _parse_positive_int(value: Any, field_name: str) -> tuple[int | None, str | None]: - if value is None: - return None, None - try: - parsed = int(value) - except (TypeError, ValueError): - return None, f"{field_name} 必须是整数" - if parsed <= 0: - return None, f"{field_name} 必须是正整数" - return parsed, None - - def _resolve_target( args: Dict[str, Any], context: Dict[str, Any] ) -> tuple[tuple[TargetType, int] | None, str | None]: @@ -132,7 +121,7 @@ def _resolve_target( ) if has_target_id: - target_id, target_id_error = _parse_positive_int(target_id_raw, "target_id") + target_id, target_id_error = parse_positive_int(target_id_raw, "target_id") if target_id_error or target_id is None: return None, target_id_error or "target_id 非法" return (normalized_target_type, target_id), None @@ -142,33 +131,33 @@ def _resolve_target( return None, "target_type 与当前会话类型不一致，无法推断 target_id" if normalized_target_type == "group": - group_id, group_error = _parse_positive_int( + group_id, group_error = parse_positive_int( context.get("group_id"), "group_id" ) if group_error or group_id is None: return None, group_error or "无法根据 target_type 推断 target_id" return ("group", group_id), None - user_id, user_error = _parse_positive_int(context.get("user_id"), "user_id") + user_id, user_error = parse_positive_int(context.get("user_id"), "user_id") if user_error or user_id is None: return None, user_error or "无法根据 target_type 推断 target_id" return ("private", user_id), None request_type = context.get("request_type") if request_type == "group": - group_id, group_error = _parse_positive_int(context.get("group_id"), "group_id") + group_id, group_error = parse_positive_int(context.get("group_id"), "group_id") if group_error: return None, group_error if group_id is not None: return ("group", group_id), None elif request_type == "private": - user_id, user_error = _parse_positive_int(context.get("user_id"), "user_id") + user_id, user_error = parse_positive_int(context.get("user_id"), "user_id") if user_error: return None, user_error if user_id is not None: return ("private", user_id), None - fallback_group_id, fallback_group_error = _parse_positive_int( + fallback_group_id, fallback_group_error = parse_positive_int( context.get("group_id"), "group_id" ) if fallback_group_error: @@ -176,7 +165,7 @@ def _resolve_target( if fallback_group_id is not None: return ("group", fallback_group_id), None - fallback_user_id, fallback_user_error = _parse_positive_int( + fallback_user_id, fallback_user_error = parse_positive_int( context.get("user_id"), "user_id" ) if fallback_user_error: @@ -364,14 +353,6 @@ def _group_access_error(runtime_config: Any, group_id: int) -> str: return f"发送失败：目标群 {group_id} 不在允许列表内（access.allowed_group_ids）" -def _private_access_error(runtime_config: Any, user_id: int) -> str: - reason_getter = getattr(runtime_config, "private_access_denied_reason", None) - reason = reason_getter(user_id) if callable(reason_getter) else None - if reason == "blacklist": - return f"发送失败：目标用户 {user_id} 在黑名单内（access.blocked_private_ids）" - return f"发送失败：目标用户 {user_id} 不在允许列表内（access.allowed_private_ids）" - - async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: """发送单文件文本内容到群聊或私聊。""" request_id = str(context.get("request_id", "-")) @@ -422,7 +403,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if target_type == "private" and not runtime_config.is_private_allowed( target_id ): - return _private_access_error(runtime_config, target_id) + return private_access_error(runtime_config, target_id) send_file_callable, history_recorded_by_sender, sender_error = ( _resolve_file_send_callable(context, target_type) diff --git a/src/Undefined/skills/toolsets/messages/send_url_file/handler.py b/src/Undefined/skills/toolsets/messages/send_url_file/handler.py index 158533f5..6e55e52a 100644 --- a/src/Undefined/skills/toolsets/messages/send_url_file/handler.py +++ b/src/Undefined/skills/toolsets/messages/send_url_file/handler.py @@ -22,6 +22,7 @@ probe_remote_file, ) from Undefined.utils.message_turn import mark_message_sent_this_turn +from Undefined.skills.shared import parse_positive_int, private_access_error logger = logging.getLogger(__name__) @@ -32,18 +33,6 @@ DOWNLOAD_CHUNK_SIZE = 64 * 1024 -def _parse_positive_int(value: Any, field_name: str) -> tuple[int | None, str | None]: - if value is None: - return None, None - try: - parsed = int(value) - except (TypeError, ValueError): - return None, f"{field_name} 必须是整数" - if parsed <= 0: - return None, f"{field_name} 必须是正整数" - return parsed, None - - def _resolve_target( args: Dict[str, Any], context: Dict[str, Any] ) -> tuple[tuple[TargetType, int] | None, str | None]: @@ -68,7 +57,7 @@ def _resolve_target( ) if has_target_id: - target_id, target_id_error = _parse_positive_int(target_id_raw, "target_id") + target_id, target_id_error = parse_positive_int(target_id_raw, "target_id") if target_id_error or target_id is None: return None, target_id_error or "target_id 非法" return (normalized_target_type, target_id), None @@ -78,33 +67,33 @@ def _resolve_target( return None, "target_type 与当前会话类型不一致，无法推断 target_id" if normalized_target_type == "group": - group_id, group_error = _parse_positive_int( + group_id, group_error = parse_positive_int( context.get("group_id"), "group_id" ) if group_error or group_id is None: return None, group_error or "无法根据 target_type 推断 target_id" return ("group", group_id), None - user_id, user_error = _parse_positive_int(context.get("user_id"), "user_id") + user_id, user_error = parse_positive_int(context.get("user_id"), "user_id") if user_error or user_id is None: return None, user_error or "无法根据 target_type 推断 target_id" return ("private", user_id), None request_type = context.get("request_type") if request_type == "group": - group_id, group_error = _parse_positive_int(context.get("group_id"), "group_id") + group_id, group_error = parse_positive_int(context.get("group_id"), "group_id") if group_error: return None, group_error if group_id is not None: return ("group", group_id), None elif request_type == "private": - user_id, user_error = _parse_positive_int(context.get("user_id"), "user_id") + user_id, user_error = parse_positive_int(context.get("user_id"), "user_id") if user_error: return None, user_error if user_id is not None: return ("private", user_id), None - fallback_group_id, fallback_group_error = _parse_positive_int( + fallback_group_id, fallback_group_error = parse_positive_int( context.get("group_id"), "group_id" ) if fallback_group_error: @@ -112,7 +101,7 @@ def _resolve_target( if fallback_group_id is not None: return ("group", fallback_group_id), None - fallback_user_id, fallback_user_error = _parse_positive_int( + fallback_user_id, fallback_user_error = parse_positive_int( context.get("user_id"), "user_id" ) if fallback_user_error: @@ -381,14 +370,6 @@ def _group_access_error(runtime_config: Any, group_id: int) -> str: return f"发送失败：目标群 {group_id} 不在允许列表内（access.allowed_group_ids）" -def _private_access_error(runtime_config: Any, user_id: int) -> str: - reason_getter = getattr(runtime_config, "private_access_denied_reason", None) - reason = reason_getter(user_id) if callable(reason_getter) else None - if reason == "blacklist": - return f"发送失败：目标用户 {user_id} 在黑名单内（access.blocked_private_ids）" - return f"发送失败：目标用户 {user_id} 不在允许列表内（access.allowed_private_ids）" - - async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: """从 URL 下载文件并发送到群聊或私聊。""" request_id = str(context.get("request_id", "-")) @@ -417,7 +398,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if target_type == "private" and not runtime_config.is_private_allowed( target_id ): - return _private_access_error(runtime_config, target_id) + return private_access_error(runtime_config, target_id) send_file_callable, history_recorded_by_sender, sender_error = ( _resolve_file_send_callable(context, target_type) diff --git a/tests/test_skills_import_boundary.py b/tests/test_skills_import_boundary.py new file mode 100644 index 00000000..c66897ab --- /dev/null +++ b/tests/test_skills_import_boundary.py @@ -0,0 +1,195 @@ +"""Skills 导入边界检查（棘轮）。 + +`AGENTS.md` / `skills/README.md` 规定：handler 只能依赖 Python 标准库、第三方包、 +`skills/` 内部模块（含 `Undefined.skills.*` 与本目录相对导入）以及执行上下文注入 +的依赖，不得直接 import `skills/` 之外的仓库模块。 + +历史代码里存在一批越界导入，一次性全部改完风险过高，因此这里用“棘轮”方式收敛： + +- 新增越界导入会让测试失败； +- 既有越界导入记录在 `_BASELINE` 中，随重构逐步从清单里移除； +- 清单里已经不存在的条目同样会让测试失败，避免基线腐烂。 + +需要新增跨 skills 的公共能力时，把实现放到 `src/Undefined/skills/shared.py` +（或同类 skills 内模块），而不是直接引用 `services/`、`utils/` 等内部实现。 +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +_SKILLS_ROOT = Path(__file__).resolve().parents[1] / "src" / "Undefined" / "skills" + +# 既有越界导入基线：{"<相对 skills 的 handler 路径>::<被导入模块>"} +_BASELINE: frozenset[str] = frozenset( + { + "agents/entertainment_agent/tools/ai_draw_one/handler.py::Undefined.ai.parsing", + "agents/entertainment_agent/tools/ai_draw_one/handler.py::Undefined.attachments", + "agents/entertainment_agent/tools/ai_draw_one/handler.py::Undefined.config", + "agents/entertainment_agent/tools/ai_draw_one/handler.py::Undefined.token_usage_storage", + "agents/entertainment_agent/tools/ai_draw_one/handler.py::Undefined.utils.io", + "agents/entertainment_agent/tools/ai_draw_one/handler.py::Undefined.utils.paths", + "agents/entertainment_agent/tools/ai_draw_one/handler.py::Undefined.utils.request_params", + "agents/entertainment_agent/tools/ai_draw_one/handler.py::Undefined.utils.resources", + "agents/entertainment_agent/tools/minecraft_skin/handler.py::Undefined.attachments", + "agents/entertainment_agent/tools/minecraft_skin/handler.py::Undefined.utils.paths", + "agents/entertainment_agent/tools/wenchang_dijun/handler.py::Undefined.attachments", + "agents/file_analysis_agent/handler.py::Undefined.attachments", + "agents/file_analysis_agent/tools/cleanup_temp/handler.py::Undefined.utils.paths", + "agents/file_analysis_agent/tools/describe_pdf_page/handler.py::Undefined.utils", + "agents/file_analysis_agent/tools/describe_pdf_page/handler.py::Undefined.utils.paths", + "agents/info_agent/tools/arxiv_search/handler.py::Undefined.arxiv.client", + "agents/info_agent/tools/bilibili_search/handler.py::Undefined.bilibili.wbi", + "agents/info_agent/tools/bilibili_search/handler.py::Undefined.bilibili.wbi_request", + "agents/info_agent/tools/bilibili_search/handler.py::Undefined.config", + "agents/info_agent/tools/bilibili_user_info/handler.py::Undefined.bilibili.wbi", + "agents/info_agent/tools/bilibili_user_info/handler.py::Undefined.config", + "agents/info_agent/tools/net_check/handler.py::Undefined.config", + "agents/web_agent/tools/crawl_webpage/handler.py::Undefined.ai.crawl4ai_support", + "agents/web_agent/tools/crawl_webpage/handler.py::Undefined.config", + "commands/admin/handler.py::Undefined.services.commands.context", + "commands/bugfix/handler.py::Undefined.services.commands.context", + "commands/changelog/handler.py::Undefined.changelog", + "commands/changelog/handler.py::Undefined.services.commands.context", + "commands/copyright/handler.py::Undefined.services.commands.context", + "commands/faq/handler.py::Undefined.services.commands.context", + "commands/feedback/handler.py::Undefined.render", + "commands/feedback/handler.py::Undefined.services.commands.context", + "commands/feedback/handler.py::Undefined.utils", + "commands/feedback/handler.py::Undefined.utils.paths", + "commands/help/handler.py::Undefined.render", + "commands/help/handler.py::Undefined.services.commands.catalog", + "commands/help/handler.py::Undefined.services.commands.context", + "commands/help/handler.py::Undefined.services.commands.registry", + "commands/help/handler.py::Undefined.utils.paths", + "commands/naga/handler.py::Undefined.api.naga_store", + "commands/naga/handler.py::Undefined.services.commands.context", + "commands/profile/handler.py::Undefined.cognitive.service.helpers", + "commands/profile/handler.py::Undefined.render", + "commands/profile/handler.py::Undefined.services.commands.context", + "commands/profile/handler.py::Undefined.utils.paths", + "commands/stats/handler.py::Undefined.services.commands.context", + "commands/summary/handler.py::Undefined.services.commands.context", + "commands/version/handler.py::Undefined", + "commands/version/handler.py::Undefined.changelog", + "commands/version/handler.py::Undefined.services.commands.context", + "tools/arxiv_paper/handler.py::Undefined.arxiv.client", + "tools/arxiv_paper/handler.py::Undefined.arxiv.sender", + "tools/arxiv_paper/handler.py::Undefined.attachments", + "tools/bilibili_video/handler.py::Undefined.attachments", + "tools/bilibili_video/handler.py::Undefined.bilibili.downloader", + "tools/bilibili_video/handler.py::Undefined.bilibili.parser", + "tools/bilibili_video/handler.py::Undefined.bilibili.sender", + "tools/changelog_query/handler.py::Undefined.changelog", + "tools/douyin_video/handler.py::Undefined.attachments", + "tools/douyin_video/handler.py::Undefined.douyin.client", + "tools/douyin_video/handler.py::Undefined.douyin.downloader", + "tools/douyin_video/handler.py::Undefined.douyin.sender", + "tools/end/handler.py::Undefined.ai.prompts.current_input", + "tools/end/handler.py::Undefined.context", + "tools/end/handler.py::Undefined.end_summary_storage", + "tools/end/handler.py::Undefined.utils.coerce", + "tools/end/handler.py::Undefined.utils.xml", + "tools/fetch_image_uid/handler.py::Undefined.attachments", + "tools/get_picture/handler.py::Undefined.attachments", + "tools/get_picture/handler.py::Undefined.config", + "tools/get_picture/handler.py::Undefined.utils.paths", + "toolsets/group/get_avatar/handler.py::Undefined.attachments", + "toolsets/group/get_member_title/handler.py::Undefined.context", + "toolsets/group_analysis/activity_trend/handler.py::Undefined.onebot", + "toolsets/group_analysis/activity_trend/handler.py::Undefined.utils.group_metrics", + "toolsets/group_analysis/activity_trend/handler.py::Undefined.utils.message_utils", + "toolsets/group_analysis/activity_trend/handler.py::Undefined.utils.time_utils", + "toolsets/group_analysis/filter_members/handler.py::Undefined.utils.group_metrics", + "toolsets/group_analysis/filter_members/handler.py::Undefined.utils.time_utils", + "toolsets/group_analysis/inactive_risk/handler.py::Undefined.utils.group_metrics", + "toolsets/group_analysis/join_statistics/handler.py::Undefined.utils.member_utils", + "toolsets/group_analysis/join_statistics/handler.py::Undefined.utils.time_utils", + "toolsets/group_analysis/level_distribution/handler.py::Undefined.utils.group_metrics", + "toolsets/group_analysis/member_activity/handler.py::Undefined.onebot", + "toolsets/group_analysis/member_activity/handler.py::Undefined.utils.group_metrics", + "toolsets/group_analysis/member_activity/handler.py::Undefined.utils.message_utils", + "toolsets/group_analysis/member_activity/handler.py::Undefined.utils.time_utils", + "toolsets/group_analysis/member_messages/handler.py::Undefined.utils.message_utils", + "toolsets/group_analysis/member_messages/handler.py::Undefined.utils.time_utils", + "toolsets/group_analysis/member_structure/handler.py::Undefined.utils.group_metrics", + "toolsets/group_analysis/message_mix/handler.py::Undefined.onebot", + "toolsets/group_analysis/message_mix/handler.py::Undefined.utils.group_metrics", + "toolsets/group_analysis/message_mix/handler.py::Undefined.utils.message_utils", + "toolsets/group_analysis/message_mix/handler.py::Undefined.utils.time_utils", + "toolsets/group_analysis/new_member_activity/handler.py::Undefined.utils.member_utils", + "toolsets/group_analysis/new_member_activity/handler.py::Undefined.utils.message_utils", + "toolsets/group_analysis/new_member_activity/handler.py::Undefined.utils.time_utils", + "toolsets/group_analysis/rank_members/handler.py::Undefined.onebot", + "toolsets/group_analysis/rank_members/handler.py::Undefined.utils.group_metrics", + "toolsets/group_analysis/rank_members/handler.py::Undefined.utils.message_utils", + "toolsets/group_analysis/rank_members/handler.py::Undefined.utils.time_utils", + "toolsets/messages/get_forward_msg/handler.py::Undefined.attachments", + "toolsets/messages/get_forward_msg/handler.py::Undefined.attachments.forward_snapshot", + "toolsets/messages/get_forward_msg/handler.py::Undefined.attachments.segments", + "toolsets/messages/get_forward_msg/handler.py::Undefined.utils.xml", + "toolsets/messages/list_emojis/handler.py::Undefined.utils.qq_emoji", + "toolsets/messages/lookup_emoji_id/handler.py::Undefined.utils.qq_emoji", + "toolsets/messages/react_message_emoji/handler.py::Undefined.context", + "toolsets/messages/react_message_emoji/handler.py::Undefined.utils.qq_emoji", + "toolsets/messages/send_message/handler.py::Undefined.attachments", + "toolsets/messages/send_message/handler.py::Undefined.utils.message_targets", + "toolsets/messages/send_poke/handler.py::Undefined.context", + "toolsets/messages/send_private_message/handler.py::Undefined.attachments", + "toolsets/messages/send_text_file/handler.py::Undefined.utils.message_turn", + "toolsets/messages/send_text_file/handler.py::Undefined.utils.paths", + "toolsets/messages/send_url_file/handler.py::Undefined.utils.http_download", + "toolsets/messages/send_url_file/handler.py::Undefined.utils.message_turn", + "toolsets/messages/send_url_file/handler.py::Undefined.utils.paths", + "toolsets/render/render_html/handler.py::Undefined.attachments", + "toolsets/render/render_html/handler.py::Undefined.utils.cache", + "toolsets/render/render_html/handler.py::Undefined.utils.paths", + "toolsets/render/render_latex/handler.py::Undefined.attachments", + "toolsets/render/render_markdown/handler.py::Undefined.attachments", + "toolsets/render/render_markdown/handler.py::Undefined.utils.cache", + "toolsets/render/render_markdown/handler.py::Undefined.utils.paths", + } +) + + +def _collect_violations() -> set[str]: + violations: set[str] = set() + for handler in sorted(_SKILLS_ROOT.rglob("handler.py")): + tree = ast.parse(handler.read_text(encoding="utf-8")) + modules: set[str] = set() + for node in ast.walk(tree): + names: list[str] = [] + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and not node.level: + names = [node.module or ""] + for name in names: + if name == "Undefined" or ( + name.startswith("Undefined.") + and not name.startswith("Undefined.skills") + ): + modules.add(name) + relative = handler.relative_to(_SKILLS_ROOT).as_posix() + for module in sorted(modules): + violations.add(f"{relative}::{module}") + return violations + + +def test_no_new_out_of_skills_imports() -> None: + violations = _collect_violations() + new_violations = sorted(violations - _BASELINE) + assert not new_violations, ( + "检测到新的 skills 越界导入（handler 不得直接依赖 skills/ 之外的仓库模块）：\n " + + "\n ".join(new_violations) + + "\n请改用 Undefined.skills.shared 或同目录相对导入；确需长期保留时说明理由后再更新基线。" + ) + + +def test_baseline_has_no_stale_entries() -> None: + violations = _collect_violations() + stale = sorted(_BASELINE - violations) + assert not stale, ( + "以下越界导入已不存在，请从基线中移除，保持棘轮只减不增：\n " + + "\n ".join(stale) + ) From b6714cf448b9c3762a5f709e17c2a7c75dcf1bfb Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 16:21:59 +0800 Subject: [PATCH 14/30] =?UTF-8?q?chore(ci):=20=E8=A1=A5=E9=BD=90=20CI=20?= =?UTF-8?q?=E6=B2=BB=E7=90=86=E5=B9=B6=E6=95=B4=E7=90=86=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=E7=BB=84=E4=B8=8E=E5=BF=85=E9=9C=80=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI： - 工作流级 permissions: contents: read、并发取消、每个 job 的 timeout-minutes； - quality-check 增加 setup-node，让 4 个 node 前端行为测试真正执行而不是静默 skip； - 新增 python-compat（3.11 / 3.13）覆盖 pyproject 声明的版本区间两端； - 测试改用 --cov，覆盖率门禁 65% 写入 pyproject（当前 68.8%）； - 新增 CONTRIBUTING.md 与 PR 模板；新增端到端链路烟测 tests/test_end_to_end_message_pipeline.py（真实 Config/MessageHandler/ 命令注册表/队列，仅替身 OneBot 与 LLM）。 依赖卫生： - crawl4ai 与 langchain-community 是必需依赖，删除“未安装则降级”的探测与 文案（缺失时直接暴露环境问题），并同步 handler 提示与用例； - types-markdown / types-aiofiles 从运行期依赖移入 dev； - 删除与 dependency-groups 完全重复的 [project.optional-dependencies] 与 纯子集 ci 组，CI/Release 统一使用 --group dev。 --- .github/pull_request_template.md | 25 ++ .github/workflows/ci.yml | 49 +++- .github/workflows/release.yml | 8 +- CONTRIBUTING.md | 50 ++++ docs/build.md | 12 +- pyproject.toml | 31 +-- src/Undefined/ai/client/ask_loop.py | 3 - src/Undefined/ai/client/setup.py | 81 ++---- src/Undefined/ai/crawl4ai_support.py | 45 ++-- .../web_agent/tools/crawl_webpage/handler.py | 13 +- .../web_agent/tools/web_search/handler.py | 2 +- tests/test_crawl_webpage_tool.py | 24 +- tests/test_end_to_end_message_pipeline.py | 236 ++++++++++++++++++ uv.lock | 37 +-- 14 files changed, 450 insertions(+), 166 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 CONTRIBUTING.md create mode 100644 tests/test_end_to_end_message_pipeline.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..b1b1c76f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,25 @@ +## 变更说明 + + + +## 影响范围 + + + +## 关联 Issue + + + +## 自检 + +- [ ] `uv run ruff check .` 与 `uv run ruff format --check .` 通过 +- [ ] `uv run mypy .` 通过 +- [ ] `uv run pytest tests/ --cov` 通过（覆盖率不低于 `pyproject.toml` 中的 `fail_under`） +- [ ] 改动 `apps/undefined-console/` 或 `src/Undefined/webui/static/js/`：已跑 `cd apps/undefined-console && npm run check` +- [ ] 改动 `apps/undefined-chat/`：已跑 `cd apps/undefined-chat && npm run check` +- [ ] 涉及配置项：已同步 `config.toml.example` 与 `docs/configuration.md` +- [ ] 涉及 WebUI / Tauri 界面：已附截图 + +## 备注 + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69002615..c2ffe113 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,12 +7,21 @@ on: pull_request: branches: [ "main", "develop" ] +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: NODE_VERSION: "22" + PYTHON_VERSION: "3.12" jobs: quality-check: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v4 @@ -23,8 +32,13 @@ jobs: enable-cache: true cache-dependency-glob: "uv.lock" + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + - name: Install dependencies - run: uv sync --group ci -p 3.12 + run: uv sync --group dev -p ${{ env.PYTHON_VERSION }} - name: Cache Ruff uses: actions/cache@v4 @@ -51,8 +65,8 @@ jobs: - name: Run Mypy run: uv run mypy . - - name: Run Tests - run: uv run pytest tests/ + - name: Run Tests (with coverage gate) + run: uv run pytest tests/ --cov --cov-report=term - name: Build wheel run: uv build --wheel @@ -61,9 +75,38 @@ jobs: run: | uv run python -c "import glob, zipfile; whl=glob.glob('dist/*.whl')[0]; z=zipfile.ZipFile(whl); names=set(z.namelist()); required={'config.toml.example','res/prompts/undefined.xml','img/xlwy.jpg'}; missing=sorted([p for p in required if p not in names]); assert not missing, f'missing in wheel: {missing}'" + python-compat: + name: Python compat (${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + # pyproject 声明 >=3.11,<3.14；主岗位跑 3.12，这里覆盖两端边界 + python-version: ["3.11", "3.13"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install dependencies + run: uv sync --group dev -p ${{ matrix.python-version }} + + - name: Run Mypy + run: uv run mypy . + + - name: Run Tests + run: uv run pytest tests/ + native-app-quality-check: name: Native app quality (${{ matrix.app_dir }}) runs-on: ubuntu-latest + timeout-minutes: 45 strategy: fail-fast: false matrix: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7552811f..10896cb6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,6 +22,7 @@ jobs: verify-python: name: Verify Python package runs-on: ubuntu-latest + timeout-minutes: 30 environment: release steps: - name: Checkout code @@ -36,7 +37,7 @@ jobs: cache-dependency-glob: uv.lock - name: Install Python dependencies - run: uv sync --group ci -p ${{ env.PYTHON_VERSION }} + run: uv sync --group dev -p ${{ env.PYTHON_VERSION }} - name: Validate release tag, build versions, and changelog env: @@ -88,6 +89,7 @@ jobs: verify-native-app: name: Verify ${{ matrix.product }} app runs-on: ubuntu-latest + timeout-minutes: 60 environment: release strategy: fail-fast: false @@ -150,6 +152,7 @@ jobs: build-tauri-desktop: name: Build Tauri desktop (${{ matrix.product }} ${{ matrix.label }}) runs-on: ${{ matrix.os }} + timeout-minutes: 90 environment: release needs: - verify-python @@ -338,6 +341,7 @@ jobs: build-tauri-android: name: Build Tauri Android (${{ matrix.product }} ${{ matrix.abi_label }}) runs-on: ubuntu-latest + timeout-minutes: 120 environment: release needs: - verify-python @@ -646,6 +650,7 @@ jobs: publish-release: name: Publish release assets runs-on: ubuntu-latest + timeout-minutes: 30 needs: - verify-python - build-tauri-desktop @@ -678,6 +683,7 @@ jobs: publish-pypi: name: Publish Python package to PyPI runs-on: ubuntu-latest + timeout-minutes: 30 needs: - verify-python - publish-release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..72708378 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,50 @@ +# 参与贡献 + +感谢参与 Undefined！本文说明提 PR 前的最低要求；模块职责与开发细节见 [docs/development.md](docs/development.md) 与 [CLAUDE.md](CLAUDE.md)。 + +## 环境准备 + +```bash +uv sync # 安装运行期 + dev 依赖（同一份 dev 组供 CI 使用） +uv run playwright install # 截图 / 网页抓取所需的浏览器运行时 +bash scripts/install_git_hooks.sh # 启用 .githooks/pre-commit（ruff + mypy + 条件性前端检查） +``` + +前端（按需）： + +```bash +cd apps/undefined-console && npm ci && npm run check # Biome + tsc + cargo fmt/check +cd apps/undefined-chat && npm ci && npm run check # Biome + tsc + Vitest 单测/E2E + cargo +``` + +## 提交前自检 + +```bash +uv run ruff check . +uv run ruff format --check . +uv run mypy . +uv run pytest tests/ --cov +``` + +- 覆盖率门禁写在 `pyproject.toml` 的 `[tool.coverage.report].fail_under`（当前 65%），改动会降压时请补测试而不是调低阈值。 +- 只改 WebUI 脚本（`src/Undefined/webui/static/js/`）与两个原生 App 时，除 Python 检查外还要跑对应 App 的 `npm run check`。 +- 提交信息使用 Conventional Commits（如 `fix(queue): ...`、`feat(config): ...`），保持祈使句与简洁主题。 + +## 代码约定（摘要） + +- Python 4 空格缩进、全量类型注解、严格 mypy；磁盘 I/O 走 `src/Undefined/utils/io.py`。 +- Skills handler 只能依赖标准库、第三方包、`Undefined.skills.*`、同目录相对导入与 `context` 注入；跨技能共享助手放 `src/Undefined/skills/shared.py`。越界导入由 `tests/test_skills_import_boundary.py` 的棘轮基线拦截。 +- 新增/修改配置项必须同时更新 `config.toml.example`（中英双语注释）与 `docs/configuration.md`；热更新语义变化要在文档中写明。 +- 架构行为变化请在 PR 描述中给出理由与验证方式，并在必要时更新 `ARCHITECTURE.md` / 相关文档。 + +## Pull Request + +- 使用仓库的 PR 模板，填写变更说明、影响范围、关联 Issue 与自检项。 +- PR 会运行 CI：`quality-check`（lint / mypy / pytest + 覆盖率 / wheel 构建）、`python-compat`（3.11 与 3.13）、`native-app-quality-check`（Console 与 Chat）。 +- CI 未通过前不要请求评审；确需抢跑时请在描述中说明原因。 +- 用户可见的界面改动请附截图。 + +## 报告问题 + +- 功能缺陷与建议：使用仓库 Issue（附复现步骤、配置片段、日志关键字，注意脱敏 token / API Key）。 +- 安全相关问题：请不要公开提交细节，先私下联系维护者。 diff --git a/docs/build.md b/docs/build.md index dd5c0860..5542d31b 100644 --- a/docs/build.md +++ b/docs/build.md @@ -321,6 +321,16 @@ npm install 5. `publish-release`：汇总所有产物并上传 GitHub Release；Release notes 从 `CHANGELOG.md` 最新版本条目生成，不读取 tag 注释。 6. `publish-pypi`：发布 Python 包到 PyPI。 +### CI 工作流（ci.yml） + +拉取请求与 `main` / `develop` 推送会触发 `.github/workflows/ci.yml`，工作流级声明 `permissions: contents: read` 与并发取消（同一 ref 的新推送会取消旧运行），每个 job 都带 `timeout-minutes`： + +1. `quality-check`（Python 3.12）：`ruff` + `ruff format --check` + `mypy` + `pytest tests/ --cov`（覆盖率低于 `pyproject.toml` 的 `fail_under` 即失败）+ `uv build --wheel` 并校验 wheel 内含资源。该 job 会 `setup-node`，以便 WebUI 前端的 4 个 node 行为测试真正执行而不是静默 skip。 +2. `python-compat`（3.11 / 3.13）：`pyproject.toml` 声明 `>=3.11,<3.14`，因此两端边界各跑一次 `mypy` 与 `pytest`。 +3. `native-app-quality-check`（Console / Chat 矩阵）：`npm run check`。 + +依赖统一通过 `uv sync --group dev` 安装：`dev` 是唯一一份工具清单（含 `pytest-cov` 与 `types-*` 类型桩），不再维护与它重复的 `ci` 组或 `[project.optional-dependencies]`。 + ## 8. 手动 Artifact 工作流 如果只想让 GitHub Actions 编译一次原生 App 并从 workflow run 页面手动下载产物，不创建 GitHub Release，也不发布 PyPI，可以使用： @@ -395,7 +405,7 @@ uv sync --group dev -p 3.12 uv run ruff check . uv run ruff format --check . uv run mypy . -uv run pytest tests/ +uv run pytest tests/ --cov uv build ``` diff --git a/pyproject.toml b/pyproject.toml index 31a902bf..ba4cd282 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ dependencies = [ "matplotlib", "pillow", "markdown>=3.10", - "types-markdown>=3.10.0.20251106", "aiofiles>=25.1.0", "markdown-it-py[plugins]>=4.0.0", "mdit-py-plugins>=0.5.0", @@ -28,7 +27,6 @@ dependencies = [ "pymdown-extensions>=10.20", "playwright>=1.57.0", "aiohttp>=3.13.2", - "types-aiofiles>=25.1.0.20251011", "APScheduler>=3.10.0", "pymupdf>=1.27.2,!=1.27.2.2", "python-docx>=1.2.0", @@ -61,19 +59,6 @@ Issues = "https://github.com/69gg/Undefined/issues" Undefined = "Undefined.main:run" Undefined-webui = "Undefined.webui:run" -[project.optional-dependencies] -dev = [ - "mypy>=1.8.0", - "ruff>=0.15.8", - "pytest>=8.0.0", - "pytest-asyncio>=0.23.0", - "pytest-cov>=7.1.0", -] -ci = [ - "ruff>=0.15.8", - "mypy>=1.8.0", -] - [[tool.uv.index]] url = "https://pypi.org/simple" @@ -127,7 +112,17 @@ exclude = ["code"] asyncio_mode = "auto" testpaths = ["tests"] +[tool.coverage.run] +source = ["Undefined"] +branch = false + +[tool.coverage.report] +# 覆盖率棘轮：整体不低于 65%（当前约 69%），防止明显回退 +fail_under = 65 +skip_covered = true + [dependency-groups] +# 开发与 CI 共用同一组工具（CI 通过 --group dev 安装，避免两份清单漂移） dev = [ "mypy>=1.8.0", "ruff>=0.15.8", @@ -135,8 +130,6 @@ dev = [ "pytest-asyncio>=0.23.0", "pytest-cov>=7.1.0", "types-pyyaml>=6.0.12.20250915", -] -ci = [ - "ruff>=0.15.8", - "mypy>=1.8.0", + "types-markdown>=3.10.0.20251106", + "types-aiofiles>=25.1.0.20251011", ] diff --git a/src/Undefined/ai/client/ask_loop.py b/src/Undefined/ai/client/ask_loop.py index 6055b01c..66c0b8ae 100644 --- a/src/Undefined/ai/client/ask_loop.py +++ b/src/Undefined/ai/client/ask_loop.py @@ -322,9 +322,6 @@ async def emit_webchat_stage(stage: str, detail: Any | None = None) -> None: ): tool_context[MUSIC_TRACK_STORE_CONTEXT_KEY] = MusicTrackReferenceStore() tool_context.setdefault("search_wrapper", self._search_wrapper) - tool_context.setdefault( - "crawl4ai_available", self._crawl4ai_capabilities.available - ) tool_context.setdefault( "crawl4ai_proxy_config_available", self._crawl4ai_capabilities.proxy_config_available, diff --git a/src/Undefined/ai/client/setup.py b/src/Undefined/ai/client/setup.py index 9b38aafc..99d0b863 100644 --- a/src/Undefined/ai/client/setup.py +++ b/src/Undefined/ai/client/setup.py @@ -7,9 +7,10 @@ import re from collections.abc import Collection from pathlib import Path -from typing import Any, Awaitable, Callable, Optional, Protocol, TYPE_CHECKING +from typing import Any, Awaitable, Callable, Optional, Protocol import httpx +from langchain_community.utilities import SearxSearchWrapper from Undefined.attachments import AttachmentRegistry from Undefined.ai.llm import ModelRequester @@ -98,27 +99,6 @@ def __call__( ) -> Awaitable[None]: ... -# 尝试导入 langchain SearxSearchWrapper -if TYPE_CHECKING: - from langchain_community.utilities import ( - SearxSearchWrapper as SearxSearchWrapperType, - ) -else: - SearxSearchWrapperType = object - -_SearxSearchWrapper: type[SearxSearchWrapperType] | None -try: - from langchain_community.utilities import SearxSearchWrapper as _SearxSearchWrapper - - _SEARX_AVAILABLE = True -except Exception: - _SearxSearchWrapper = None - _SEARX_AVAILABLE = False - logger.warning( - "[初始化] langchain_community 未安装或 SearxSearchWrapper 不可用，搜索功能将禁用" - ) - - def _attachment_remote_download_max_bytes(runtime_config: Config) -> int: value = int(runtime_config.attachment_remote_download_max_size_mb) return max(0, value) * 1024 * 1024 @@ -295,35 +275,25 @@ def __init__( else: logger.info("[初始化] 技能热重载已禁用") - # 初始化搜索 wrapper + # 初始化搜索 wrapper（langchain_community 为必需依赖） self._search_wrapper: Optional[Any] = None - if _SEARX_AVAILABLE and _SearxSearchWrapper is not None: - searxng_url = runtime_config.searxng_url - if searxng_url: - try: - self._search_wrapper = _SearxSearchWrapper( - searx_host=searxng_url, k=10 - ) - logger.info( - "[初始化] SearxSearchWrapper 初始化成功: url=%s k=10", - redact_string(searxng_url), - ) - except Exception as exc: - logger.warning("[初始化] SearxSearchWrapper 初始化失败: %s", exc) - else: - logger.info("[初始化] SEARXNG_URL 未配置，搜索功能禁用") - - if self._crawl4ai_capabilities.available: - logger.info("[初始化] crawl4ai 可用，网页获取功能已启用") - else: - detail = self._crawl4ai_capabilities.error - if detail: - logger.warning( - "[初始化] crawl4ai 不可用，网页获取功能将禁用: %s", - detail, + searxng_url = runtime_config.searxng_url + if searxng_url: + try: + self._search_wrapper = SearxSearchWrapper(searx_host=searxng_url, k=10) + logger.info( + "[初始化] SearxSearchWrapper 初始化成功: url=%s k=10", + redact_string(searxng_url), ) - else: - logger.warning("[初始化] crawl4ai 不可用，网页获取功能将禁用") + except Exception as exc: + logger.warning("[初始化] SearxSearchWrapper 初始化失败: %s", exc) + else: + logger.info("[初始化] SEARXNG_URL 未配置，搜索功能禁用") + + logger.info( + "[初始化] crawl4ai 已就绪，网页获取功能已启用: proxy_config=%s", + self._crawl4ai_capabilities.proxy_config_available, + ) self._prompt_builder = PromptBuilder( bot_qq=self.bot_qq, @@ -561,24 +531,13 @@ def set_meme_service(self, service: Any) -> None: def apply_search_config(self, searxng_url: str) -> None: """应用搜索服务配置（支持热更新）。""" - if not _SEARX_AVAILABLE or _SearxSearchWrapper is None: - if searxng_url: - logger.warning( - "[配置] 搜索组件不可用，已忽略 SEARXNG_URL=%s", - redact_string(searxng_url), - ) - else: - logger.info("[配置] 搜索组件不可用，搜索已禁用") - self._search_wrapper = None - return - if not searxng_url: self._search_wrapper = None logger.info("[配置] SEARXNG_URL 未配置，搜索功能已禁用") return try: - self._search_wrapper = _SearxSearchWrapper(searx_host=searxng_url, k=10) + self._search_wrapper = SearxSearchWrapper(searx_host=searxng_url, k=10) logger.info( "[配置] 搜索服务已更新: url=%s k=10", redact_string(searxng_url), diff --git a/src/Undefined/ai/crawl4ai_support.py b/src/Undefined/ai/crawl4ai_support.py index 36f5b454..f36f04f6 100644 --- a/src/Undefined/ai/crawl4ai_support.py +++ b/src/Undefined/ai/crawl4ai_support.py @@ -1,4 +1,9 @@ -"""Shared Crawl4AI capability detection helpers.""" +"""Shared Crawl4AI capability helpers. + +`crawl4ai` 是必需依赖（声明在 `pyproject.toml`），这里不再做“未安装则降级”的 +探测：导入失败会直接抛错。保留下来的只有版本能力差异（如部分版本没有 +`ProxyConfig`），它决定是否能把代理配置传给 crawler。 +""" from __future__ import annotations @@ -7,43 +12,39 @@ from functools import lru_cache from typing import Any +_REQUIRED_ATTRIBUTES = ("AsyncWebCrawler", "BrowserConfig", "CrawlerRunConfig") + @dataclass(frozen=True, slots=True) class Crawl4AICapabilities: """Resolved Crawl4AI runtime capabilities.""" - available: bool proxy_config_available: bool - async_web_crawler: Any = None - browser_config: Any = None - crawler_run_config: Any = None + async_web_crawler: Any + browser_config: Any + crawler_run_config: Any proxy_config: Any = None - error: str | None = None @lru_cache(maxsize=1) def get_crawl4ai_capabilities() -> Crawl4AICapabilities: - """Detect whether Crawl4AI core classes are importable.""" - - try: - module = importlib.import_module("crawl4ai") - async_web_crawler = getattr(module, "AsyncWebCrawler") - browser_config = getattr(module, "BrowserConfig") - crawler_run_config = getattr(module, "CrawlerRunConfig") - except Exception as exc: - return Crawl4AICapabilities( - available=False, - proxy_config_available=False, - error=f"{type(exc).__name__}: {exc}", + """读取 Crawl4AI 运行时能力；缺少必需类时抛 RuntimeError。""" + + module = importlib.import_module("crawl4ai") + missing = [name for name in _REQUIRED_ATTRIBUTES if not hasattr(module, name)] + if missing: + raise RuntimeError( + "已安装的 crawl4ai 缺少必需接口: " + + ", ".join(missing) + + "；请升级 crawl4ai 到受支持的版本" ) proxy_config = getattr(module, "ProxyConfig", None) return Crawl4AICapabilities( - available=True, proxy_config_available=proxy_config is not None, - async_web_crawler=async_web_crawler, - browser_config=browser_config, - crawler_run_config=crawler_run_config, + async_web_crawler=module.AsyncWebCrawler, + browser_config=module.BrowserConfig, + crawler_run_config=module.CrawlerRunConfig, proxy_config=proxy_config, ) diff --git a/src/Undefined/skills/agents/web_agent/tools/crawl_webpage/handler.py b/src/Undefined/skills/agents/web_agent/tools/crawl_webpage/handler.py index c89ab7a7..2dc57cdb 100644 --- a/src/Undefined/skills/agents/web_agent/tools/crawl_webpage/handler.py +++ b/src/Undefined/skills/agents/web_agent/tools/crawl_webpage/handler.py @@ -35,14 +35,11 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if not url: return "URL 不能为空" - capabilities = get_crawl4ai_capabilities() - if ( - not capabilities.available - or capabilities.async_web_crawler is None - or capabilities.browser_config is None - or capabilities.crawler_run_config is None - ): - return "网页获取功能未启用（crawl4ai 未安装）" + try: + capabilities = get_crawl4ai_capabilities() + except Exception as exc: + logger.error("[crawl_webpage] crawl4ai 初始化失败: %s", exc) + return f"网页获取功能不可用（crawl4ai 环境异常: {exc}）" AsyncWebCrawler = capabilities.async_web_crawler BrowserConfig = capabilities.browser_config diff --git a/src/Undefined/skills/agents/web_agent/tools/web_search/handler.py b/src/Undefined/skills/agents/web_agent/tools/web_search/handler.py index 2fa0ce62..3e3c5a37 100644 --- a/src/Undefined/skills/agents/web_agent/tools/web_search/handler.py +++ b/src/Undefined/skills/agents/web_agent/tools/web_search/handler.py @@ -12,7 +12,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: search_wrapper = context.get("search_wrapper") if not search_wrapper: - return "搜索功能未启用（SEARXNG_URL 未配置或 langchain_community 未安装）" + return "搜索功能未启用（未配置 SEARXNG_URL）" num_results = args.get("num_results", 5) diff --git a/tests/test_crawl_webpage_tool.py b/tests/test_crawl_webpage_tool.py index 7df7e727..74f20d1f 100644 --- a/tests/test_crawl_webpage_tool.py +++ b/tests/test_crawl_webpage_tool.py @@ -60,7 +60,6 @@ def _successful_capabilities( crawler_factory: Any, ) -> Crawl4AICapabilities: return Crawl4AICapabilities( - available=True, proxy_config_available=False, async_web_crawler=crawler_factory, browser_config=_FakeBrowserConfig, @@ -100,7 +99,7 @@ def _crawler_factory(*, config: _FakeBrowserConfig) -> _FakeCrawler: @pytest.mark.asyncio -async def test_crawl_webpage_ignores_stale_false_context_flag( +async def test_crawl_webpage_runs_regardless_of_legacy_context_flags( monkeypatch: pytest.MonkeyPatch, ) -> None: result_payload = SimpleNamespace( @@ -124,7 +123,6 @@ def _crawler_factory(*, config: _FakeBrowserConfig) -> _FakeCrawler: {"url": "https://example.com", "max_chars": 8}, { "runtime_config": _runtime_config(), - "crawl4ai_available": False, }, ) @@ -133,22 +131,20 @@ def _crawler_factory(*, config: _FakeBrowserConfig) -> _FakeCrawler: @pytest.mark.asyncio -async def test_crawl_webpage_returns_unavailable_when_core_import_is_missing( +async def test_crawl_webpage_reports_broken_crawl4ai_environment( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr( - crawl_handler, - "get_crawl4ai_capabilities", - lambda: Crawl4AICapabilities( - available=False, - proxy_config_available=False, - error="ImportError: No module named crawl4ai", - ), - ) + """crawl4ai 为必需依赖：探测失败时给出环境异常说明，而不是“未安装”。""" + + def _raise() -> Any: + raise RuntimeError("已安装的 crawl4ai 缺少必需接口: BrowserConfig") + + monkeypatch.setattr(crawl_handler, "get_crawl4ai_capabilities", _raise) result = await crawl_handler.execute({"url": "https://example.com"}, {}) - assert result == "网页获取功能未启用（crawl4ai 未安装）" + assert "网页获取功能不可用" in result + assert "缺少必需接口" in result @pytest.mark.asyncio diff --git a/tests/test_end_to_end_message_pipeline.py b/tests/test_end_to_end_message_pipeline.py new file mode 100644 index 00000000..466895c3 --- /dev/null +++ b/tests/test_end_to_end_message_pipeline.py @@ -0,0 +1,236 @@ +"""端到端烟测：真实组件串联的消息处理链路。 + +与其它单测不同，这里不使用 `MessageHandler.__new__` + SimpleNamespace 拼装， +而是走真实构造路径： + + Config（真实解析） → MessageHandler（真实 __init__） + → MessageSender / CommandDispatcher / 命令注册表 / SecurityService + → 队列服务 / 消息合并器 / 管线注册表 + +只替换两处外部 I/O：OneBot 协议端（记录出站动作）与 LLM（不会在命令链路上被调用）。 +覆盖目标：一条群聊斜杠命令能完整走完 access → command → sender → OneBot 调用。 +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, cast +from unittest.mock import AsyncMock + +import httpx +import pytest + +from Undefined import __version__ +from Undefined.config import ConfigBuilder +from Undefined.config.loader import Config +from Undefined.faq import FAQStorage +from Undefined.handlers import MessageHandler + +_BASE_MAPPING: dict[str, Any] = { + "onebot": {"ws_url": "ws://127.0.0.1:3001", "token": ""}, + "core": { + "bot_qq": 10001, + "superadmin_qq": 99999, + "admin_qqs": [20001], + "process_every_message": False, + "process_private_message": True, + "process_poke_message": False, + "ai_request_max_retries": 0, + "context_recent_messages_limit": 5, + "history_max_records": 20, + }, + "access": {"mode": "off"}, + "models": { + "chat": { + "api_url": "https://api.example.com/v1", + "api_key": "key", + "model_name": "chat-model", + "max_tokens": 512, + }, + "agent": { + "api_url": "https://api.example.com/v1", + "api_key": "key", + "model_name": "agent-model", + "max_tokens": 512, + }, + "vision": { + "api_url": "https://api.example.com/v1", + "api_key": "key", + "model_name": "vision-model", + "max_tokens": 512, + }, + # 安全模型检查会真的发起 LLM 请求，端到端用例里关闭 + "security": {"enabled": False}, + }, + "message_batcher": {"enabled": False}, + "automations": {"enabled": False}, + "skills": {"hot_reload": False}, +} + + +def _message_text(message: Any) -> str: + """把出站消息统一成可断言的文本（可能是字符串或消息段数组）。""" + if isinstance(message, str): + return message + if isinstance(message, list): + parts: list[str] = [] + for segment in message: + if not isinstance(segment, dict): + parts.append(str(segment)) + continue + data = segment.get("data") or {} + parts.append(str(data.get("text", ""))) + return "".join(parts) + return str(message) + + +class _FakeOneBot: + """记录出站动作的协议端替身。""" + + def __init__(self) -> None: + self.group_messages: list[tuple[int, str]] = [] + self.private_messages: list[tuple[int, str]] = [] + + async def send_group_message( + self, group_id: int, message: Any, **kwargs: Any + ) -> dict[str, Any]: + self.group_messages.append((group_id, _message_text(message))) + return {"status": "ok", "retcode": 0, "data": {"message_id": 1}} + + async def send_private_message( + self, user_id: int, message: Any, **kwargs: Any + ) -> dict[str, Any]: + self.private_messages.append((user_id, _message_text(message))) + return {"status": "ok", "retcode": 0, "data": {"message_id": 2}} + + async def get_group_info(self, group_id: int) -> dict[str, Any] | None: + return {"group_id": group_id, "group_name": "端到端测试群"} + + async def get_stranger_info(self, user_id: int) -> dict[str, Any] | None: + return {"user_id": user_id, "nickname": f"用户{user_id}"} + + async def get_group_member_info( + self, group_id: int, user_id: int, **kwargs: Any + ) -> dict[str, Any] | None: + return {"group_id": group_id, "user_id": user_id, "card": f"群名片{user_id}"} + + async def get_msg(self, message_id: int) -> dict[str, Any] | None: + return None + + async def get_forward_msg(self, forward_id: str) -> list[dict[str, Any]]: + return [] + + def set_message_handler(self, handler: Any) -> None: # pragma: no cover - 兼容 + self._message_handler = handler + + +@pytest.fixture(autouse=True) +def _isolated_history_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """把消息历史写进临时目录，避免污染仓库 data/。""" + import Undefined.utils.history as history_module + + monkeypatch.setattr(history_module, "HISTORY_DIR", str(tmp_path / "history")) + + +def _build_handler(tmp_path: Path) -> tuple[MessageHandler, _FakeOneBot]: + config: Config = ( + ConfigBuilder() + .with_mapping(_BASE_MAPPING) + .override( + archive_path=str(tmp_path / "archive"), + knowledge_base_dir=str(tmp_path / "knowledge"), + ) + .build(strict=False) + ) + onebot = _FakeOneBot() + http_client = httpx.AsyncClient(trust_env=False) + ai = AsyncMock() + ai._http_client = http_client + ai.attachment_registry = None + ai._meme_service = None + ai._cognitive_service = None + ai.memory_storage = None + ai.model_pool = None + ai.set_queue_manager = lambda *args, **kwargs: None + ai.set_command_registry = lambda *args, **kwargs: None + + handler = MessageHandler(config, cast(Any, onebot), ai, FAQStorage()) + return handler, onebot + + +def _group_event( + *, + group_id: int = 30001, + sender_id: int = 20001, + text: str = "/version", + at_bot: bool = False, +) -> dict[str, Any]: + """构造一条群聊事件；斜杠命令只在 @bot 时生效，因此命令用例需要 at_bot=True。""" + message: list[dict[str, Any]] = [] + if at_bot: + message.append({"type": "at", "data": {"qq": "10001"}}) + message.append({"type": "text", "data": {"text": text}}) + return { + "post_type": "message", + "message_type": "group", + "group_id": group_id, + "user_id": sender_id, + "message_id": 100, + "sender": { + "user_id": sender_id, + "card": f"群名片{sender_id}", + "nickname": f"昵称{sender_id}", + "role": "owner", + "title": "", + }, + "message": message, + } + + +@pytest.mark.asyncio +async def test_group_slash_command_round_trip_invokes_onebot(tmp_path: Path) -> None: + handler, onebot = _build_handler(tmp_path) + try: + await handler.handle_message(_group_event(at_bot=True, text="/version")) + + assert onebot.group_messages, "命令回复没有通过 OneBot 发送" + group_id, reply = onebot.group_messages[-1] + assert group_id == 30001 + assert f"Undefined v{__version__}" in reply + finally: + await handler.close() + + +@pytest.mark.asyncio +async def test_plain_group_message_records_history_without_reply( + tmp_path: Path, +) -> None: + """process_every_message=false 且未 @bot 时不应触发 AI 回复，但仍写历史。""" + handler, onebot = _build_handler(tmp_path) + try: + await handler.handle_message(_group_event(at_bot=True, text="今天天气不错")) + + assert onebot.group_messages == [] + await handler.history_manager._ensure_initialized() + recent = handler.history_manager.get_recent("30001", "group", 0, 10) + assert any("今天天气不错" in str(item) for item in recent) + finally: + await handler.close() + + +@pytest.mark.asyncio +async def test_disallowed_group_is_ignored(tmp_path: Path) -> None: + """访问控制生效时，来自未授权群的消息不进历史、不回复。""" + handler, onebot = _build_handler(tmp_path) + handler.config.access_mode = "allowlist" + handler.config.allowed_group_ids = [40000] + handler.config._refresh_runtime_sets() + assert handler.config.is_group_allowed(30001) is False + try: + await handler.handle_message(_group_event(at_bot=True, text="今天天气不错")) + + assert onebot.group_messages == [] + await handler.history_manager._ensure_initialized() + assert handler.history_manager.get_recent("30001", "group", 0, 10) == [] + finally: + await handler.close() diff --git a/uv.lock b/uv.lock index 679e372f..c3e54623 100644 --- a/uv.lock +++ b/uv.lock @@ -4721,36 +4721,19 @@ dependencies = [ { name = "rich" }, { name = "silk-python" }, { name = "tiktoken" }, - { name = "types-aiofiles" }, - { name = "types-markdown" }, { name = "websockets" }, { name = "weixin-ilink-client" }, ] -[package.optional-dependencies] -ci = [ - { name = "mypy" }, - { name = "ruff" }, -] -dev = [ - { name = "mypy" }, - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "pytest-cov" }, - { name = "ruff" }, -] - [package.dev-dependencies] -ci = [ - { name = "mypy" }, - { name = "ruff" }, -] dev = [ { name = "mypy" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "ruff" }, + { name = "types-aiofiles" }, + { name = "types-markdown" }, { name = "types-pyyaml" }, ] @@ -4773,8 +4756,6 @@ requires-dist = [ { name = "markdown-it-py", extras = ["plugins"], specifier = ">=4.0.0" }, { name = "matplotlib" }, { name = "mdit-py-plugins", specifier = ">=0.5.0" }, - { name = "mypy", marker = "extra == 'ci'", specifier = ">=1.8.0" }, - { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8.0" }, { name = "numba", specifier = ">=0.61.0" }, { name = "openai", specifier = ">=2.30.0" }, { name = "openpyxl", specifier = ">=3.1.5" }, @@ -4785,9 +4766,6 @@ requires-dist = [ { name = "pymdown-extensions", specifier = ">=10.20" }, { name = "pymupdf", specifier = ">=1.27.2,!=1.27.2.2" }, { name = "pypinyin", specifier = ">=0.53.0" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, - { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=7.1.0" }, { name = "python-docx", specifier = ">=1.2.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "python-markdown-math", specifier = ">=0.9" }, @@ -4797,28 +4775,21 @@ requires-dist = [ { name = "rarfile", specifier = ">=4.2" }, { name = "regex", specifier = ">=2026.2.28" }, { name = "rich", specifier = ">=14.2.0" }, - { name = "ruff", marker = "extra == 'ci'", specifier = ">=0.15.8" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.8" }, { name = "silk-python", specifier = ">=0.2.8,<0.3.0" }, { name = "tiktoken", specifier = ">=0.7.0" }, - { name = "types-aiofiles", specifier = ">=25.1.0.20251011" }, - { name = "types-markdown", specifier = ">=3.10.0.20251106" }, { name = "websockets", specifier = ">=12.0" }, { name = "weixin-ilink-client", specifier = ">=0.1.3,<0.2.0" }, ] -provides-extras = ["dev", "ci"] [package.metadata.requires-dev] -ci = [ - { name = "mypy", specifier = ">=1.8.0" }, - { name = "ruff", specifier = ">=0.15.8" }, -] dev = [ { name = "mypy", specifier = ">=1.8.0" }, { name = "pytest", specifier = ">=8.0.0" }, { name = "pytest-asyncio", specifier = ">=0.23.0" }, { name = "pytest-cov", specifier = ">=7.1.0" }, { name = "ruff", specifier = ">=0.15.8" }, + { name = "types-aiofiles", specifier = ">=25.1.0.20251011" }, + { name = "types-markdown", specifier = ">=3.10.0.20251106" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250915" }, ] From 40058d7b4376c8778fd94931f6a06e26d757e73c Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 16:34:12 +0800 Subject: [PATCH 15/30] =?UTF-8?q?docs(skills):=20=E4=BF=AE=E6=AD=A3=20READ?= =?UTF-8?q?ME=20=E4=B8=8E=E4=BB=A3=E7=A0=81=E4=B8=8D=E7=AC=A6=E7=9A=84=201?= =?UTF-8?q?8=20=E5=A4=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 高严重度： - 技能/工具/工具集/Agent 超时描述改为与代码一致（工具/工具集默认 480s， Agent 调用未启用超时）； - get_current_time README 补全 format / include_lunar / include_almanac 参数。 中严重度： - skills/README.md 目录树与示例改用真实存在的工具（删除 send_message / get_recent_messages / save_memory 等不存在的名字）； - pipelines 目录树改为扁平结构并补 douyin； - python_interpreter README 补 libraries / send_files，修正“无法访问网络”的 绝对化描述； - toolsets 目录树补齐全部 13 个分类；music 列出全部 10 个工具（原树只画 4 个）； - get_picture README 补 delivery 参数并修正“默认发送”的描述。 低严重度： - end README 补 perspective；render README 补 delivery / target_id / message_type / output_format； - commands 目录树补齐 12 个命令； - agents README 补 douyin_video 共享工具与内置 Agent 清单； - tools README 修正 summary_agent.fetch_messages 的写法； - anthropic_skills 命名规则改用真实 TOML 路径 [tools].dot_delimiter； - 为 12 个缺 README 的工具 / 工具集 / 命令补齐文档（参数表与 config.json 对齐）。 --- src/Undefined/skills/README.md | 28 ++--- src/Undefined/skills/agents/README.md | 6 +- src/Undefined/skills/commands/README.md | 9 +- .../skills/commands/summary/README.md | 14 +++ .../skills/commands/version/README.md | 11 ++ src/Undefined/skills/tools/README.md | 4 +- .../skills/tools/calculator/README.md | 11 ++ src/Undefined/skills/tools/end/README.md | 1 + .../skills/tools/fetch_image_uid/README.md | 12 ++ .../skills/tools/get_current_time/README.md | 12 +- .../skills/tools/get_picture/README.md | 22 ++-- .../skills/tools/knowledge_list/README.md | 16 +++ .../tools/knowledge_semantic_search/README.md | 20 ++++ .../tools/knowledge_text_search/README.md | 19 ++++ .../skills/tools/python_interpreter/README.md | 13 ++- .../skills/tools/task_progress/README.md | 12 ++ src/Undefined/skills/toolsets/README.md | 40 +++---- .../skills/toolsets/attachments/README.md | 18 +++ .../skills/toolsets/cognitive/README.md | 36 ++++++ .../skills/toolsets/contacts/README.md | 22 ++++ src/Undefined/skills/toolsets/music/README.md | 104 ++++++++++++++++++ .../skills/toolsets/render/README.md | 6 + 22 files changed, 382 insertions(+), 54 deletions(-) create mode 100644 src/Undefined/skills/commands/summary/README.md create mode 100644 src/Undefined/skills/commands/version/README.md create mode 100644 src/Undefined/skills/tools/calculator/README.md create mode 100644 src/Undefined/skills/tools/fetch_image_uid/README.md create mode 100644 src/Undefined/skills/tools/knowledge_list/README.md create mode 100644 src/Undefined/skills/tools/knowledge_semantic_search/README.md create mode 100644 src/Undefined/skills/tools/knowledge_text_search/README.md create mode 100644 src/Undefined/skills/tools/task_progress/README.md create mode 100644 src/Undefined/skills/toolsets/attachments/README.md create mode 100644 src/Undefined/skills/toolsets/cognitive/README.md create mode 100644 src/Undefined/skills/toolsets/contacts/README.md create mode 100644 src/Undefined/skills/toolsets/music/README.md diff --git a/src/Undefined/skills/README.md b/src/Undefined/skills/README.md index d4e05f41..c1696ec1 100644 --- a/src/Undefined/skills/README.md +++ b/src/Undefined/skills/README.md @@ -11,22 +11,24 @@ skills/ ├── pipelines/ # 自动处理管线，斜杠命令之后、AI 之前并行检测/处理 │ ├── __init__.py │ ├── registry.py -│ └── pipelines/ -│ ├── bilibili/ -│ ├── arxiv/ -│ └── github/ +│ ├── models.py +│ ├── context.py +│ ├── bilibili/ +│ ├── douyin/ +│ ├── arxiv/ +│ └── github/ │ ├── tools/ # 基础小工具，直接暴露给 AI 调用 │ ├── __init__.py -│ ├── send_message/ -│ ├── get_recent_messages/ -│ ├── save_memory/ +│ ├── end/ +│ ├── get_current_time/ +│ ├── get_picture/ +│ ├── python_interpreter/ │ └── ... │ ├── agents/ # 智能代理，封装复杂任务的 AI Agent │ ├── __init__.py │ ├── web_agent/ -│ │ ├── anthropic_skills/ # Agent 私有 Anthropic Skills（可选） │ │ ├── tools/ │ │ ├── config.json │ │ ├── handler.py @@ -75,16 +77,16 @@ skills/ - **目录结构**: `pipelines/{pipeline_name}/config.json + handler.py`。 - **执行方式**: 同一条非命令消息会并行检测全部管线，并行处理全部命中结果；处理产出的消息通过统一发送层写入历史并自动登记本地媒体/文件附件后，再进入 AI 自动回复。 - **热重载**: 跟随 `[skills]` 的 `hot_reload`、`hot_reload_interval`、`hot_reload_debounce` 配置。 -- **示例**: `bilibili`, `arxiv`, `github` +- **示例**: `bilibili`, `douyin`, `arxiv`, `github` ### 基础工具 - **定位**: 单一功能的原子操作 - **调用方式**: 注册到主 AI 完整工具池；启用 Tool Search 时，除始终加载项外由主 AI 按需检索 schema - **Agent 可见性**: 默认仅主 AI 可见；可通过 `skills/tools/{tool_name}/callable.json` 按白名单暴露给 Agent -- **命名规则**: 简单名称（如 `send_message`, `save_memory`） +- **命名规则**: 简单名称（如 `end`, `get_current_time`）；发消息等带业务前缀的能力在工具集里（如 `messages.send_message`） - **适用场景**: 通用、高频使用的简单操作 -- **示例**: `send_message`, `get_recent_messages`, `save_memory`, `end` +- **示例**: `end`, `get_current_time`, `get_picture`, `python_interpreter` ### 工具集 @@ -110,7 +112,7 @@ skills/ - **定位**: 领域知识/指令注入，遵循 [agentskills.io](https://agentskills.io) 开放标准 - **调用方式**: 注册为 `skills-_-` function tool，AI 调用后返回完整指令内容 -- **命名规则**: 内部 `skills.`，注册为 `skills-_-`（使用 `config.tools_dot_delimiter`） +- **命名规则**: 内部 `skills.`，注册为 `skills-_-`（分隔符取 TOML 中 `[tools].dot_delimiter`，`Config` 属性名为 `tools_dot_delimiter`） - **目录结构**: `anthropic_skills//SKILL.md` 或 `agents//anthropic_skills//` - **适用场景**: 提供领域专业知识、工作流程指导、最佳实践 - **特性**: 渐进式披露（元数据始终注入，完整内容按需获取）、热重载；启用 Tool Search 时，对应 function schema 也可能需要先检索 @@ -122,7 +124,7 @@ skills/ - **handler 模块名即真实包路径**: 随包技能的 handler 按 `Undefined.skills.<...>.handler` 导入，因此 `handler.py` 内可以使用同目录相对导入（`from .helper import ...`）；常规 `import` 与注册表加载得到同一个模块对象。 - **模型 schema 按需投影**: 可通过 `skills.tool_search_enabled`（即 `[skills]` 下的 `tool_search_enabled`）让主 AI 首轮只看到配置为始终加载的工具和 `tool_search` schema，其余工具以名称目录提示，检索后从下一模型轮开始可调用。它只降低模型上下文占用，不会卸载注册表或提前导入 handler；子 Agent 不使用该投影。 - **结构化日志 + 统计**: 统一输出 `event=execute`、`status=success/timeout/error` 等结构化字段，并记录执行耗时与成功/失败计数。 -- **超时与取消**: 所有技能执行默认 120 秒超时，超时会返回提示并记录统计。 +- **超时与取消**: 工具 / 工具集执行默认 480 秒超时（Agent 调用未启用超时），超时会返回提示并记录统计。 - **热重载**: 自动扫描 `skills/` 目录，检测到 `config.json` 或 `handler.py` 变更后自动重载。 Tool Search 的配置、查询语法、请求级生命周期和权限边界详见 [Tool Search 按需工具加载](../../../docs/tool-search.md)。 diff --git a/src/Undefined/skills/agents/README.md b/src/Undefined/skills/agents/README.md index 3c0e7381..231822da 100644 --- a/src/Undefined/skills/agents/README.md +++ b/src/Undefined/skills/agents/README.md @@ -21,6 +21,8 @@ agent_name/ └── __init__.py ``` +当前内置 Agent：`web_agent`、`file_analysis_agent`、`naga_code_analysis_agent`、`undefined_self_code_agent`、`info_agent`、`entertainment_agent`、`summary_agent`、`code_delivery_agent`。 + ## 模型配置 智能体默认使用 `config.toml` 中的 `[models.agent]` 配置；同名环境变量仍可作为兼容覆盖（用于临时调试或无文件配置场景）。 @@ -183,7 +185,7 @@ Agent 的执行逻辑，负责： ## 运行特性 - **加载即校验**：Agent `handler.py` 在注册阶段导入；导入失败（如相对导入错误、缺少依赖）会记录 `load_error` 并从 Agent schema 中排除，主 AI 不会被告知一个不可用的 Agent。 -- **超时与取消**：Agent 调用默认 120s 超时，超时返回提示并记录统计。 +- **超时与取消**：Agent 调用**未启用超时**（`AgentRegistry` 显式传入 `timeout_seconds=0`），命令式超时保护由调用方负责；超时/取消语义仍会记录统计。 - **结构化日志**：统一输出 `event=execute`、`status=success/timeout/error` 等字段。 - **热重载**：检测到 `skills/agents/` 变更后自动重载 Agent 注册表。 @@ -276,7 +278,7 @@ mv skills/tools/my_tool skills/agents/my_agent/tools/ - **功能**：分析用户提供的附件、内部 UID、URL、legacy file_id、arXiv 论文标识或 Bilibili 视频标识，提取文件内容。 - **适用场景**：PDF/Word/Excel/PPT/文本/代码/压缩包解析，图片、音频、视频等多模态内容识别，arXiv 论文 PDF 分析，Bilibili 视频内容分析。 - **不适用**：没有文件来源的开放式搜索、需要联网查资料的问题、执行文件或安全鉴定。 -- **子工具**：`download_file`, `detect_file_type`, `read_text_file`, `extract_pdf`, `describe_pdf_page`, `extract_docx`, `extract_xlsx`, `extract_pptx`, `extract_archive`, `analyze_code`, `analyze_multimodal`, `cleanup_temp`；还可调用共享主工具 `arxiv_paper(output_mode=uid)` 与 `bilibili_video(output_mode=uid)` 获取待分析附件 UID。 +- **子工具**：`download_file`, `detect_file_type`, `read_text_file`, `extract_pdf`, `describe_pdf_page`, `extract_docx`, `extract_xlsx`, `extract_pptx`, `extract_archive`, `analyze_code`, `analyze_multimodal`, `cleanup_temp`；还可调用共享主工具 `arxiv_paper(output_mode=uid)`、`bilibili_video(output_mode=uid)` 与 `douyin_video(output_mode=uid)` 获取待分析附件 UID。 ### naga_code_analysis_agent（NagaAgent 代码分析助手） - **功能**：只读分析 NagaAgent 项目的结构、源码、配置、构建、部署和实现细节。 diff --git a/src/Undefined/skills/commands/README.md b/src/Undefined/skills/commands/README.md index 1bcf5826..4165588d 100644 --- a/src/Undefined/skills/commands/README.md +++ b/src/Undefined/skills/commands/README.md @@ -11,11 +11,16 @@ commands/ ├── admin/ # 管理员管理：列表/添加/移除（支持子命令，自动推断） ├── bugfix/ # 一键读取群上下文帮你诊断并回复 bug 发作原因的娱乐工具 +├── changelog/ # 版本历史与单版本变更摘要（支持子命令与自动推断） ├── copyright/ # 输出版权信息、开源协议与风险免责声明 ├── faq/ # FAQ 管理：列表/查看/搜索/删除（支持自动推断子命令） ├── feedback/ # 意见反馈：提交/查看/删除公开反馈（支持自动推断子命令） -├── help/ # 打印基础指令集列表 -├── ... +├── help/ # 打印基础指令集列表或单个命令的详细帮助 +├── naga/ # Naga 集成状态与策略查询（支持子命令） +├── profile/ # 认知记忆侧写查询（/me） +├── stats/ # Token 使用统计和图表 +├── summary/ # 消息总结（/sum） +├── version/ # 当前版本号与最新版本变更标题（/v） └── my_cmd/ # 开发你的新指令所放置的位置 ``` diff --git a/src/Undefined/skills/commands/summary/README.md b/src/Undefined/skills/commands/summary/README.md new file mode 100644 index 00000000..cd4f33fe --- /dev/null +++ b/src/Undefined/skills/commands/summary/README.md @@ -0,0 +1,14 @@ +# /summary（/sum） + +总结聊天消息。 + +- 用法：`/summary [条数|时间范围] [自定义描述]` +- 数据来源：`fetch_session_messages_callback` 拉取当前会话消息后交给 summary 模型总结； + 未单独配置 `[models.summary]` 时回退 `[models.agent]` +- 别名：`sum` +- 注意：斜杠命令由命令层直连 summary 模型；主 AI 对话里的 `summary_agent` 走 agent 模型， + 两者互不影响 + +目录结构： +- `config.json`：命令定义（权限 / 限流 / 别名） +- `handler.py`：执行逻辑 diff --git a/src/Undefined/skills/commands/version/README.md b/src/Undefined/skills/commands/version/README.md new file mode 100644 index 00000000..08c9f15a --- /dev/null +++ b/src/Undefined/skills/commands/version/README.md @@ -0,0 +1,11 @@ +# /version（/v） + +查看当前版本号与最新版本变更标题。 + +- 用法：`/version` +- 数据来源：`pyproject.toml` 的构建版本 + `CHANGELOG.md` 最新条目标题 +- 别名：`v` + +目录结构： +- `config.json`：命令定义（权限 / 限流 / 别名） +- `handler.py`：执行逻辑 diff --git a/src/Undefined/skills/tools/README.md b/src/Undefined/skills/tools/README.md index 0987cdf1..2c48cf1f 100644 --- a/src/Undefined/skills/tools/README.md +++ b/src/Undefined/skills/tools/README.md @@ -77,7 +77,7 @@ async def execute(args: dict[str, Any], context: dict[str, Any]) -> str: | `send_private_message_callback` | `Callable` | 发送私聊消息的回调（建议使用 sender） | | `send_image_callback` | `Callable` | 发送图片的回调 | | `get_recent_messages_callback` | `Callable` | 获取历史消息的回调（建议使用 history_manager） | -| `fetch_session_messages_callback` | `Callable` | 拉取当前会话消息用于总结（`/summary` 与 `summary_agent.fetch_messages`） | +| `fetch_session_messages_callback` | `Callable` | 拉取当前会话消息用于总结（`/summary` 与 `summary_agent` 的 `fetch_messages` 子工具） | | `memory_storage` | `MemoryStorage` | 记忆存储实例 | | `ai_client` | `AIClient` | AI 客户端实例（用于调用图片描述等） | | `base_path` | `Path` | 默认基础路径（通常锁定在 `code/NagaAgent`） | @@ -111,7 +111,7 @@ async def execute(args: dict[str, Any], context: dict[str, Any]) -> str: ## 运行特性 - **延迟加载 (Lazy Load)**：`handler.py` 仅在首次调用时导入，减少启动耗时。 -- **超时与取消**：单次执行默认 120s 超时；超时会返回提示并记录统计。 +- **超时与取消**：单次执行默认 480s 超时（`ToolRegistry` 未覆写 `timeout_seconds`）；超时会返回提示并记录统计。 - **结构化日志**：统一输出 `event=execute`、`status=success/timeout/error` 等字段，便于检索与统计。 - **热重载**：检测到工具变更会自动重新加载（默认开启）。 diff --git a/src/Undefined/skills/tools/calculator/README.md b/src/Undefined/skills/tools/calculator/README.md new file mode 100644 index 00000000..5bd8d7fd --- /dev/null +++ b/src/Undefined/skills/tools/calculator/README.md @@ -0,0 +1,11 @@ +# calculator 工具 + +安全的多功能数学计算器。支持算术运算、科学函数、统计函数和常量。输入数学表达式即可计算。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `expression` | `string` | 是 | 数学表达式，例如：'2+3*4'、'sqrt(144)'、'sin(pi/6)'、'log(1000,10)'、'mean(1,2,3,4,5)'、'2**10'、'factorial(10)'、'gcd(48,18)' | + +目录结构： +- `config.json`：工具定义 +- `handler.py`：执行逻辑 diff --git a/src/Undefined/skills/tools/end/README.md b/src/Undefined/skills/tools/end/README.md index 89245b53..9cdc4bc4 100644 --- a/src/Undefined/skills/tools/end/README.md +++ b/src/Undefined/skills/tools/end/README.md @@ -7,6 +7,7 @@ - `observations`（可选）：字符串数组，本轮写实新观察（严格一条一个要点，可多条）——只允许来自当前输入批次直接出现的实质新事实，或本轮回复行为产生的有价值事实（帮谁解决了什么问题）。不要求与 bot 相关，也不要求长期稳定，但必须值得日后检索；宁缺毋滥，无实质事实时用空数组。用户中心观察必须写成 `QQ号（昵称）`，保留稳定数字标识。禁止硬凑静默决策、否定清单、元评论或消费碎碎念。历史消息、认知记忆、侧写和最近消息参考只能用于消歧，不能作为新事实来源。纯流水账写 memo 而非此处。若当前输入批次包含 MessageBatcher 合并的多条消息且存在实质可记事实，必须覆盖整批消息内容，不能只记录最后一条。 - 专名拼写：涉及本项目或 bot 主名时必须写作 `Undefined`。工具会在入队认知记忆前把已知错拼 `Unfined`、`Undefind`、`undefind` 规范为 `Undefined`，避免污染长期观察。 - `force`（可选）：`true` 时可跳过"本轮未发送消息"的结束检查；同时在认知史官绝对化正则闸门失败时允许强制入库 +- `perspective`（可选）：记录视角（如 `group`/`sender`），用于多史官并行场景区分记录 - 两者都可为空；为空时仅结束会话，不写认知队列 目录结构： diff --git a/src/Undefined/skills/tools/fetch_image_uid/README.md b/src/Undefined/skills/tools/fetch_image_uid/README.md new file mode 100644 index 00000000..cde9b516 --- /dev/null +++ b/src/Undefined/skills/tools/fetch_image_uid/README.md @@ -0,0 +1,12 @@ +# fetch_image_uid 工具 + +从 URL 获取图片并注册到附件系统，返回可在回复中嵌入的图片 UID。仅支持图片类型（PNG, JPEG, GIF, WebP, BMP）。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `url` | `string` | 是 | 图片 URL（必须是 http/https 链接） | +| `display_name` | `string` | 否 | 图片的显示名称（可选，默认从 URL 推断） | + +目录结构： +- `config.json`：工具定义 +- `handler.py`：执行逻辑 diff --git a/src/Undefined/skills/tools/get_current_time/README.md b/src/Undefined/skills/tools/get_current_time/README.md index 59332b43..1a52249f 100644 --- a/src/Undefined/skills/tools/get_current_time/README.md +++ b/src/Undefined/skills/tools/get_current_time/README.md @@ -1,8 +1,16 @@ # get_current_time 工具 -用于获取当前系统时间。 +用于获取当前系统时间，可附带农历与黄历信息。 -参数：无 +## 参数 + +| 参数 | 类型 | 默认值 | 说明 | +|------|------|--------|------| +| `format` | string | `iso` | 输出格式：`iso`=ISO8601（仅时间戳），`text`=人类可读文本，`json`=结构化 JSON | +| `include_lunar` | boolean | `false` | 是否包含农历信息（年、月、日、生肖、干支等），需显式开启 | +| `include_almanac` | boolean | `false` | 是否包含黄历信息（宜忌、节气、节日、冲煞、胎神等），需显式开启 | + +默认只返回 ISO8601 时间戳；农历 / 黄历 / 文本输出需要显式传参。 目录结构： - `config.json`：工具定义 diff --git a/src/Undefined/skills/tools/get_picture/README.md b/src/Undefined/skills/tools/get_picture/README.md index 41a2786e..498bd490 100644 --- a/src/Undefined/skills/tools/get_picture/README.md +++ b/src/Undefined/skills/tools/get_picture/README.md @@ -1,14 +1,18 @@ # get_picture 工具 -用于获取指定类型图片并发送到群聊或私聊。 - -常用参数： -- `message_type`：消息类型（`group`/`private`） -- `target_id`：目标 ID（群号或 QQ 号） -- `picture_type`：图片类型（如二次元、壁纸等） -- `count`：图片数量 -- `device`：设备类型（acg 类型支持 pc/wap） -- `fourk_type`：4K 图片类型（随机 4K 时使用） +用于获取指定类型的图片，默认以可嵌入回复的图片 UID 返回（不直接发送）；也可以选择立即发送到群聊或私聊。 + +## 参数 + +| 参数 | 类型 | 默认值 | 说明 | +|------|------|--------|------| +| `delivery` | string | `embed` | `embed`：返回可插入回复的图片 UID；`send`：立即发送到目标 | +| `message_type` | string | 从会话推断 | 消息类型（`group`/`private`），仅 `delivery=send` 时需要 | +| `target_id` | integer | 从会话推断 | 目标 ID（群号或 QQ 号），仅 `delivery=send` 时需要 | +| `picture_type` | string | `acg` | 图片类型：`baisi`/`heisi`/`head`/`jk`/`acg`/`meinvpic`/`wallpaper`/`ys`/`historypic`/`random4kPic` | +| `count` | integer | `1` | 获取图片数量 | +| `device` | string | `pc` | 设备类型（`pc`/`wap`），仅 `acg` 类型支持 | +| `fourk_type` | string | `acg` | 4K 图片类型（`acg`/`wallpaper`），仅 `random4kPic` 类型支持 | 目录结构： - `config.json`：工具定义 diff --git a/src/Undefined/skills/tools/knowledge_list/README.md b/src/Undefined/skills/tools/knowledge_list/README.md new file mode 100644 index 00000000..a6b38d68 --- /dev/null +++ b/src/Undefined/skills/tools/knowledge_list/README.md @@ -0,0 +1,16 @@ +# knowledge_list 工具 + +列出知识库（紧凑结构化输出），支持按名称过滤、控制简介与返回数量。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `only_ready` | `boolean` | 否 | 是否仅返回已配置 intro.md 的知识库，默认 true | +| `include_intro` | `boolean` | 否 | 是否返回简介内容，默认 true | +| `include_has_intro` | `boolean` | 否 | 是否返回 has_intro 字段，默认 false | +| `intro_max_chars` | `integer` | 否 | 每个知识库简介最大字符数，默认 120 | +| `max_items` | `integer` | 否 | 最多返回多少个知识库，默认 50 | +| `name_keyword` | `string` | 否 | 按知识库名称关键词过滤（不区分大小写） | + +目录结构： +- `config.json`：工具定义 +- `handler.py`：执行逻辑 diff --git a/src/Undefined/skills/tools/knowledge_semantic_search/README.md b/src/Undefined/skills/tools/knowledge_semantic_search/README.md new file mode 100644 index 00000000..f9db1ff5 --- /dev/null +++ b/src/Undefined/skills/tools/knowledge_semantic_search/README.md @@ -0,0 +1,20 @@ +# knowledge_semantic_search 工具 + +语义检索（结构化紧凑输出），支持重排与结果后过滤。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `knowledge_base` | `string` | 是 | 知识库名称 | +| `query` | `string` | 是 | 查询文本 | +| `top_k` | `integer` | 否 | 语义召回数量，默认取知识库配置值 | +| `enable_rerank` | `boolean` | 否 | 是否启用重排。不传时使用知识库默认配置 | +| `rerank_top_k` | `integer` | 否 | 重排后返回数量，需小于语义召回数量 | +| `min_relevance` | `number` | 否 | 最小相关度阈值（0-1），默认 0 | +| `source_keyword` | `string` | 否 | 按 source 路径关键词过滤（不区分大小写） | +| `max_chars_per_item` | `integer` | 否 | 每条结果最大字符数，默认 220 | +| `include_rerank_score` | `boolean` | 否 | 是否输出 rerank_score，默认 true | +| `deduplicate` | `boolean` | 否 | 按 source+text 去重，默认 true | + +目录结构： +- `config.json`：工具定义 +- `handler.py`：执行逻辑 diff --git a/src/Undefined/skills/tools/knowledge_text_search/README.md b/src/Undefined/skills/tools/knowledge_text_search/README.md new file mode 100644 index 00000000..2e38a9ab --- /dev/null +++ b/src/Undefined/skills/tools/knowledge_text_search/README.md @@ -0,0 +1,19 @@ +# knowledge_text_search 工具 + +在知识库文本中关键词搜索（结构化紧凑输出），支持大小写与文件路径过滤。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `knowledge_base` | `string` | 是 | 知识库名称 | +| `keyword` | `string` | 是 | 搜索关键词 | +| `max_lines` | `integer` | 否 | 最多返回行数，默认 20 | +| `max_chars` | `integer` | 否 | 总字符上限（搜索阶段），默认 2000 | +| `max_chars_per_item` | `integer` | 否 | 每条结果最大字符数（输出阶段裁剪），默认 180 | +| `case_sensitive` | `boolean` | 否 | 是否大小写敏感，默认 false | +| `source_keyword` | `string` | 否 | 按 source 路径关键词过滤（例如 docs/faq） | +| `include_source` | `boolean` | 否 | 是否输出 source 字段，默认 true | +| `include_line` | `boolean` | 否 | 是否输出 line 字段，默认 true | + +目录结构： +- `config.json`：工具定义 +- `handler.py`：执行逻辑 diff --git a/src/Undefined/skills/tools/python_interpreter/README.md b/src/Undefined/skills/tools/python_interpreter/README.md index 6f6030ed..f934f487 100644 --- a/src/Undefined/skills/tools/python_interpreter/README.md +++ b/src/Undefined/skills/tools/python_interpreter/README.md @@ -3,11 +3,16 @@ 在隔离的 Docker 容器内执行 Python 代码，适用于计算、数据处理与逻辑验证。 限制说明： -- 无法访问网络 -- 无法访问宿主机文件系统 +- 主执行容器**无法访问网络**且只读（`--network none --read-only`） +- 无法访问宿主机文件系统（`send_files` 除外，见下） -常用参数： -- `code`：要执行的 Python 代码 +## 参数 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `code` | string | 是 | 要执行的 Python 代码 | +| `libraries` | list[string] | 否 | 需要的第三方库；指定后容器会先联网执行 `pip install`，再在无网环境运行代码 | +| `send_files` | list[string] | 否 | 需要回传的容器内 `/tmp/` 产物路径；执行后作为附件返回 | 目录结构： - `config.json`：工具定义 diff --git a/src/Undefined/skills/tools/task_progress/README.md b/src/Undefined/skills/tools/task_progress/README.md new file mode 100644 index 00000000..4b6c3ae1 --- /dev/null +++ b/src/Undefined/skills/tools/task_progress/README.md @@ -0,0 +1,12 @@ +# task_progress 工具 + +任务进度追踪。处理需要多步骤、多 Agent 协作的复杂请求时，先用 plan 动作规划步骤，再在每步完成后用 update 动作标记进度。每次调用都会返回当前完整进度。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `action` | `string(plan/update)` | 是 | plan: 创建任务计划（会替换已有计划）；update: 更新指定步骤的状态 | +| `tasks` | `list[object]` | 是 | plan: 完整步骤列表（需要 id + description）；update: 要更新的步骤（需要 id + status） | + +目录结构： +- `config.json`：工具定义 +- `handler.py`：执行逻辑 diff --git a/src/Undefined/skills/toolsets/README.md b/src/Undefined/skills/toolsets/README.md index 6a9a769b..451166c0 100644 --- a/src/Undefined/skills/toolsets/README.md +++ b/src/Undefined/skills/toolsets/README.md @@ -8,28 +8,28 @@ ``` toolsets/ -├── music/ # lxmusic2api 高层音乐能力 -│ ├── search_songs/ # 歌曲搜索 -│ ├── browse_playlists/ # 歌单标签、列表与详情 -│ ├── browse_rankings/ # 排行榜列表与详情 -│ └── get_audio/ # 直链或会话音频附件 -├── memes/ # 表情包工具集 -│ ├── search_memes/ # 表情包检索 -│ └── send_meme_by_uid/ # 按 uid 发送表情包 +├── attachments/ # 附件 UID 与 URL 互查 +├── automation/ # 条件驱动自动化 +│ ├── list/ get/ create/ update/ delete/ set_enabled/ +├── cognitive/ # 认知记忆检索（get_profile / search_events / search_profiles） ├── commands/ # 斜杠命令查询（文本匹配，不接 RAG） │ ├── search/ # 按名称/别名/说明/文档检索全部命令 │ └── get/ # 取单条命令的权限、限流、用法和 README -├── render/ # 渲染工具集 -│ ├── render_html/ # HTML 渲染 -│ ├── render_latex/ # LaTeX 渲染 -│ └── render_markdown/ # Markdown 渲染 -└── automation/ # 条件驱动自动化 - ├── list/ - ├── get/ - ├── create/ - ├── update/ - ├── delete/ - └── set_enabled/ +├── contacts/ # 好友与群列表查询（query_friends / query_groups） +├── group/ # 群信息与成员能力（get_member_info / get_avatar 等） +├── group_analysis/ # 群聊深度分析 +├── mcp/ # MCP 工具集接入 +├── memes/ # 表情包工具集 +│ ├── search_memes/ # 表情包检索 +│ └── send_meme_by_uid/ # 按 uid 发送表情包 +├── memory/ # 长期记忆 +├── messages/ # 消息发送 +├── music/ # lxmusic2api 高层音乐能力（10 个工具，见下文清单） +├── notices/ # 通知发送 +└── render/ # 渲染工具集 + ├── render_html/ # HTML 渲染 + ├── render_latex/ # LaTeX 渲染 + └── render_markdown/ # Markdown 渲染 ``` ## 命名规范 @@ -75,7 +75,7 @@ toolsets/{category}/callable.json ## 运行特性 - **延迟加载**：仅在首次调用时导入 `handler.py`。 -- **超时与取消**：单次执行默认 120 秒超时，超时会返回提示并记录统计。 +- **超时与取消**：单次执行默认 480 秒超时（`ToolSetRegistry` 未覆写 `timeout_seconds`），超时会返回提示并记录统计。 - **结构化日志**：统一输出 `event=execute`、`status=success/timeout/error` 等字段。 - **热重载**：检测到 `toolsets/` 中的变更会自动重新加载。 diff --git a/src/Undefined/skills/toolsets/attachments/README.md b/src/Undefined/skills/toolsets/attachments/README.md new file mode 100644 index 00000000..ad4828cf --- /dev/null +++ b/src/Undefined/skills/toolsets/attachments/README.md @@ -0,0 +1,18 @@ +# attachments 工具集 + +## get_uid_by_url + +通过 URL 查询对应的附件 UID。适用于需要从已知来源链接查找已注册附件的场景。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `url` | `string` | 是 | 来源 URL | + +## get_url_by_uid + +通过附件 UID 查询对应的 URL（source_ref）。适用于需要从已注册的附件追溯其来源链接的场景。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `uid` | `string` | 是 | 附件 UID，如 pic_xxx 或 file_xxx | + diff --git a/src/Undefined/skills/toolsets/cognitive/README.md b/src/Undefined/skills/toolsets/cognitive/README.md new file mode 100644 index 00000000..3cd96298 --- /dev/null +++ b/src/Undefined/skills/toolsets/cognitive/README.md @@ -0,0 +1,36 @@ +# cognitive 工具集 + +## get_profile + +获取认知记忆中的用户或群聊侧写信息。该工具是检索用途，不用于手动写入长期事实；手动长期事实请使用 memory.add。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `entity_type` | `string(user/group)` | 是 | 实体类型 | +| `entity_id` | `string` | 是 | 用户ID或群ID | + +## search_events + +搜索认知记忆中的历史事件，用于回忆之前发生过的事情。支持用户/群与时间范围过滤，并应用时间衰减加权排序。该工具是检索用途，不用于手动写入长期事实；手动长期事实请使用 memory.add。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `query` | `string` | 是 | 搜索关键词或语义描述 | +| `target_user_id` | `string` | 否 | 限定用户ID（可选） | +| `target_group_id` | `string` | 否 | 限定群ID（可选） | +| `sender_id` | `string` | 否 | 限定发送者ID（可选，与target_user_id可组合使用） | +| `request_type` | `string(private/group)` | 否 | 限定消息来源类型：private=私聊，group=群聊（可选） | +| `top_k` | `integer` | 否 | 返回条数，默认取配置 cognitive.query.tool_default_top_k | +| `time_from` | `string` | 否 | 起始时间 ISO格式（可选） | +| `time_to` | `string` | 否 | 截止时间 ISO格式（可选） | + +## search_profiles + +语义搜索认知记忆中的用户/群聊侧写，用于查找具有特定特征的用户或群。该工具是检索用途，不用于手动写入长期事实；手动长期事实请使用 memory.add。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `query` | `string` | 是 | 搜索关键词 | +| `entity_type` | `string(user/group)` | 否 | 限定类型（可选） | +| `top_k` | `integer` | 否 | 返回条数，默认8 | + diff --git a/src/Undefined/skills/toolsets/contacts/README.md b/src/Undefined/skills/toolsets/contacts/README.md new file mode 100644 index 00000000..f9461e5e --- /dev/null +++ b/src/Undefined/skills/toolsets/contacts/README.md @@ -0,0 +1,22 @@ +# contacts 工具集 + +## query_friends + +查询好友列表。支持按昵称或备注精确/模糊匹配，或获取全部好友。返回好友的QQ号、昵称和备注。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `keyword` | `string` | 否 | 搜索关键词（昵称或备注）。留空则返回全部好友。 | +| `exact` | `boolean` | 否 | 是否精确匹配（默认 false 模糊匹配）（默认 `False`） | +| `count` | `integer` | 否 | 返回的最大结果数（默认 20）（默认 `20`） | + +## query_groups + +查询群聊列表。支持按群名称精确/模糊匹配，或获取全部群聊。返回群号、群名称和成员数。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `keyword` | `string` | 否 | 搜索关键词（群名称）。留空则返回全部群聊。 | +| `exact` | `boolean` | 否 | 是否精确匹配（默认 false 模糊匹配）（默认 `False`） | +| `count` | `integer` | 否 | 返回的最大结果数（默认 20）（默认 `20`） | + diff --git a/src/Undefined/skills/toolsets/music/README.md b/src/Undefined/skills/toolsets/music/README.md new file mode 100644 index 00000000..7f4e1823 --- /dev/null +++ b/src/Undefined/skills/toolsets/music/README.md @@ -0,0 +1,104 @@ +# music 工具集 + +## browse_playlists + +浏览指定平台的歌单。action=tags 获取可用分类与排序；action=list 按 tag_id/sort_id 分页列出歌单；action=detail 按 playlist_id 读取歌单歌曲，并为每首歌返回当前任务有效的 track_ref。后续歌曲工具只需传递所选引用，不要构造 Track。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `action` | `string(tags/list/detail)` | 是 | 浏览动作 | +| `source` | `string(kw/kg/tx/wy/mg)` | 是 | 音乐平台（默认 `wy`） | +| `tag_id` | `string` | 否 | action=list 时可选，来自 tags 结果 | +| `sort_id` | `string` | 否 | action=list 时可选，来自 tags 结果 | +| `playlist_id` | `string` | 否 | action=detail 时必填，来自搜索或列表结果 | +| `page` | `integer` | 否 | list/detail 的页码（默认 `1`） | + +## browse_rankings + +浏览指定音乐平台的排行榜。action=list 获取榜单及 ranking_id；action=detail 分页读取榜单歌曲，并为每首歌返回当前任务有效的 track_ref。后续歌曲工具只需传递所选引用，不要构造 Track。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `action` | `string(list/detail)` | 是 | 浏览动作 | +| `source` | `string(kw/kg/tx/wy/mg)` | 是 | 音乐平台（默认 `wy`） | +| `ranking_id` | `string` | 否 | action=detail 时必填，来自 list 结果 | +| `page` | `integer` | 否 | 榜单详情页码（默认 `1`） | + +## find_song_matches + +根据当前任务中的 track_ref 查找其他音乐平台上的匹配版本，可用于比较音质或在原平台音频不可用时选择候选。返回精简候选列表，每项的新 track_ref 可直接传给其他歌曲工具。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `track_ref` | `string` | 是 | 歌曲候选返回的当前任务内引用 | + +## get_audio + +下载或解析 track_ref 对应歌曲的音频，但只准备交付内容，本工具本身绝不会向用户发送消息或文件。track_ref 必须来自当前任务中的歌曲候选，不要自行构造 Track。若用户未指定音质，应先查看该候选的 qualities，灵活选择其中实际列出的最高可用值并显式传入 quality；用户指定版本、格式或音质时优先遵从，不要用固定音质覆盖。delivery=attachment（默认）会下载音频、注册当前会话附件，并返回 JSON 中的 attachment 标签和 uid：普通音频文件必须在下一轮调用 messages.send_message，把返回的 原样放入 message；仅当用户明确要求原生语音消息时，才调用 messages.send_voice 并传入返回的 uid，不要两种方式重复发送。delivery=url 只返回可能快速失效的直链，也必须再用 messages.send_message 发给用户。获取成功不等于交付完成，发送工具成功后才可结束。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `track_ref` | `string` | 是 | 歌曲候选返回的当前任务内引用 | +| `quality` | `string(flac24bit/flac/wav/ape/320k/192k/128k)` | 否 | 期望音质。用户未指定时，不要机械省略或固定填写；应检查所选候选的 qualities 并传入其中实际列出的最高可用值 | +| `strict_quality` | `boolean` | 否 | 是否禁止自动降级到其他音质（默认 `False`） | +| `delivery` | `string(attachment/url)` | 否 | attachment=只下载并注册会话普通音频附件，不发送；url=只解析短时有效直链，不发送。两种模式都必须再调用相应消息工具完成交付（默认 `attachment`） | + +## get_comments + +分页获取 track_ref 对应歌曲的最新评论、热门评论或指定评论的回复。mode=replies 时必须传入前一次评论结果中的 comment_id。track_ref 必须来自当前任务中的歌曲候选。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `track_ref` | `string` | 是 | 歌曲候选返回的当前任务内引用 | +| `mode` | `string(latest/hot/replies)` | 否 | 评论类型（默认 `latest`） | +| `comment_id` | `string` | 否 | mode=replies 时必填 | +| `page` | `integer` | 否 | 页码（默认 `1`） | +| `limit` | `integer` | 否 | 每页数量（默认 `20`） | + +## get_cover + +获取 track_ref 对应歌曲的封面。默认下载并注册为当前会话图片附件，返回的 应直接嵌入回复；只有明确需要原始地址时才使用 delivery=url。track_ref 必须来自当前任务中的歌曲候选。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `track_ref` | `string` | 是 | 歌曲候选返回的当前任务内引用 | +| `delivery` | `string(attachment/url)` | 否 | attachment=返回会话附件；url=返回封面原始地址（默认 `attachment`） | + +## get_hot_search + +获取一个或全部音乐平台的实时热搜词，可用于发现当前热门歌曲后再调用 music.search_songs。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `source` | `string(all/kw/kg/tx/wy/mg)` | 否 | 平台，all 表示汇总全部平台（默认 `all`） | + +## get_lyrics + +获取歌曲歌词及平台提供的翻译、逐字歌词等数据。track_ref 必须来自当前任务中的 music.search_songs、歌单详情、排行榜详情或匹配结果；不要自行构造 Track。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `track_ref` | `string` | 是 | 歌曲候选返回的当前任务内引用 | + +## search_playlists + +按关键词跨平台搜索歌单。先从结果取得歌单的平台和 ID，再用 music.browse_playlists 的 detail 动作读取歌曲；详情内每首歌曲会提供当前任务有效的 track_ref，后续歌曲能力只需传递该引用。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `query` | `string` | 是 | 歌单名称或主题关键词 | +| `source` | `string(all/kw/kg/tx/wy/mg)` | 否 | 搜索平台（默认 `all`） | +| `page` | `integer` | 否 | 页码（默认 `1`） | +| `limit` | `integer` | 否 | 每页数量（默认 `20`） | + +## search_songs + +仅按关键词跨平台搜索歌曲并返回精简候选；每项包含当前任务内有效的 track_ref、歌名、歌手、专辑、时长、平台和可用音质，不包含需要复制的底层 Track。本工具不下载音频、不注册附件，也不向用户发送任何内容。若用户要求收听、下载或发送音频，看到结果后必须继续自主完成任务：不要固定取第一条或固定平台；先遵循用户指定的歌手、版本、平台和音质，未指定时结合歌名、歌手、专辑、版本标记与 qualities 灵活选择明确的原唱标准版，避开非用户所求的翻唱、现场、DJ、Remix、伴奏或纯音乐。匹配明确时无需询问，直接把所选候选的 track_ref 传给 music.get_audio；调用时从该候选的 qualities 中选择实际列出的最高可用音质，再按其返回说明调用消息发送工具。仅当没有结果或无法可靠判断原唱/目标版本时才询问用户。仅搜索成功不代表发歌任务完成。后续获取歌词、封面、评论、跨平台匹配或音频时只需传递对应 track_ref，不要自行构造 Track、歌曲 ID 或平台字段。 + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `query` | `string` | 是 | 歌曲名、歌手或组合关键词 | +| `source` | `string(all/kw/kg/tx/wy/mg)` | 否 | 平台：all=全部，kw=酷我，kg=酷狗，tx=QQ 音乐，wy=网易云，mg=咪咕（默认 `all`） | +| `page` | `integer` | 否 | 页码（默认 `1`） | +| `limit` | `integer` | 否 | 每页数量（默认 `20`） | + diff --git a/src/Undefined/skills/toolsets/render/README.md b/src/Undefined/skills/toolsets/render/README.md index 89dee946..d6f528df 100644 --- a/src/Undefined/skills/toolsets/render/README.md +++ b/src/Undefined/skills/toolsets/render/README.md @@ -10,5 +10,11 @@ `layout=default` 保持原有布局。`layout=long` 时，`width` 是最终图片像素宽度，高度按内容自动延伸；`padding=0` 可用于 HTML 全幅设计。 +交付参数（`render_html` / `render_markdown` 均支持，`render_latex` 不支持）： +- `delivery`（默认 `embed`）：`embed` 返回可插入回复的图片 UID；`send` 立即发送到目标 +- `target_id` / `message_type`：仅 `delivery=send` 时需要，缺省时从当前会话推断 + +`render_latex` 额外支持 `output_format`（`png` / `pdf`）。 + 目录结构： - 每个子目录对应一个工具（`config.json` + `handler.py`） From acd995fde46cd8278febf6bc396613503e757096 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 16:42:56 +0800 Subject: [PATCH 16/30] =?UTF-8?q?refactor(sender):=20=E6=94=B6=E6=95=9B=20?= =?UTF-8?q?7=20=E5=A4=84=E9=87=8D=E5=A4=8D=E7=9A=84=E7=A7=81=E8=81=8A?= =?UTF-8?q?=E8=AE=BF=E9=97=AE=E6=8E=A7=E5=88=B6=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 统一为 _ensure_private_allowed(user_id, action)：未放行时按动作名记录 warning 日志并抛出同样的 PermissionError，行为与日志保持不变； send_group/wechat 文件/语音/消息、私聊合并转发、拍一拍、文件发送共用。 --- src/Undefined/utils/sender.py | 122 +++++++--------------------------- 1 file changed, 25 insertions(+), 97 deletions(-) diff --git a/src/Undefined/utils/sender.py b/src/Undefined/utils/sender.py index 5e1f3325..707edf7d 100644 --- a/src/Undefined/utils/sender.py +++ b/src/Undefined/utils/sender.py @@ -389,22 +389,7 @@ async def send_address_file( history_attachment=history_attachment, ) return - if not self.config.is_private_allowed(address.target_id): - enabled = self.config.access_control_enabled() - reason = ( - self.config.private_access_denied_reason(address.target_id) or "unknown" - ) - logger.warning( - "[访问控制] 已拦截微信文件发送: user=%s reason=%s (access enabled=%s)", - address.target_id, - reason, - enabled, - ) - raise PermissionError( - "blocked by access control: " - f"type=private reason={reason} user_id={address.target_id} " - f"enabled={enabled}" - ) + self._ensure_private_allowed(address.target_id, "微信文件发送") service = self._require_weixin_service() sent_message_id = await service.send_file( address.target_id, @@ -486,22 +471,7 @@ async def send_address_voice( auto_history=auto_history, attachments=history_attachments or None, ) - if not self.config.is_private_allowed(address.target_id): - enabled = self.config.access_control_enabled() - reason = ( - self.config.private_access_denied_reason(address.target_id) or "unknown" - ) - logger.warning( - "[访问控制] 已拦截微信语音发送: user=%s reason=%s (access enabled=%s)", - address.target_id, - reason, - enabled, - ) - raise PermissionError( - "blocked by access control: " - f"type=private reason={reason} user_id={address.target_id} " - f"enabled={enabled}" - ) + self._ensure_private_allowed(address.target_id, "微信语音发送") service = self._require_weixin_service() prepared = await service.prepare_voice(path) @@ -600,19 +570,7 @@ async def _send_weixin_message( history_message: str | None, attachments: list[dict[str, str]] | None, ) -> str | None: - if not self.config.is_private_allowed(user_id): - enabled = self.config.access_control_enabled() - reason = self.config.private_access_denied_reason(user_id) or "unknown" - logger.warning( - "[访问控制] 已拦截微信消息发送: user=%s reason=%s (access enabled=%s)", - user_id, - reason, - enabled, - ) - raise PermissionError( - "blocked by access control: " - f"type=private reason={reason} user_id={user_id} enabled={enabled}" - ) + self._ensure_private_allowed(user_id, "微信消息发送") service = self._require_weixin_service() reply_context: ReplyContext | None = None reference: RefMessage | None = None @@ -937,6 +895,24 @@ async def _register_local_segment_attachments( ) return attachments + def _ensure_private_allowed(self, user_id: int, action: str) -> None: + """私聊访问控制统一门禁：未放行时记录日志并抛出 PermissionError。""" + if self.config.is_private_allowed(user_id): + return + enabled = self.config.access_control_enabled() + reason = self.config.private_access_denied_reason(user_id) or "unknown" + logger.warning( + "[访问控制] 已拦截%s: user=%s reason=%s (access enabled=%s)", + action, + user_id, + reason, + enabled, + ) + raise PermissionError( + "blocked by access control: " + f"type=private reason={reason} user_id={int(user_id)} enabled={enabled}" + ) + async def send_group_message( self, group_id: int, @@ -1091,19 +1067,7 @@ async def send_private_message( attachments: list[dict[str, str]] | None = None, ) -> int | None: """发送私聊消息""" - if not self.config.is_private_allowed(user_id): - enabled = self.config.access_control_enabled() - reason = self.config.private_access_denied_reason(user_id) or "unknown" - logger.warning( - "[访问控制] 已拦截私聊消息发送: user=%s reason=%s (access enabled=%s)", - user_id, - reason, - enabled, - ) - raise PermissionError( - "blocked by access control: " - f"type=private reason={reason} user_id={int(user_id)} enabled={enabled}" - ) + self._ensure_private_allowed(user_id, "私聊消息发送") safe_message = redact_string(message) logger.info(f"[发送消息] 目标用户:{user_id} | 内容摘要:{safe_message[:100]}...") @@ -1224,19 +1188,7 @@ async def send_private_forward_message( auto_history: bool = True, ) -> None: """发送私聊合并转发，并将可读摘要写入历史。""" - if not self.config.is_private_allowed(user_id): - enabled = self.config.access_control_enabled() - reason = self.config.private_access_denied_reason(user_id) or "unknown" - logger.warning( - "[访问控制] 已拦截私聊合并转发: user=%s reason=%s (access enabled=%s)", - user_id, - reason, - enabled, - ) - raise PermissionError( - "blocked by access control: " - f"type=private reason={reason} user_id={int(user_id)} enabled={enabled}" - ) + self._ensure_private_allowed(user_id, "私聊合并转发") send_private_forward = getattr(self.onebot, "send_private_forward_msg", None) if not callable(send_private_forward): @@ -1516,19 +1468,7 @@ async def send_private_poke( mark_sent: bool = True, ) -> None: """在私聊中拍一拍指定用户。""" - if not self.config.is_private_allowed(user_id): - enabled = self.config.access_control_enabled() - reason = self.config.private_access_denied_reason(user_id) or "unknown" - logger.warning( - "[访问控制] 已拦截私聊拍一拍: user=%s reason=%s (access enabled=%s)", - user_id, - reason, - enabled, - ) - raise PermissionError( - "blocked by access control: " - f"type=private reason={reason} user_id={int(user_id)} enabled={enabled}" - ) + self._ensure_private_allowed(user_id, "私聊拍一拍") logger.info("[拍一拍] 私聊用户=%s", user_id) await self.onebot.send_private_poke(user_id, mark_sent=mark_sent) @@ -1602,19 +1542,7 @@ async def send_private_file( history_attachment: AttachmentRecord | None = None, ) -> None: """通过统一发送层上传私聊文件。""" - if not self.config.is_private_allowed(user_id): - enabled = self.config.access_control_enabled() - reason = self.config.private_access_denied_reason(user_id) or "unknown" - logger.warning( - "[访问控制] 已拦截私聊文件发送: user=%s reason=%s (access enabled=%s)", - user_id, - reason, - enabled, - ) - raise PermissionError( - "blocked by access control: " - f"type=private reason={reason} user_id={int(user_id)} enabled={enabled}" - ) + self._ensure_private_allowed(user_id, "私聊文件发送") file_name = name or Path(file_path).name logger.info("[发送文件] 目标用户:%s | 文件:%s", user_id, file_name) From 1b68bed09820d9f6b3aa0f8cd079dddb7a837a54 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 16:54:07 +0800 Subject: [PATCH 17/30] =?UTF-8?q?refactor(command):=20=E6=8B=86=E5=88=86?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E4=B8=8E=20bugfix=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=EF=BC=8C=E5=85=AC=E5=BC=80=20skills=20=E8=B0=83=E7=94=A8?= =?UTF-8?q?=E5=85=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /stats 的统计、绘图与私聊投递（约 850 行，含 matplotlib 绘图）拆到 services/stats_command.py 的 StatsCommandMixin； - /bugfix 的上下文收集与诊断拆到 services/bugfix_command.py 的 BugfixCommandMixin；command.py 只保留命令解析、分发、权限与限流 （1582 行 → 550 行），不再内联 matplotlib 绘图； - 修复 skills 层调用 dispatcher 私有方法的分层倒置：handle_stats / handle_stats_private / build_private_stats_image_message / handle_bugfix 改为公开方法，skills/commands/{stats,bugfix}/handler.py 与相关测试同步； - 行为与日志保持不变，全部 3272 个测试通过。 --- src/Undefined/services/bugfix_command.py | 188 +++ src/Undefined/services/command.py | 1044 +---------------- src/Undefined/services/stats_command.py | 919 +++++++++++++++ .../skills/commands/bugfix/handler.py | 2 +- .../skills/commands/stats/handler.py | 4 +- tests/test_stats_handler_scope.py | 4 +- tests/test_stats_private_delivery.py | 22 +- tests/test_stats_private_images.py | 4 +- 8 files changed, 1131 insertions(+), 1056 deletions(-) create mode 100644 src/Undefined/services/bugfix_command.py create mode 100644 src/Undefined/services/stats_command.py diff --git a/src/Undefined/services/bugfix_command.py b/src/Undefined/services/bugfix_command.py new file mode 100644 index 00000000..7cb6e569 --- /dev/null +++ b/src/Undefined/services/bugfix_command.py @@ -0,0 +1,188 @@ +"""Bugfix（/bugfix）命令实现：读取群上下文并生成娱乐性诊断。 + +从 `services/command.py` 拆出，作为 `CommandDispatcher` 的 mixin。 +""" + +from __future__ import annotations + +import logging +from datetime import datetime +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from Undefined.faq import extract_faq_title +from Undefined.onebot import ( + get_message_content, + get_message_sender_id, + parse_message_time, +) + +if TYPE_CHECKING: + from Undefined.ai import AIClient + from Undefined.faq import FAQStorage + from Undefined.config import Config + from Undefined.onebot import OneBotClient + from Undefined.utils.sender import MessageSender + +logger = logging.getLogger(__name__) + + +class BugfixCommandMixin: + """`/bugfix` 命令的上下文收集与诊断实现。""" + + if TYPE_CHECKING: + config: Config + ai: AIClient + sender: MessageSender + onebot: OneBotClient + faq_storage: FAQStorage + + async def handle_bugfix( + self, group_id: int, admin_id: int, args: list[str] + ) -> None: + """处理 /bugfix 命令，通过分析聊天记录自动生成 FAQ 归档""" + # 1. 参数解析 + parsed = self._parse_bugfix_args(args) + if isinstance(parsed, str): + await self.sender.send_group_message(group_id, parsed) + return + + target_qqs, start_date, end_date, start_str, end_str = parsed + + await self.sender.send_group_message( + group_id, "🔍 正在获取对话记录进行回溯分析..." + ) + + try: + # 2. 获取并处理消息 + messages = await self._fetch_messages( + group_id, target_qqs, start_date, end_date + ) + if not messages: + await self.sender.send_group_message( + group_id, "❌ 未找到符合条件的对话记录。" + ) + return + + processed_text = await self._process_messages(messages) + + # 3. 生成摘要总结 + summary = await self._obtain_bugfix_summary(group_id, processed_text) + + # 4. 生成标题并入库 + title = extract_faq_title(summary) + if not title or title == "未命名问题": + title = await self.ai.generate_title(summary) + + faq = await self.faq_storage.create( + group_id=group_id, + target_qq=target_qqs[0], + start_time=start_str, + end_time=end_str, + title=title, + content=summary, + ) + + result_msg = f"✅ Bug 修复分析完成！\n\n📌 FAQ ID: {faq.id}\n📋 标题: {title}\n\n{summary}" + await self.sender.send_group_message(group_id, result_msg) + + except Exception as e: + error_id = uuid4().hex[:8] + logger.exception("Bugfix 失败: error_id=%s err=%s", error_id, e) + await self.sender.send_group_message( + group_id, + f"❌ Bug 修复分析失败，请稍后重试（错误码: {error_id}）", + ) + + def _parse_bugfix_args( + self, args: list[str] + ) -> tuple[list[int], datetime, datetime, str, str] | str: + """解析 bugfix 命令的参数""" + if len(args) < 3: + return ( + "❌ 用法: /bugfix [QQ号|@用户2] ... <开始时间> <结束时间>\n" + "时间格式: YYYY/MM/DD/HH:MM，结束时间可用 now\n" + "示例: /bugfix 123456 2024/12/01/09:00 now" + ) + + try: + target_qqs = [int(arg) for arg in args[:-2]] + start_str, end_str_raw = args[-2], args[-1] + start_date = datetime.strptime(start_str, "%Y/%m/%d/%H:%M") + + if end_str_raw.lower() == "now": + end_date, end_str = datetime.now(), "now" + else: + end_date, end_str = ( + datetime.strptime(end_str_raw, "%Y/%m/%d/%H:%M"), + end_str_raw, + ) + + return target_qqs, start_date, end_date, start_str, end_str + except ValueError: + return "❌ 参数格式错误：QQ号应为数字或 @ 提及，时间格式应为 YYYY/MM/DD/HH:MM。" + + async def _obtain_bugfix_summary(self, group_id: int, processed_text: str) -> str: + """利用 AI 生成聊天记录的 Bug 分析摘要""" + total_tokens = self.ai.count_tokens(processed_text) + max_tokens = self.config.chat_model.max_tokens + + if total_tokens <= max_tokens: + return str(await self.ai.summarize_chat(processed_text)) + + await self.sender.send_group_message( + group_id, f"📊 消息较长（{total_tokens} tokens），正在分段处理..." + ) + chunks = self.ai.split_messages_by_tokens(processed_text, max_tokens) + summaries = [await self.ai.summarize_chat(chunk) for chunk in chunks] + return str(await self.ai.merge_summaries(summaries)) + + async def _fetch_messages( + self, + group_id: int, + target_qqs: list[int], + start_date: datetime, + end_date: datetime, + ) -> list[dict[str, Any]]: + batch = await self.onebot.get_group_msg_history(group_id, count=2500) + if not batch: + return [] + target_qqs_set = set(target_qqs) + results = [] + for msg in batch: + msg_time = parse_message_time(msg) + if ( + start_date <= msg_time <= end_date + and get_message_sender_id(msg) in target_qqs_set + ): + results.append(msg) + return sorted(results, key=lambda m: m.get("time", 0)) + + async def _process_messages(self, messages: list[dict[str, Any]]) -> str: + lines = [] + for msg in messages: + sender_id = get_message_sender_id(msg) + msg_time = parse_message_time(msg).strftime("%Y-%m-%d %H:%M:%S") + content = get_message_content(msg) + text_parts = [] + for segment in content: + seg_type, seg_data = segment.get("type", ""), segment.get("data", {}) + if seg_type == "text": + text_parts.append(seg_data.get("text", "")) + elif seg_type == "image": + file = seg_data.get("file", "") or seg_data.get("url", "") + if file: + try: + url = await self.onebot.get_image(file) + if url: + res = await self.ai.analyze_multimodal(url, "image") + text_parts.append( + f"[pic]{res.get('description', '')}{res.get('ocr_text', '')}[/pic]" + ) + except Exception: + text_parts.append("[pic]图片处理失败[/pic]") + elif seg_type == "at": + text_parts.append(f"@{seg_data.get('qq', '')}") + if text_parts: + lines.append(f"[{msg_time}] {sender_id}: {''.join(text_parts)}") + return "\n".join(lines) diff --git a/src/Undefined/services/command.py b/src/Undefined/services/command.py index f672f68e..e902b0c9 100644 --- a/src/Undefined/services/command.py +++ b/src/Undefined/services/command.py @@ -1,26 +1,16 @@ import asyncio -import base64 import logging import re import time from uuid import uuid4 -from datetime import datetime from typing import Any, Awaitable, Callable, Optional, cast from pathlib import Path from Undefined.config import Config -from Undefined.faq import FAQStorage, extract_faq_title -from Undefined.onebot import ( - OneBotClient, - get_message_content, - get_message_sender_id, - parse_message_time, -) -from Undefined.utils.sender import ( - MessageSender, - is_definitive_weixin_delivery_rejection, -) -from Undefined.utils import io +from Undefined.faq import FAQStorage +from Undefined.onebot import OneBotClient +from Undefined.utils.sender import MessageSender +from Undefined.services.bugfix_command import BugfixCommandMixin from Undefined.services.commands.context import ( CommandContext, CommandSender, @@ -32,34 +22,12 @@ CommandRegistry, ) from Undefined.services.security import SecurityService +from Undefined.services.stats_command import StatsCommandMixin from Undefined.token_usage_storage import TokenUsageStorage -from Undefined.ai.queue_budget import ( - compute_queued_llm_timeout_seconds, - resolve_effective_retry_count, -) - -# 尝试导入 matplotlib -plt: Any -try: - import matplotlib.pyplot as plt - - _MATPLOTLIB_AVAILABLE = True -except ImportError: - plt = None - _MATPLOTLIB_AVAILABLE = False logger = logging.getLogger(__name__) -_STATS_DEFAULT_DAYS = 7 -_STATS_MIN_DAYS = 1 -_STATS_MAX_DAYS = 365 -_STATS_MODEL_TOP_N = 8 -_STATS_CALL_TYPE_TOP_N = 12 -_STATS_DATA_SUMMARY_MAX_CHARS = 12000 -_STATS_AI_FLAGS = {"--ai", "-a"} -_STATS_TIME_RANGE_RE = re.compile(r"^\d+[dwm]?$", re.IGNORECASE) - # 命令参数中的 @ 提及：[@QQ] / [@QQ(昵称)] / [@{QQ}] _AT_ARG_RE = re.compile(r"^\[@\s*\{?(\d{5,15})\}?(?:\(.*?\))?\]$") _ARG_TOKEN_RE = re.compile(r"\[@\s*\{?\d{5,15}\}?(?:\([^\]]*\))?\]|\S+") @@ -168,7 +136,7 @@ async def send_private_forward_message( ) -class CommandDispatcher: +class CommandDispatcher(BugfixCommandMixin, StatsCommandMixin): """命令分发处理器，负责解析和执行斜杠命令""" def __init__( @@ -253,856 +221,6 @@ def parse_command(self, text: str) -> Optional[dict[str, Any]]: "args": args, } - def _parse_time_range(self, time_str: str) -> int: - """解析时间范围字符串，返回天数 - - 参数: - time_str: 时间范围字符串（如 "7d", "1w", "30d"） - - 返回: - 天数 - """ - if not time_str: - return _STATS_DEFAULT_DAYS - - def _clamp_days(value: int) -> int: - if value < _STATS_MIN_DAYS: - return _STATS_DEFAULT_DAYS - if value > _STATS_MAX_DAYS: - return _STATS_MAX_DAYS - return value - - time_str = time_str.lower().strip() - - # 解析快捷格式 - if time_str.endswith("d"): - try: - return _clamp_days(int(time_str[:-1])) - except ValueError: - return _STATS_DEFAULT_DAYS - elif time_str.endswith("w"): - try: - return _clamp_days(int(time_str[:-1]) * 7) - except ValueError: - return _STATS_DEFAULT_DAYS - elif time_str.endswith("m"): - try: - return _clamp_days(int(time_str[:-1]) * 30) - except ValueError: - return _STATS_DEFAULT_DAYS - - # 尝试直接解析为数字（默认为天） - try: - return _clamp_days(int(time_str)) - except ValueError: - return _STATS_DEFAULT_DAYS - - def _parse_stats_options(self, args: list[str]) -> tuple[int, bool]: - """解析 /stats 参数：时间范围 + AI 分析开关。""" - days = _STATS_DEFAULT_DAYS - enable_ai_analysis = False - picked_days = False - - for raw in args: - token = str(raw or "").strip() - if not token: - continue - lower = token.lower() - if lower in _STATS_AI_FLAGS: - enable_ai_analysis = True - continue - if not picked_days and _STATS_TIME_RANGE_RE.match(lower): - days = self._parse_time_range(lower) - picked_days = True - - return days, enable_ai_analysis - - async def _handle_stats( - self, group_id: int, sender_id: int, args: list[str] - ) -> None: - """处理 /stats 命令，生成 token 使用统计图表（可选 AI 分析）""" - if not _MATPLOTLIB_AVAILABLE: - await self.sender.send_group_message( - group_id, "❌ 缺少必要的库，无法生成图表。请安装 matplotlib。" - ) - return - - days, enable_ai_analysis = self._parse_stats_options(args) - img_dir: Path | None = None - try: - summary = await self._token_usage_storage.get_summary(days=days) - if summary["total_calls"] == 0: - await self.sender.send_group_message( - group_id, f"📊 最近 {days} 天内无 Token 使用记录。" - ) - return - - ai_analysis = "" - if enable_ai_analysis: - ai_analysis = await self._run_stats_ai_analysis( - scope="group", - scope_id=group_id, - sender_id=sender_id, - summary=summary, - days=days, - ) - - img_dir = await self._create_stats_render_dir() - await self._generate_stats_charts(summary, img_dir, days) - - forward_messages = await self._build_stats_forward_nodes( - summary, img_dir, days, ai_analysis - ) - await self._send_group_forward_message( - group_id, - forward_messages, - history_message=self._build_stats_history_message( - summary, - days, - ai_analysis, - ), - ) - - except Exception as e: - error_id = uuid4().hex[:8] - logger.exception( - "[Stats] 生成统计图表失败: error_id=%s err=%s", error_id, e - ) - await self.sender.send_group_message( - group_id, - f"❌ 生成统计图表失败，请稍后重试（错误码: {error_id}）", - ) - finally: - if img_dir is not None: - await io.delete_tree(img_dir) - - async def _send_group_forward_message( - self, - group_id: int, - messages: list[dict[str, Any]], - *, - history_message: str, - ) -> None: - send_forward = getattr(self.sender, "send_group_forward_message", None) - if callable(send_forward): - await send_forward(group_id, messages, history_message=history_message) - return - - await self.onebot.send_forward_msg(group_id, messages) - if self.history_manager is None: - return - text_content = history_message.strip() - if not text_content: - return - await self.history_manager.add_group_message( - group_id=group_id, - sender_id=getattr(self.config, "bot_qq", 0), - text_content=text_content, - sender_nickname="Bot", - group_name="", - ) - - @staticmethod - def _build_stats_history_message( - summary: dict[str, Any], - days: int, - ai_analysis: str, - ) -> str: - lines = [ - f"[命令输出] /stats 最近 {days} 天 Token 使用统计", - f"总调用: {summary.get('total_calls', 0)}", - f"总 Token: {summary.get('total_tokens', 0)}", - f"输入 Token: {summary.get('prompt_tokens', 0)}", - f"输出 Token: {summary.get('completion_tokens', 0)}", - ] - if ai_analysis.strip(): - lines.extend(["", "AI 分析:", ai_analysis.strip()]) - return "\n".join(lines) - - async def _handle_stats_private( - self, - user_id: int, - sender_id: int, - args: list[str], - send_message: Callable[[str], Awaitable[None]] | None = None, - send_forward: PrivateForwardCallback | None = None, - *, - is_webui_session: bool = False, - ) -> None: - """处理私聊 /stats（含 WebUI 虚拟私聊适配）。""" - - async def _send_private(message: str) -> None: - if send_message is not None: - await send_message(message) - else: - await self.sender.send_private_message(user_id, message) - - days, enable_ai_analysis = self._parse_stats_options(args) - img_dir: Path | None = None - try: - summary = await self._token_usage_storage.get_summary(days=days) - if summary["total_calls"] == 0: - await _send_private(f"📊 最近 {days} 天内无 Token 使用记录。") - return - - ai_analysis = "" - if enable_ai_analysis: - ai_analysis = await self._run_stats_ai_analysis( - scope="private", - scope_id=0, - sender_id=sender_id, - summary=summary, - days=days, - ) - - if not _MATPLOTLIB_AVAILABLE: - message = "❌ 缺少必要的库，无法生成图表。请安装 matplotlib。" - if is_webui_session: - message += "\n\n" + self._build_stats_summary_text(summary) - if ai_analysis: - message += f"\n\n🤖 AI 智能分析\n{ai_analysis}" - await _send_private(message) - return - - img_dir = await self._create_stats_render_dir() - await self._generate_stats_charts(summary, img_dir, days) - - if send_forward is not None: - nodes = await self._build_stats_forward_nodes( - summary, - img_dir, - days, - ai_analysis, - ) - try: - await send_forward( - user_id, - nodes, - history_message=self._build_stats_history_message( - summary, - days, - ai_analysis, - ), - ) - except Exception as exc: - if not is_definitive_weixin_delivery_rejection(exc): - logger.exception( - "[Stats] 私聊图表投递结果不确定，不执行二次发送: " - "user=%s err=%s", - user_id, - exc, - ) - return - logger.exception( - "[Stats] 私聊图表投递失败，回退文本摘要: user=%s err=%s", - user_id, - exc, - ) - fallback = self._build_stats_summary_text(summary) - if ai_analysis: - fallback += f"\n\n🤖 AI 智能分析\n{ai_analysis}" - fallback += "\n\n⚠️ 图表发送失败，已保留统计摘要。" - await _send_private(fallback) - return - - await _send_private(f"📊 最近 {days} 天的 Token 使用统计：") - for img_name in ["line_chart", "bar_chart", "pie_chart", "table"]: - img_path = img_dir / f"stats_{img_name}.png" - if await io.is_file(img_path): - message = await self._build_private_stats_image_message( - img_path, - inline_base64=is_webui_session, - ) - await _send_private(message) - - await _send_private(self._build_stats_summary_text(summary)) - if ai_analysis: - await _send_private(f"🤖 AI 智能分析\n{ai_analysis}") - except Exception as e: - error_id = uuid4().hex[:8] - logger.exception( - "[Stats] 私聊统计生成失败: error_id=%s user=%s err=%s", - error_id, - user_id, - e, - ) - await _send_private( - f"❌ 生成统计图表失败，请稍后重试（错误码: {error_id}）" - ) - finally: - if img_dir is not None: - await io.delete_tree(img_dir) - - async def _build_private_stats_image_message( - self, - image_path: Path, - *, - inline_base64: bool, - ) -> str: - file_uri = image_path.absolute().as_uri() - if not inline_base64: - return f"[CQ:image,file={file_uri}]" - - try: - content = await io.read_bytes(image_path) - encoded = base64.b64encode(content).decode("ascii") - except Exception as exc: - logger.warning( - "[Stats] 图像 base64 编码失败，回退文件路径: path=%s err=%s", - file_uri, - exc, - ) - return f"[CQ:image,file={file_uri}]" - - return f"[CQ:image,file=base64://{encoded}]" - - async def _run_stats_ai_analysis( - self, - *, - scope: str, - scope_id: int, - sender_id: int, - summary: dict[str, Any], - days: int, - ) -> str: - if not self.queue_manager: - return "" - - data_summary = self._build_data_summary(summary, days) - request_id = uuid4().hex - analysis_event = asyncio.Event() - self._stats_analysis_events[request_id] = analysis_event - request_data = { - "type": "stats_analysis", - "group_id": scope_id, - "request_id": request_id, - "sender_id": sender_id, - "data_summary": data_summary, - "summary": summary, - "days": days, - "scope": scope, - } - receipt = await self.queue_manager.add_group_mention_request( - request_data, model_name=self.config.chat_model.model_name - ) - logger.info("[Stats] 已投递 AI 分析请求: scope=%s target=%s", scope, scope_id) - - wait_timeout = compute_queued_llm_timeout_seconds( - self.ai.runtime_config, - self.config.chat_model, - retry_count=resolve_effective_retry_count( - self.ai.runtime_config, self.queue_manager - ), - initial_wait_seconds=float( - getattr(receipt, "estimated_wait_seconds", 0.0) or 0.0 - ), - ) - try: - await asyncio.wait_for(analysis_event.wait(), timeout=wait_timeout) - ai_analysis = self._stats_analysis_results.pop(request_id, "") - logger.info( - "[Stats] 已获取 AI 分析结果: scope=%s len=%s", scope, len(ai_analysis) - ) - return ai_analysis - except asyncio.TimeoutError: - logger.warning( - "[Stats] AI 分析超时: scope=%s target=%s timeout=%.1fs", - scope, - scope_id, - wait_timeout, - ) - return "AI 分析在当前动态等待期限内超时。" - finally: - self._stats_analysis_events.pop(request_id, None) - self._stats_analysis_results.pop(request_id, None) - - def _build_data_summary(self, summary: dict[str, Any], days: int) -> str: - """构建用于 AI 分析的统计数据摘要""" - lines = [] - lines.append("📊 Token 使用综合分析数据：") - lines.append("") - - # 整体概况 - lines.append("【整体概况】") - lines.append(f"统计周期: {days} 天") - lines.append(f"总调用次数: {summary['total_calls']}") - lines.append(f"总 Token 消耗: {summary['total_tokens']:,}") - lines.append(f"平均响应时间: {summary['avg_duration']:.2f}s") - lines.append(f"涉及模型数: {len(summary['models'])}") - lines.append("") - - # 时间维度 - daily_stats = summary.get("daily_stats", {}) - if daily_stats: - dates = sorted(daily_stats.keys()) - total_daily_calls = sum(daily_stats[d]["calls"] for d in dates) - total_daily_tokens = sum(daily_stats[d]["tokens"] for d in dates) - avg_daily_calls = total_daily_calls / len(dates) if dates else 0 - avg_daily_tokens = total_daily_tokens / len(dates) if dates else 0 - - # 找出高峰日 - peak_day = ( - max(dates, key=lambda d: daily_stats[d]["tokens"]) if dates else "" - ) - peak_day_tokens = daily_stats[peak_day]["tokens"] if peak_day else 0 - - lines.append("【时间维度】") - lines.append(f"统计天数: {len(dates)} 天") - lines.append(f"每日平均调用: {avg_daily_calls:.1f} 次") - lines.append(f"每日平均 Token: {avg_daily_tokens:,.0f} 个") - lines.append(f"高峰日期: {peak_day} ({peak_day_tokens:,} tokens)") - lines.append("") - - # 模型维度 - models = summary.get("models", {}) - if models: - lines.append("【模型维度】") - total_tokens_all = summary["total_tokens"] - sorted_models = sorted( - models.items(), key=lambda x: x[1]["tokens"], reverse=True - ) - for model_name, model_data in sorted_models[:_STATS_MODEL_TOP_N]: - calls = model_data["calls"] - tokens = model_data["tokens"] - prompt_tokens = model_data["prompt_tokens"] - completion_tokens = model_data["completion_tokens"] - token_pct = ( - (tokens / total_tokens_all * 100) if total_tokens_all > 0 else 0 - ) - avg_per_call = tokens / calls if calls > 0 else 0 - io_ratio = completion_tokens / prompt_tokens if prompt_tokens > 0 else 0 - - lines.append(f"模型: {model_name}") - lines.append( - f" - 调用次数: {calls} ({calls / summary['total_calls'] * 100:.1f}%)" - ) - lines.append(f" - Token 消耗: {tokens:,} ({token_pct:.1f}%)") - lines.append(f" - 平均每次调用: {avg_per_call:.0f} tokens") - lines.append( - f" - 输入: {prompt_tokens:,} / 输出: {completion_tokens:,}" - ) - lines.append(f" - 输入/输出比: 1:{io_ratio:.2f}") - lines.append("") - - if len(sorted_models) > _STATS_MODEL_TOP_N: - others = sorted_models[_STATS_MODEL_TOP_N:] - others_calls = sum(int(item[1].get("calls", 0)) for item in others) - others_tokens = sum(int(item[1].get("tokens", 0)) for item in others) - others_pct = ( - (others_tokens / total_tokens_all * 100) - if total_tokens_all > 0 - else 0.0 - ) - lines.append( - f"其余 {len(others)} 个模型合计: 调用 {others_calls} 次, Token {others_tokens:,} ({others_pct:.1f}%)" - ) - lines.append("") - - # 调用类型维度 - call_types = summary.get("call_types", {}) - if call_types: - lines.append("【调用类型维度】") - sorted_types = sorted( - call_types.items(), key=lambda item: int(item[1]), reverse=True - ) - total_calls = max(1, int(summary.get("total_calls", 0))) - for call_type, count in sorted_types[:_STATS_CALL_TYPE_TOP_N]: - ratio = int(count) / total_calls * 100 - lines.append(f"- {call_type}: {count} 次 ({ratio:.1f}%)") - if len(sorted_types) > _STATS_CALL_TYPE_TOP_N: - rest_count = sum( - int(item[1]) for item in sorted_types[_STATS_CALL_TYPE_TOP_N:] - ) - ratio = rest_count / total_calls * 100 - lines.append( - f"- 其他 {len(sorted_types) - _STATS_CALL_TYPE_TOP_N} 类: {rest_count} 次 ({ratio:.1f}%)" - ) - lines.append("") - - # 效率指标 - prompt_tokens = summary.get("prompt_tokens", 0) - completion_tokens = summary.get("completion_tokens", 0) - total_tokens = summary.get("total_tokens", 0) - input_ratio = (prompt_tokens / total_tokens * 100) if total_tokens > 0 else 0 - output_ratio = ( - (completion_tokens / total_tokens * 100) if total_tokens > 0 else 0 - ) - output_per_input = completion_tokens / prompt_tokens if prompt_tokens > 0 else 0 - - lines.append("【效率指标】") - lines.append(f"输入 Token: {prompt_tokens:,} ({input_ratio:.1f}%)") - lines.append(f"输出 Token: {completion_tokens:,} ({output_ratio:.1f}%)") - lines.append(f"输入/输出比: 1:{output_per_input:.2f}") - lines.append("") - - # 趋势分析 - if daily_stats and len(daily_stats) > 1: - lines.append("【趋势分析】") - dates = sorted(daily_stats.keys()) - first_day_tokens = daily_stats[dates[0]]["tokens"] - last_day_tokens = daily_stats[dates[-1]]["tokens"] - trend_change = ( - ((last_day_tokens - first_day_tokens) / first_day_tokens * 100) - if first_day_tokens > 0 - else 0 - ) - trend_desc = "增长" if trend_change > 0 else "下降" - lines.append( - f"总体趋势: {trend_desc} {abs(trend_change):.1f}% (从首日到末日)" - ) - lines.append("") - - summary_text = "\n".join(lines) - if len(summary_text) > _STATS_DATA_SUMMARY_MAX_CHARS: - trimmed = summary_text[: _STATS_DATA_SUMMARY_MAX_CHARS - 80].rstrip() - summary_text = ( - f"{trimmed}\n\n[数据摘要已截断，总长度 {len(summary_text)} 字符，" - f"仅保留前 {_STATS_DATA_SUMMARY_MAX_CHARS} 字符]" - ) - logger.info( - "[Stats] 数据摘要过长已截断: original_len=%s max_len=%s", - len("\n".join(lines)), - _STATS_DATA_SUMMARY_MAX_CHARS, - ) - return summary_text - - def _build_stats_summary_text(self, summary: dict[str, Any]) -> str: - return f"""📈 摘要汇总: -• 总调用次数: {summary["total_calls"]} -• 总消耗 Tokens: {summary["total_tokens"]:,} - └─ 输入: {summary["prompt_tokens"]:,} - └─ 输出: {summary["completion_tokens"]:,} -• 平均耗时: {summary["avg_duration"]:.2f}s -• 涉及模型数: {len(summary["models"])}""" - - def set_stats_analysis_result( - self, group_id: int, request_id: str, analysis: str - ) -> None: - """设置 AI 分析结果（由队列处理器调用）""" - event = self._stats_analysis_events.get(request_id) - if not event: - logger.warning( - "[StatsAnalysis] 未找到等待事件，群: %s, 请求: %s", - group_id, - request_id, - ) - return - self._stats_analysis_results[request_id] = analysis - event.set() - - async def _create_stats_render_dir(self) -> Path: - from Undefined.utils.paths import RENDER_CACHE_DIR - - base_dir = await io.ensure_dir(RENDER_CACHE_DIR) - return await io.ensure_dir(base_dir / f"stats_{uuid4().hex}") - - async def _generate_stats_charts( - self, - summary: dict[str, Any], - img_dir: Path, - days: int, - ) -> None: - async with self._stats_render_lock: - await asyncio.to_thread( - self._generate_stats_charts_sync, - summary, - img_dir, - days, - ) - - def _generate_stats_charts_sync( - self, - summary: dict[str, Any], - img_dir: Path, - days: int, - ) -> None: - self._generate_line_chart(summary, img_dir, days) - self._generate_bar_chart(summary, img_dir) - self._generate_pie_chart(summary, img_dir) - self._generate_stats_table(summary, img_dir) - - async def _build_stats_forward_nodes( - self, - summary: dict[str, Any], - img_dir: Path, - days: int, - ai_analysis: str = "", - ) -> list[dict[str, Any]]: - """构建用于合并转发的统计图表节点列表""" - nodes = [] - bot_qq = str(self.config.bot_qq) - - # 辅助函数：创建消息节点 - def add_node(content: str) -> None: - nodes.append( - { - "type": "node", - "data": {"name": "Bot", "uin": bot_qq, "content": content}, - } - ) - - add_node(f"📊 最近 {days} 天的 Token 使用统计：") - - # 添加所有生成的图片 - for img_name in ["line_chart", "bar_chart", "pie_chart", "table"]: - img_path = img_dir / f"stats_{img_name}.png" - if await io.is_file(img_path): - add_node(f"[CQ:image,file={img_path.absolute().as_uri()}]") - - # 添加文本摘要 - add_node(self._build_stats_summary_text(summary)) - - # 添加 AI 分析结果（如果有） - if ai_analysis: - add_node(f"🤖 AI 智能分析\n{ai_analysis}") - - return nodes - - def _generate_line_chart( - self, summary: dict[str, Any], img_dir: Path, days: int - ) -> None: - """生成折线图：时间趋势""" - daily_stats = summary["daily_stats"] - if not daily_stats: - return - - # 准备数据 - dates = sorted(daily_stats.keys()) - tokens = [daily_stats[d]["tokens"] for d in dates] - prompt_tokens = [daily_stats[d]["prompt_tokens"] for d in dates] - completion_tokens = [daily_stats[d]["completion_tokens"] for d in dates] - - # 创建图表 - fig, ax = plt.subplots(figsize=(12, 7)) - - # 绘制折线 - ax.plot( - dates, tokens, marker="o", linewidth=2, label="Total Token", color="#2196F3" - ) - ax.plot( - dates, - prompt_tokens, - marker="s", - linewidth=2, - label="Input Token", - color="#4CAF50", - ) - ax.plot( - dates, - completion_tokens, - marker="^", - linewidth=2, - label="Output Token", - color="#FF9800", - ) - - # 设置标题和标签 - ax.set_title( - f"Token Usage Trend for Last {days} Days", fontsize=16, fontweight="bold" - ) - ax.set_xlabel("Date", fontsize=12) - ax.set_ylabel("Token Count", fontsize=12) - ax.legend(loc="upper left", fontsize=10) - ax.grid(True, alpha=0.3) - - # 旋转 x 轴标签 - plt.xticks(rotation=45, ha="right") - - # 调整布局 - plt.tight_layout() - - # 保存图表 - filepath = img_dir / "stats_line_chart.png" - plt.savefig(filepath, dpi=150, bbox_inches="tight") - plt.close(fig) - - def _generate_bar_chart(self, summary: dict[str, Any], img_dir: Path) -> None: - """生成柱状图：模型对比""" - models = summary["models"] - if not models: - return - - # 准备数据 - model_names = list(models.keys()) - tokens = [models[m]["tokens"] for m in model_names] - prompt_tokens = [models[m]["prompt_tokens"] for m in model_names] - completion_tokens = [models[m]["completion_tokens"] for m in model_names] - - # 创建图表 - fig, ax = plt.subplots(figsize=(14, 8)) - - # 设置柱状图位置 - x = range(len(model_names)) - width = 0.25 - - # 绘制柱状图 - bars1 = ax.bar( - [i - width for i in x], - tokens, - width, - label="Total Token", - color="#2196F3", - alpha=0.8, - ) - bars2 = ax.bar( - x, - prompt_tokens, - width, - label="Input Token", - color="#4CAF50", - alpha=0.8, - ) - bars3 = ax.bar( - [i + width for i in x], - completion_tokens, - width, - label="Output Token", - color="#FF9800", - alpha=0.8, - ) - - # 设置标题和标签 - ax.set_title("Token Usage Comparison by Model", fontsize=16, fontweight="bold") - ax.set_xlabel("Model", fontsize=12) - ax.set_ylabel("Token Count", fontsize=12) - ax.set_xticks(x) - ax.set_xticklabels(model_names, rotation=45, ha="right") - ax.legend(loc="upper right", fontsize=10) - ax.grid(True, alpha=0.3, axis="y") - - # 在柱子上添加数值标签 - for bars in [bars1, bars2, bars3]: - for bar in bars: - height = bar.get_height() - if height > 0: - ax.text( - bar.get_x() + bar.get_width() / 2.0, - height, - f"{int(height):,}", - ha="center", - va="bottom", - fontsize=8, - ) - - # 调整布局 - plt.tight_layout() - - # 保存图表 - filepath = img_dir / "stats_bar_chart.png" - plt.savefig(filepath, dpi=150, bbox_inches="tight") - plt.close(fig) - - def _generate_pie_chart(self, summary: dict[str, Any], img_dir: Path) -> None: - """生成饼图：输入/输出比例""" - prompt_tokens = summary["prompt_tokens"] - completion_tokens = summary["completion_tokens"] - - if prompt_tokens == 0 and completion_tokens == 0: - return - - # 创建图表 - fig, ax = plt.subplots(figsize=(12, 8)) - - # 准备数据 - labels = ["Input Token", "Output Token"] - sizes = [prompt_tokens, completion_tokens] - colors = ["#4CAF50", "#FF9800"] - explode = (0.05, 0.05) # 突出显示 - - # 绘制饼图 - wedges, *_ = ax.pie( - sizes, - explode=explode, - labels=labels, - colors=colors, - autopct="%1.1f%%", - startangle=90, - textprops={"fontsize": 12}, - ) - - # 设置标题 - ax.set_title("Input/Output Token Ratio", fontsize=16, fontweight="bold", pad=20) - - # 添加图例 - ax.legend( - wedges, - [f"{labels[i]}: {sizes[i]:,}" for i in range(len(labels))], - loc="center left", - bbox_to_anchor=(1, 0, 0.5, 1), - fontsize=10, - ) - - # 调整布局 - plt.tight_layout() - - # 保存图表 - filepath = img_dir / "stats_pie_chart.png" - plt.savefig(filepath, dpi=150, bbox_inches="tight") - plt.close(fig) - - def _generate_stats_table(self, summary: dict[str, Any], img_dir: Path) -> None: - """生成统计表格""" - models = summary["models"] - if not models: - return - - # 准备数据 - model_names = list(models.keys()) - data = [] - for model in model_names: - m = models[model] - data.append( - [ - model, - m["calls"], - f"{m['tokens']:,}", - f"{m['prompt_tokens']:,}", - f"{m['completion_tokens']:,}", - ] - ) - - # 创建图表 - fig, ax = plt.subplots(figsize=(14, 9)) - ax.axis("tight") - ax.axis("off") - - # 创建表格 - table = ax.table( - cellText=data, - colLabels=["Model", "Calls", "Total Token", "Input Token", "Output Token"], - cellLoc="center", - loc="center", - ) - - # 设置表格样式 - table.auto_set_font_size(False) - table.set_fontsize(10) - table.scale(1.2, 1.5) - - # 设置表头样式 - for i in range(5): - table[(0, i)].set_facecolor("#2196F3") - table[(0, i)].set_text_props(weight="bold", color="white") - - # 设置行样式 - for i in range(1, len(data) + 1): - for j in range(5): - if i % 2 == 0: - table[(i, j)].set_facecolor("#f0f0f0") - - # 设置标题 - ax.set_title( - "Model Usage Statistics Details", fontsize=16, fontweight="bold", pad=20 - ) - - # 调整布局 - plt.tight_layout() - - # 保存图表 - filepath = img_dir / "stats_table.png" - plt.savefig(filepath, dpi=150, bbox_inches="tight") - plt.close(fig) - async def dispatch( self, group_id: int, sender_id: int, command: dict[str, Any] ) -> None: @@ -1430,153 +548,3 @@ async def _send_no_permission( ) -> None: logger.warning("[命令] 权限不足: sender=%s cmd=/%s", sender_id, cmd_name) await send_message(f"⚠️ 权限不足：只有{required_role}可以使用此命令") - - async def _handle_bugfix( - self, group_id: int, admin_id: int, args: list[str] - ) -> None: - """处理 /bugfix 命令，通过分析聊天记录自动生成 FAQ 归档""" - # 1. 参数解析 - parsed = self._parse_bugfix_args(args) - if isinstance(parsed, str): - await self.sender.send_group_message(group_id, parsed) - return - - target_qqs, start_date, end_date, start_str, end_str = parsed - - await self.sender.send_group_message( - group_id, "🔍 正在获取对话记录进行回溯分析..." - ) - - try: - # 2. 获取并处理消息 - messages = await self._fetch_messages( - group_id, target_qqs, start_date, end_date - ) - if not messages: - await self.sender.send_group_message( - group_id, "❌ 未找到符合条件的对话记录。" - ) - return - - processed_text = await self._process_messages(messages) - - # 3. 生成摘要总结 - summary = await self._obtain_bugfix_summary(group_id, processed_text) - - # 4. 生成标题并入库 - title = extract_faq_title(summary) - if not title or title == "未命名问题": - title = await self.ai.generate_title(summary) - - faq = await self.faq_storage.create( - group_id=group_id, - target_qq=target_qqs[0], - start_time=start_str, - end_time=end_str, - title=title, - content=summary, - ) - - result_msg = f"✅ Bug 修复分析完成！\n\n📌 FAQ ID: {faq.id}\n📋 标题: {title}\n\n{summary}" - await self.sender.send_group_message(group_id, result_msg) - - except Exception as e: - error_id = uuid4().hex[:8] - logger.exception("Bugfix 失败: error_id=%s err=%s", error_id, e) - await self.sender.send_group_message( - group_id, - f"❌ Bug 修复分析失败，请稍后重试（错误码: {error_id}）", - ) - - def _parse_bugfix_args( - self, args: list[str] - ) -> tuple[list[int], datetime, datetime, str, str] | str: - """解析 bugfix 命令的参数""" - if len(args) < 3: - return ( - "❌ 用法: /bugfix [QQ号|@用户2] ... <开始时间> <结束时间>\n" - "时间格式: YYYY/MM/DD/HH:MM，结束时间可用 now\n" - "示例: /bugfix 123456 2024/12/01/09:00 now" - ) - - try: - target_qqs = [int(arg) for arg in args[:-2]] - start_str, end_str_raw = args[-2], args[-1] - start_date = datetime.strptime(start_str, "%Y/%m/%d/%H:%M") - - if end_str_raw.lower() == "now": - end_date, end_str = datetime.now(), "now" - else: - end_date, end_str = ( - datetime.strptime(end_str_raw, "%Y/%m/%d/%H:%M"), - end_str_raw, - ) - - return target_qqs, start_date, end_date, start_str, end_str - except ValueError: - return "❌ 参数格式错误：QQ号应为数字或 @ 提及，时间格式应为 YYYY/MM/DD/HH:MM。" - - async def _obtain_bugfix_summary(self, group_id: int, processed_text: str) -> str: - """利用 AI 生成聊天记录的 Bug 分析摘要""" - total_tokens = self.ai.count_tokens(processed_text) - max_tokens = self.config.chat_model.max_tokens - - if total_tokens <= max_tokens: - return str(await self.ai.summarize_chat(processed_text)) - - await self.sender.send_group_message( - group_id, f"📊 消息较长（{total_tokens} tokens），正在分段处理..." - ) - chunks = self.ai.split_messages_by_tokens(processed_text, max_tokens) - summaries = [await self.ai.summarize_chat(chunk) for chunk in chunks] - return str(await self.ai.merge_summaries(summaries)) - - async def _fetch_messages( - self, - group_id: int, - target_qqs: list[int], - start_date: datetime, - end_date: datetime, - ) -> list[dict[str, Any]]: - batch = await self.onebot.get_group_msg_history(group_id, count=2500) - if not batch: - return [] - target_qqs_set = set(target_qqs) - results = [] - for msg in batch: - msg_time = parse_message_time(msg) - if ( - start_date <= msg_time <= end_date - and get_message_sender_id(msg) in target_qqs_set - ): - results.append(msg) - return sorted(results, key=lambda m: m.get("time", 0)) - - async def _process_messages(self, messages: list[dict[str, Any]]) -> str: - lines = [] - for msg in messages: - sender_id = get_message_sender_id(msg) - msg_time = parse_message_time(msg).strftime("%Y-%m-%d %H:%M:%S") - content = get_message_content(msg) - text_parts = [] - for segment in content: - seg_type, seg_data = segment.get("type", ""), segment.get("data", {}) - if seg_type == "text": - text_parts.append(seg_data.get("text", "")) - elif seg_type == "image": - file = seg_data.get("file", "") or seg_data.get("url", "") - if file: - try: - url = await self.onebot.get_image(file) - if url: - res = await self.ai.analyze_multimodal(url, "image") - text_parts.append( - f"[pic]{res.get('description', '')}{res.get('ocr_text', '')}[/pic]" - ) - except Exception: - text_parts.append("[pic]图片处理失败[/pic]") - elif seg_type == "at": - text_parts.append(f"@{seg_data.get('qq', '')}") - if text_parts: - lines.append(f"[{msg_time}] {sender_id}: {''.join(text_parts)}") - return "\n".join(lines) diff --git a/src/Undefined/services/stats_command.py b/src/Undefined/services/stats_command.py new file mode 100644 index 00000000..c2ff7672 --- /dev/null +++ b/src/Undefined/services/stats_command.py @@ -0,0 +1,919 @@ +"""Stats（/stats）命令实现：统计汇总、图表绘制与私聊投递。 + +从 `services/command.py` 拆出，作为 `CommandDispatcher` 的 mixin； +绘图依赖 matplotlib（必需依赖），相关常量也随实现一起迁移。 +""" + +from __future__ import annotations + +import asyncio +import base64 +import logging +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any +from uuid import uuid4 +from collections.abc import Awaitable, Callable + +import matplotlib.pyplot as plt + +from Undefined.ai.queue_budget import ( + compute_queued_llm_timeout_seconds, + resolve_effective_retry_count, +) +from Undefined.services.commands.context import PrivateForwardCallback +from Undefined.utils import io +from Undefined.utils.paths import RENDER_CACHE_DIR +from Undefined.utils.sender import is_definitive_weixin_delivery_rejection + +if TYPE_CHECKING: + from Undefined.ai import AIClient + from Undefined.config import Config + from Undefined.faq import FAQStorage + from Undefined.onebot import OneBotClient + from Undefined.services.commands.registry import CommandRegistry + from Undefined.token_usage_storage import TokenUsageStorage + from Undefined.utils.history import MessageHistoryManager + from Undefined.services.queue_manager import QueueManager + from Undefined.utils.sender import MessageSender + +logger = logging.getLogger(__name__) + +_STATS_DEFAULT_DAYS = 7 +_STATS_MIN_DAYS = 1 +_STATS_MAX_DAYS = 365 +_STATS_MODEL_TOP_N = 8 +_STATS_CALL_TYPE_TOP_N = 12 +_STATS_DATA_SUMMARY_MAX_CHARS = 12000 +_STATS_AI_FLAGS = {"--ai", "-a"} +_STATS_TIME_RANGE_RE = re.compile(r"^\d+[dwm]?$", re.IGNORECASE) + +# matplotlib 为必需依赖；保留标志以兼容既有日志分支 +_MATPLOTLIB_AVAILABLE = True + + +class StatsCommandMixin: + """`/stats` 命令的统计、绘图与投递实现。""" + + if TYPE_CHECKING: + config: Config + ai: AIClient + sender: MessageSender + onebot: OneBotClient + faq_storage: FAQStorage + command_registry: CommandRegistry + history_manager: MessageHistoryManager + queue_manager: QueueManager + _token_usage_storage: TokenUsageStorage + _stats_analysis_results: dict[str, str] + _stats_analysis_events: dict[str, asyncio.Event] + _stats_render_lock: asyncio.Lock + + def _parse_time_range(self, time_str: str) -> int: + """解析时间范围字符串，返回天数 + + 参数: + time_str: 时间范围字符串（如 "7d", "1w", "30d"） + + 返回: + 天数 + """ + if not time_str: + return _STATS_DEFAULT_DAYS + + def _clamp_days(value: int) -> int: + if value < _STATS_MIN_DAYS: + return _STATS_DEFAULT_DAYS + if value > _STATS_MAX_DAYS: + return _STATS_MAX_DAYS + return value + + time_str = time_str.lower().strip() + + # 解析快捷格式 + if time_str.endswith("d"): + try: + return _clamp_days(int(time_str[:-1])) + except ValueError: + return _STATS_DEFAULT_DAYS + elif time_str.endswith("w"): + try: + return _clamp_days(int(time_str[:-1]) * 7) + except ValueError: + return _STATS_DEFAULT_DAYS + elif time_str.endswith("m"): + try: + return _clamp_days(int(time_str[:-1]) * 30) + except ValueError: + return _STATS_DEFAULT_DAYS + + # 尝试直接解析为数字（默认为天） + try: + return _clamp_days(int(time_str)) + except ValueError: + return _STATS_DEFAULT_DAYS + + def _parse_stats_options(self, args: list[str]) -> tuple[int, bool]: + """解析 /stats 参数：时间范围 + AI 分析开关。""" + days = _STATS_DEFAULT_DAYS + enable_ai_analysis = False + picked_days = False + + for raw in args: + token = str(raw or "").strip() + if not token: + continue + lower = token.lower() + if lower in _STATS_AI_FLAGS: + enable_ai_analysis = True + continue + if not picked_days and _STATS_TIME_RANGE_RE.match(lower): + days = self._parse_time_range(lower) + picked_days = True + + return days, enable_ai_analysis + + async def handle_stats( + self, group_id: int, sender_id: int, args: list[str] + ) -> None: + """处理 /stats 命令，生成 token 使用统计图表（可选 AI 分析）""" + if not _MATPLOTLIB_AVAILABLE: + await self.sender.send_group_message( + group_id, "❌ 缺少必要的库，无法生成图表。请安装 matplotlib。" + ) + return + + days, enable_ai_analysis = self._parse_stats_options(args) + img_dir: Path | None = None + try: + summary = await self._token_usage_storage.get_summary(days=days) + if summary["total_calls"] == 0: + await self.sender.send_group_message( + group_id, f"📊 最近 {days} 天内无 Token 使用记录。" + ) + return + + ai_analysis = "" + if enable_ai_analysis: + ai_analysis = await self._run_stats_ai_analysis( + scope="group", + scope_id=group_id, + sender_id=sender_id, + summary=summary, + days=days, + ) + + img_dir = await self._create_stats_render_dir() + await self._generate_stats_charts(summary, img_dir, days) + + forward_messages = await self._build_stats_forward_nodes( + summary, img_dir, days, ai_analysis + ) + await self._send_group_forward_message( + group_id, + forward_messages, + history_message=self._build_stats_history_message( + summary, + days, + ai_analysis, + ), + ) + + except Exception as e: + error_id = uuid4().hex[:8] + logger.exception( + "[Stats] 生成统计图表失败: error_id=%s err=%s", error_id, e + ) + await self.sender.send_group_message( + group_id, + f"❌ 生成统计图表失败，请稍后重试（错误码: {error_id}）", + ) + finally: + if img_dir is not None: + await io.delete_tree(img_dir) + + async def _send_group_forward_message( + self, + group_id: int, + messages: list[dict[str, Any]], + *, + history_message: str, + ) -> None: + send_forward = getattr(self.sender, "send_group_forward_message", None) + if callable(send_forward): + await send_forward(group_id, messages, history_message=history_message) + return + + await self.onebot.send_forward_msg(group_id, messages) + if self.history_manager is None: + return + text_content = history_message.strip() + if not text_content: + return + await self.history_manager.add_group_message( + group_id=group_id, + sender_id=getattr(self.config, "bot_qq", 0), + text_content=text_content, + sender_nickname="Bot", + group_name="", + ) + + @staticmethod + def _build_stats_history_message( + summary: dict[str, Any], + days: int, + ai_analysis: str, + ) -> str: + lines = [ + f"[命令输出] /stats 最近 {days} 天 Token 使用统计", + f"总调用: {summary.get('total_calls', 0)}", + f"总 Token: {summary.get('total_tokens', 0)}", + f"输入 Token: {summary.get('prompt_tokens', 0)}", + f"输出 Token: {summary.get('completion_tokens', 0)}", + ] + if ai_analysis.strip(): + lines.extend(["", "AI 分析:", ai_analysis.strip()]) + return "\n".join(lines) + + async def handle_stats_private( + self, + user_id: int, + sender_id: int, + args: list[str], + send_message: Callable[[str], Awaitable[None]] | None = None, + send_forward: PrivateForwardCallback | None = None, + *, + is_webui_session: bool = False, + ) -> None: + """处理私聊 /stats（含 WebUI 虚拟私聊适配）。""" + + async def _send_private(message: str) -> None: + if send_message is not None: + await send_message(message) + else: + await self.sender.send_private_message(user_id, message) + + days, enable_ai_analysis = self._parse_stats_options(args) + img_dir: Path | None = None + try: + summary = await self._token_usage_storage.get_summary(days=days) + if summary["total_calls"] == 0: + await _send_private(f"📊 最近 {days} 天内无 Token 使用记录。") + return + + ai_analysis = "" + if enable_ai_analysis: + ai_analysis = await self._run_stats_ai_analysis( + scope="private", + scope_id=0, + sender_id=sender_id, + summary=summary, + days=days, + ) + + if not _MATPLOTLIB_AVAILABLE: + message = "❌ 缺少必要的库，无法生成图表。请安装 matplotlib。" + if is_webui_session: + message += "\n\n" + self._build_stats_summary_text(summary) + if ai_analysis: + message += f"\n\n🤖 AI 智能分析\n{ai_analysis}" + await _send_private(message) + return + + img_dir = await self._create_stats_render_dir() + await self._generate_stats_charts(summary, img_dir, days) + + if send_forward is not None: + nodes = await self._build_stats_forward_nodes( + summary, + img_dir, + days, + ai_analysis, + ) + try: + await send_forward( + user_id, + nodes, + history_message=self._build_stats_history_message( + summary, + days, + ai_analysis, + ), + ) + except Exception as exc: + if not is_definitive_weixin_delivery_rejection(exc): + logger.exception( + "[Stats] 私聊图表投递结果不确定，不执行二次发送: " + "user=%s err=%s", + user_id, + exc, + ) + return + logger.exception( + "[Stats] 私聊图表投递失败，回退文本摘要: user=%s err=%s", + user_id, + exc, + ) + fallback = self._build_stats_summary_text(summary) + if ai_analysis: + fallback += f"\n\n🤖 AI 智能分析\n{ai_analysis}" + fallback += "\n\n⚠️ 图表发送失败，已保留统计摘要。" + await _send_private(fallback) + return + + await _send_private(f"📊 最近 {days} 天的 Token 使用统计：") + for img_name in ["line_chart", "bar_chart", "pie_chart", "table"]: + img_path = img_dir / f"stats_{img_name}.png" + if await io.is_file(img_path): + message = await self.build_private_stats_image_message( + img_path, + inline_base64=is_webui_session, + ) + await _send_private(message) + + await _send_private(self._build_stats_summary_text(summary)) + if ai_analysis: + await _send_private(f"🤖 AI 智能分析\n{ai_analysis}") + except Exception as e: + error_id = uuid4().hex[:8] + logger.exception( + "[Stats] 私聊统计生成失败: error_id=%s user=%s err=%s", + error_id, + user_id, + e, + ) + await _send_private( + f"❌ 生成统计图表失败，请稍后重试（错误码: {error_id}）" + ) + finally: + if img_dir is not None: + await io.delete_tree(img_dir) + + async def build_private_stats_image_message( + self, + image_path: Path, + *, + inline_base64: bool, + ) -> str: + file_uri = image_path.absolute().as_uri() + if not inline_base64: + return f"[CQ:image,file={file_uri}]" + + try: + content = await io.read_bytes(image_path) + encoded = base64.b64encode(content).decode("ascii") + except Exception as exc: + logger.warning( + "[Stats] 图像 base64 编码失败，回退文件路径: path=%s err=%s", + file_uri, + exc, + ) + return f"[CQ:image,file={file_uri}]" + + return f"[CQ:image,file=base64://{encoded}]" + + async def _run_stats_ai_analysis( + self, + *, + scope: str, + scope_id: int, + sender_id: int, + summary: dict[str, Any], + days: int, + ) -> str: + if not self.queue_manager: + return "" + + data_summary = self._build_data_summary(summary, days) + request_id = uuid4().hex + analysis_event = asyncio.Event() + self._stats_analysis_events[request_id] = analysis_event + request_data = { + "type": "stats_analysis", + "group_id": scope_id, + "request_id": request_id, + "sender_id": sender_id, + "data_summary": data_summary, + "summary": summary, + "days": days, + "scope": scope, + } + receipt = await self.queue_manager.add_group_mention_request( + request_data, model_name=self.config.chat_model.model_name + ) + logger.info("[Stats] 已投递 AI 分析请求: scope=%s target=%s", scope, scope_id) + + wait_timeout = compute_queued_llm_timeout_seconds( + self.ai.runtime_config, + self.config.chat_model, + retry_count=resolve_effective_retry_count( + self.ai.runtime_config, self.queue_manager + ), + initial_wait_seconds=float( + getattr(receipt, "estimated_wait_seconds", 0.0) or 0.0 + ), + ) + try: + await asyncio.wait_for(analysis_event.wait(), timeout=wait_timeout) + ai_analysis = self._stats_analysis_results.pop(request_id, "") + logger.info( + "[Stats] 已获取 AI 分析结果: scope=%s len=%s", scope, len(ai_analysis) + ) + return ai_analysis + except asyncio.TimeoutError: + logger.warning( + "[Stats] AI 分析超时: scope=%s target=%s timeout=%.1fs", + scope, + scope_id, + wait_timeout, + ) + return "AI 分析在当前动态等待期限内超时。" + finally: + self._stats_analysis_events.pop(request_id, None) + self._stats_analysis_results.pop(request_id, None) + + def _build_data_summary(self, summary: dict[str, Any], days: int) -> str: + """构建用于 AI 分析的统计数据摘要""" + lines = [] + lines.append("📊 Token 使用综合分析数据：") + lines.append("") + + # 整体概况 + lines.append("【整体概况】") + lines.append(f"统计周期: {days} 天") + lines.append(f"总调用次数: {summary['total_calls']}") + lines.append(f"总 Token 消耗: {summary['total_tokens']:,}") + lines.append(f"平均响应时间: {summary['avg_duration']:.2f}s") + lines.append(f"涉及模型数: {len(summary['models'])}") + lines.append("") + + # 时间维度 + daily_stats = summary.get("daily_stats", {}) + if daily_stats: + dates = sorted(daily_stats.keys()) + total_daily_calls = sum(daily_stats[d]["calls"] for d in dates) + total_daily_tokens = sum(daily_stats[d]["tokens"] for d in dates) + avg_daily_calls = total_daily_calls / len(dates) if dates else 0 + avg_daily_tokens = total_daily_tokens / len(dates) if dates else 0 + + # 找出高峰日 + peak_day = ( + max(dates, key=lambda d: daily_stats[d]["tokens"]) if dates else "" + ) + peak_day_tokens = daily_stats[peak_day]["tokens"] if peak_day else 0 + + lines.append("【时间维度】") + lines.append(f"统计天数: {len(dates)} 天") + lines.append(f"每日平均调用: {avg_daily_calls:.1f} 次") + lines.append(f"每日平均 Token: {avg_daily_tokens:,.0f} 个") + lines.append(f"高峰日期: {peak_day} ({peak_day_tokens:,} tokens)") + lines.append("") + + # 模型维度 + models = summary.get("models", {}) + if models: + lines.append("【模型维度】") + total_tokens_all = summary["total_tokens"] + sorted_models = sorted( + models.items(), key=lambda x: x[1]["tokens"], reverse=True + ) + for model_name, model_data in sorted_models[:_STATS_MODEL_TOP_N]: + calls = model_data["calls"] + tokens = model_data["tokens"] + prompt_tokens = model_data["prompt_tokens"] + completion_tokens = model_data["completion_tokens"] + token_pct = ( + (tokens / total_tokens_all * 100) if total_tokens_all > 0 else 0 + ) + avg_per_call = tokens / calls if calls > 0 else 0 + io_ratio = completion_tokens / prompt_tokens if prompt_tokens > 0 else 0 + + lines.append(f"模型: {model_name}") + lines.append( + f" - 调用次数: {calls} ({calls / summary['total_calls'] * 100:.1f}%)" + ) + lines.append(f" - Token 消耗: {tokens:,} ({token_pct:.1f}%)") + lines.append(f" - 平均每次调用: {avg_per_call:.0f} tokens") + lines.append( + f" - 输入: {prompt_tokens:,} / 输出: {completion_tokens:,}" + ) + lines.append(f" - 输入/输出比: 1:{io_ratio:.2f}") + lines.append("") + + if len(sorted_models) > _STATS_MODEL_TOP_N: + others = sorted_models[_STATS_MODEL_TOP_N:] + others_calls = sum(int(item[1].get("calls", 0)) for item in others) + others_tokens = sum(int(item[1].get("tokens", 0)) for item in others) + others_pct = ( + (others_tokens / total_tokens_all * 100) + if total_tokens_all > 0 + else 0.0 + ) + lines.append( + f"其余 {len(others)} 个模型合计: 调用 {others_calls} 次, Token {others_tokens:,} ({others_pct:.1f}%)" + ) + lines.append("") + + # 调用类型维度 + call_types = summary.get("call_types", {}) + if call_types: + lines.append("【调用类型维度】") + sorted_types = sorted( + call_types.items(), key=lambda item: int(item[1]), reverse=True + ) + total_calls = max(1, int(summary.get("total_calls", 0))) + for call_type, count in sorted_types[:_STATS_CALL_TYPE_TOP_N]: + ratio = int(count) / total_calls * 100 + lines.append(f"- {call_type}: {count} 次 ({ratio:.1f}%)") + if len(sorted_types) > _STATS_CALL_TYPE_TOP_N: + rest_count = sum( + int(item[1]) for item in sorted_types[_STATS_CALL_TYPE_TOP_N:] + ) + ratio = rest_count / total_calls * 100 + lines.append( + f"- 其他 {len(sorted_types) - _STATS_CALL_TYPE_TOP_N} 类: {rest_count} 次 ({ratio:.1f}%)" + ) + lines.append("") + + # 效率指标 + prompt_tokens = summary.get("prompt_tokens", 0) + completion_tokens = summary.get("completion_tokens", 0) + total_tokens = summary.get("total_tokens", 0) + input_ratio = (prompt_tokens / total_tokens * 100) if total_tokens > 0 else 0 + output_ratio = ( + (completion_tokens / total_tokens * 100) if total_tokens > 0 else 0 + ) + output_per_input = completion_tokens / prompt_tokens if prompt_tokens > 0 else 0 + + lines.append("【效率指标】") + lines.append(f"输入 Token: {prompt_tokens:,} ({input_ratio:.1f}%)") + lines.append(f"输出 Token: {completion_tokens:,} ({output_ratio:.1f}%)") + lines.append(f"输入/输出比: 1:{output_per_input:.2f}") + lines.append("") + + # 趋势分析 + if daily_stats and len(daily_stats) > 1: + lines.append("【趋势分析】") + dates = sorted(daily_stats.keys()) + first_day_tokens = daily_stats[dates[0]]["tokens"] + last_day_tokens = daily_stats[dates[-1]]["tokens"] + trend_change = ( + ((last_day_tokens - first_day_tokens) / first_day_tokens * 100) + if first_day_tokens > 0 + else 0 + ) + trend_desc = "增长" if trend_change > 0 else "下降" + lines.append( + f"总体趋势: {trend_desc} {abs(trend_change):.1f}% (从首日到末日)" + ) + lines.append("") + + summary_text = "\n".join(lines) + if len(summary_text) > _STATS_DATA_SUMMARY_MAX_CHARS: + trimmed = summary_text[: _STATS_DATA_SUMMARY_MAX_CHARS - 80].rstrip() + summary_text = ( + f"{trimmed}\n\n[数据摘要已截断，总长度 {len(summary_text)} 字符，" + f"仅保留前 {_STATS_DATA_SUMMARY_MAX_CHARS} 字符]" + ) + logger.info( + "[Stats] 数据摘要过长已截断: original_len=%s max_len=%s", + len("\n".join(lines)), + _STATS_DATA_SUMMARY_MAX_CHARS, + ) + return summary_text + + def _build_stats_summary_text(self, summary: dict[str, Any]) -> str: + return f"""📈 摘要汇总: + • 总调用次数: {summary["total_calls"]} + • 总消耗 Tokens: {summary["total_tokens"]:,} + └─ 输入: {summary["prompt_tokens"]:,} + └─ 输出: {summary["completion_tokens"]:,} + • 平均耗时: {summary["avg_duration"]:.2f}s + • 涉及模型数: {len(summary["models"])}""" + + def set_stats_analysis_result( + self, group_id: int, request_id: str, analysis: str + ) -> None: + """设置 AI 分析结果（由队列处理器调用）""" + event = self._stats_analysis_events.get(request_id) + if not event: + logger.warning( + "[StatsAnalysis] 未找到等待事件，群: %s, 请求: %s", + group_id, + request_id, + ) + return + self._stats_analysis_results[request_id] = analysis + event.set() + + async def _create_stats_render_dir(self) -> Path: + + base_dir = await io.ensure_dir(RENDER_CACHE_DIR) + return await io.ensure_dir(base_dir / f"stats_{uuid4().hex}") + + async def _generate_stats_charts( + self, + summary: dict[str, Any], + img_dir: Path, + days: int, + ) -> None: + async with self._stats_render_lock: + await asyncio.to_thread( + self._generate_stats_charts_sync, + summary, + img_dir, + days, + ) + + def _generate_stats_charts_sync( + self, + summary: dict[str, Any], + img_dir: Path, + days: int, + ) -> None: + self._generate_line_chart(summary, img_dir, days) + self._generate_bar_chart(summary, img_dir) + self._generate_pie_chart(summary, img_dir) + self._generate_stats_table(summary, img_dir) + + async def _build_stats_forward_nodes( + self, + summary: dict[str, Any], + img_dir: Path, + days: int, + ai_analysis: str = "", + ) -> list[dict[str, Any]]: + """构建用于合并转发的统计图表节点列表""" + nodes = [] + bot_qq = str(self.config.bot_qq) + + # 辅助函数：创建消息节点 + def add_node(content: str) -> None: + nodes.append( + { + "type": "node", + "data": {"name": "Bot", "uin": bot_qq, "content": content}, + } + ) + + add_node(f"📊 最近 {days} 天的 Token 使用统计：") + + # 添加所有生成的图片 + for img_name in ["line_chart", "bar_chart", "pie_chart", "table"]: + img_path = img_dir / f"stats_{img_name}.png" + if await io.is_file(img_path): + add_node(f"[CQ:image,file={img_path.absolute().as_uri()}]") + + # 添加文本摘要 + add_node(self._build_stats_summary_text(summary)) + + # 添加 AI 分析结果（如果有） + if ai_analysis: + add_node(f"🤖 AI 智能分析\n{ai_analysis}") + + return nodes + + def _generate_line_chart( + self, summary: dict[str, Any], img_dir: Path, days: int + ) -> None: + """生成折线图：时间趋势""" + daily_stats = summary["daily_stats"] + if not daily_stats: + return + + # 准备数据 + dates = sorted(daily_stats.keys()) + tokens = [daily_stats[d]["tokens"] for d in dates] + prompt_tokens = [daily_stats[d]["prompt_tokens"] for d in dates] + completion_tokens = [daily_stats[d]["completion_tokens"] for d in dates] + + # 创建图表 + fig, ax = plt.subplots(figsize=(12, 7)) + + # 绘制折线 + ax.plot( + dates, tokens, marker="o", linewidth=2, label="Total Token", color="#2196F3" + ) + ax.plot( + dates, + prompt_tokens, + marker="s", + linewidth=2, + label="Input Token", + color="#4CAF50", + ) + ax.plot( + dates, + completion_tokens, + marker="^", + linewidth=2, + label="Output Token", + color="#FF9800", + ) + + # 设置标题和标签 + ax.set_title( + f"Token Usage Trend for Last {days} Days", fontsize=16, fontweight="bold" + ) + ax.set_xlabel("Date", fontsize=12) + ax.set_ylabel("Token Count", fontsize=12) + ax.legend(loc="upper left", fontsize=10) + ax.grid(True, alpha=0.3) + + # 旋转 x 轴标签 + plt.xticks(rotation=45, ha="right") + + # 调整布局 + plt.tight_layout() + + # 保存图表 + filepath = img_dir / "stats_line_chart.png" + plt.savefig(filepath, dpi=150, bbox_inches="tight") + plt.close(fig) + + def _generate_bar_chart(self, summary: dict[str, Any], img_dir: Path) -> None: + """生成柱状图：模型对比""" + models = summary["models"] + if not models: + return + + # 准备数据 + model_names = list(models.keys()) + tokens = [models[m]["tokens"] for m in model_names] + prompt_tokens = [models[m]["prompt_tokens"] for m in model_names] + completion_tokens = [models[m]["completion_tokens"] for m in model_names] + + # 创建图表 + fig, ax = plt.subplots(figsize=(14, 8)) + + # 设置柱状图位置 + x = range(len(model_names)) + width = 0.25 + + # 绘制柱状图 + bars1 = ax.bar( + [i - width for i in x], + tokens, + width, + label="Total Token", + color="#2196F3", + alpha=0.8, + ) + bars2 = ax.bar( + x, + prompt_tokens, + width, + label="Input Token", + color="#4CAF50", + alpha=0.8, + ) + bars3 = ax.bar( + [i + width for i in x], + completion_tokens, + width, + label="Output Token", + color="#FF9800", + alpha=0.8, + ) + + # 设置标题和标签 + ax.set_title("Token Usage Comparison by Model", fontsize=16, fontweight="bold") + ax.set_xlabel("Model", fontsize=12) + ax.set_ylabel("Token Count", fontsize=12) + ax.set_xticks(x) + ax.set_xticklabels(model_names, rotation=45, ha="right") + ax.legend(loc="upper right", fontsize=10) + ax.grid(True, alpha=0.3, axis="y") + + # 在柱子上添加数值标签 + for bars in [bars1, bars2, bars3]: + for bar in bars: + height = bar.get_height() + if height > 0: + ax.text( + bar.get_x() + bar.get_width() / 2.0, + height, + f"{int(height):,}", + ha="center", + va="bottom", + fontsize=8, + ) + + # 调整布局 + plt.tight_layout() + + # 保存图表 + filepath = img_dir / "stats_bar_chart.png" + plt.savefig(filepath, dpi=150, bbox_inches="tight") + plt.close(fig) + + def _generate_pie_chart(self, summary: dict[str, Any], img_dir: Path) -> None: + """生成饼图：输入/输出比例""" + prompt_tokens = summary["prompt_tokens"] + completion_tokens = summary["completion_tokens"] + + if prompt_tokens == 0 and completion_tokens == 0: + return + + # 创建图表 + fig, ax = plt.subplots(figsize=(12, 8)) + + # 准备数据 + labels = ["Input Token", "Output Token"] + sizes = [prompt_tokens, completion_tokens] + colors = ["#4CAF50", "#FF9800"] + explode = (0.05, 0.05) # 突出显示 + + # 绘制饼图 + wedges, *_ = ax.pie( + sizes, + explode=explode, + labels=labels, + colors=colors, + autopct="%1.1f%%", + startangle=90, + textprops={"fontsize": 12}, + ) + + # 设置标题 + ax.set_title("Input/Output Token Ratio", fontsize=16, fontweight="bold", pad=20) + + # 添加图例 + ax.legend( + wedges, + [f"{labels[i]}: {sizes[i]:,}" for i in range(len(labels))], + loc="center left", + bbox_to_anchor=(1, 0, 0.5, 1), + fontsize=10, + ) + + # 调整布局 + plt.tight_layout() + + # 保存图表 + filepath = img_dir / "stats_pie_chart.png" + plt.savefig(filepath, dpi=150, bbox_inches="tight") + plt.close(fig) + + def _generate_stats_table(self, summary: dict[str, Any], img_dir: Path) -> None: + """生成统计表格""" + models = summary["models"] + if not models: + return + + # 准备数据 + model_names = list(models.keys()) + data = [] + for model in model_names: + m = models[model] + data.append( + [ + model, + m["calls"], + f"{m['tokens']:,}", + f"{m['prompt_tokens']:,}", + f"{m['completion_tokens']:,}", + ] + ) + + # 创建图表 + fig, ax = plt.subplots(figsize=(14, 9)) + ax.axis("tight") + ax.axis("off") + + # 创建表格 + table = ax.table( + cellText=data, + colLabels=["Model", "Calls", "Total Token", "Input Token", "Output Token"], + cellLoc="center", + loc="center", + ) + + # 设置表格样式 + table.auto_set_font_size(False) + table.set_fontsize(10) + table.scale(1.2, 1.5) + + # 设置表头样式 + for i in range(5): + table[(0, i)].set_facecolor("#2196F3") + table[(0, i)].set_text_props(weight="bold", color="white") + + # 设置行样式 + for i in range(1, len(data) + 1): + for j in range(5): + if i % 2 == 0: + table[(i, j)].set_facecolor("#f0f0f0") + + # 设置标题 + ax.set_title( + "Model Usage Statistics Details", fontsize=16, fontweight="bold", pad=20 + ) + + # 调整布局 + plt.tight_layout() + + # 保存图表 + filepath = img_dir / "stats_table.png" + plt.savefig(filepath, dpi=150, bbox_inches="tight") + plt.close(fig) diff --git a/src/Undefined/skills/commands/bugfix/handler.py b/src/Undefined/skills/commands/bugfix/handler.py index fbfe4573..5be456c7 100644 --- a/src/Undefined/skills/commands/bugfix/handler.py +++ b/src/Undefined/skills/commands/bugfix/handler.py @@ -6,4 +6,4 @@ async def execute(args: list[str], context: CommandContext) -> None: """处理 /bugfix。""" - await context.dispatcher._handle_bugfix(context.group_id, context.sender_id, args) + await context.dispatcher.handle_bugfix(context.group_id, context.sender_id, args) diff --git a/src/Undefined/skills/commands/stats/handler.py b/src/Undefined/skills/commands/stats/handler.py index 431e4c33..b91f27c5 100644 --- a/src/Undefined/skills/commands/stats/handler.py +++ b/src/Undefined/skills/commands/stats/handler.py @@ -23,7 +23,7 @@ async def _send_message(message: str) -> None: callable(send_forward), ) ) - await context.dispatcher._handle_stats_private( + await context.dispatcher.handle_stats_private( user_id, context.sender_id, args, @@ -39,4 +39,4 @@ async def _send_message(message: str) -> None: ) return - await context.dispatcher._handle_stats(context.group_id, context.sender_id, args) + await context.dispatcher.handle_stats(context.group_id, context.sender_id, args) diff --git a/tests/test_stats_handler_scope.py b/tests/test_stats_handler_scope.py index 873ae175..6c6e178c 100644 --- a/tests/test_stats_handler_scope.py +++ b/tests/test_stats_handler_scope.py @@ -14,12 +14,12 @@ def __init__(self) -> None: self.group_calls: list[tuple[int, int, list[str]]] = [] self.private_calls: list[tuple[int, int, list[str], bool, bool, bool]] = [] - async def _handle_stats( + async def handle_stats( self, group_id: int, sender_id: int, args: list[str] ) -> None: self.group_calls.append((group_id, sender_id, list(args))) - async def _handle_stats_private( + async def handle_stats_private( self, user_id: int, sender_id: int, diff --git a/tests/test_stats_private_delivery.py b/tests/test_stats_private_delivery.py index f120b696..de2d8c55 100644 --- a/tests/test_stats_private_delivery.py +++ b/tests/test_stats_private_delivery.py @@ -12,7 +12,7 @@ UnsupportedCapabilityError, ) -import Undefined.services.command as command_module +import Undefined.services.stats_command as stats_command_module from Undefined.services.command import CommandDispatcher from Undefined.utils import io as async_io @@ -74,12 +74,12 @@ async def test_private_stats_uses_one_forward_delivery( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(command_module, "_MATPLOTLIB_AVAILABLE", True) + monkeypatch.setattr(stats_command_module, "_MATPLOTLIB_AVAILABLE", True) dispatcher, render_dir = await _dispatcher(tmp_path) send_message = AsyncMock() send_forward = AsyncMock() - await dispatcher._handle_stats_private( + await dispatcher.handle_stats_private( 12345, 12345, ["7d"], @@ -101,7 +101,7 @@ async def test_group_stats_waits_for_analysis_before_chart_rendering( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(command_module, "_MATPLOTLIB_AVAILABLE", True) + monkeypatch.setattr(stats_command_module, "_MATPLOTLIB_AVAILABLE", True) dispatcher, render_dir = await _dispatcher(tmp_path) events: list[str] = [] @@ -126,7 +126,7 @@ async def generate_charts( dynamic_dispatcher._build_stats_forward_nodes = AsyncMock(return_value=[]) dynamic_dispatcher._send_group_forward_message = AsyncMock() - await dispatcher._handle_stats(10000, 12345, ["7d", "--ai"]) + await dispatcher.handle_stats(10000, 12345, ["7d", "--ai"]) assert events == ["analysis", "render"] assert not await async_io.exists(render_dir) @@ -136,11 +136,11 @@ async def test_private_stats_callback_only_channel_sends_chart_sequence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(command_module, "_MATPLOTLIB_AVAILABLE", True) + monkeypatch.setattr(stats_command_module, "_MATPLOTLIB_AVAILABLE", True) dispatcher, render_dir = await _dispatcher(tmp_path) send_message = AsyncMock() - await dispatcher._handle_stats_private( + await dispatcher.handle_stats_private( 12345, 12345, ["7d"], @@ -160,12 +160,12 @@ async def test_private_stats_definitive_rejection_keeps_text_summary( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(command_module, "_MATPLOTLIB_AVAILABLE", True) + monkeypatch.setattr(stats_command_module, "_MATPLOTLIB_AVAILABLE", True) dispatcher, render_dir = await _dispatcher(tmp_path) send_message = AsyncMock() send_forward = AsyncMock(side_effect=UnsupportedCapabilityError("item_list")) - await dispatcher._handle_stats_private( + await dispatcher.handle_stats_private( 12345, 12345, ["7d"], @@ -194,12 +194,12 @@ async def test_private_stats_ambiguous_failure_does_not_send_fallback( monkeypatch: pytest.MonkeyPatch, error: Exception, ) -> None: - monkeypatch.setattr(command_module, "_MATPLOTLIB_AVAILABLE", True) + monkeypatch.setattr(stats_command_module, "_MATPLOTLIB_AVAILABLE", True) dispatcher, render_dir = await _dispatcher(tmp_path) send_message = AsyncMock() send_forward = AsyncMock(side_effect=error) - await dispatcher._handle_stats_private( + await dispatcher.handle_stats_private( 12345, 12345, ["7d"], diff --git a/tests/test_stats_private_images.py b/tests/test_stats_private_images.py index 1a25be16..1ae4fed1 100644 --- a/tests/test_stats_private_images.py +++ b/tests/test_stats_private_images.py @@ -15,7 +15,7 @@ async def test_build_private_stats_image_message_uses_base64_when_requested( image = tmp_path / "stats.png" image.write_bytes(b"\x89PNG\r\n\x1a\n") - message = await dispatcher._build_private_stats_image_message( + message = await dispatcher.build_private_stats_image_message( image, inline_base64=True, ) @@ -32,7 +32,7 @@ async def test_build_private_stats_image_message_uses_path_for_normal_private( image = tmp_path / "stats.png" image.write_bytes(b"fake") - message = await dispatcher._build_private_stats_image_message( + message = await dispatcher.build_private_stats_image_message( image, inline_base64=False, ) From 53e8cfb2bc9f372431c21fb28a53fc7f4f9937b0 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 17:00:32 +0800 Subject: [PATCH 18/30] =?UTF-8?q?test(budget):=20=E4=B8=BA=E6=BA=90?= =?UTF-8?q?=E7=A0=81=E5=AD=97=E7=AC=A6=E4=B8=B2=E6=96=AD=E8=A8=80=E5=BB=BA?= =?UTF-8?q?=E7=AB=8B=E5=8F=AA=E5=87=8F=E4=B8=8D=E5=A2=9E=E7=9A=84=E9=A2=84?= =?UTF-8?q?=E7=AE=97=E6=A3=98=E8=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_webui_runtime_chat_frontend.py 一类「读源码 + assert 子串」的写法是 变更检测器（全仓约 800 处，该文件占 537）；一次性改写为行为断言体量过大， 先建立 tests/test_source_assertion_budget.py 预算棘轮： - AST 统计各测试文件里对源码变量的字符串比较总数，超过预算即失败； - 新增此类断言会被拦截，改成行为断言（node+vm / Vitest / 断言解析结构） 后应顺带调低预算； - 在该测试文件头部写明迁移路径，标记为历史遗留、勿继续扩张。 --- tests/test_source_assertion_budget.py | 90 +++++++++++++++++++++++ tests/test_webui_runtime_chat_frontend.py | 13 ++++ 2 files changed, 103 insertions(+) create mode 100644 tests/test_source_assertion_budget.py diff --git a/tests/test_source_assertion_budget.py b/tests/test_source_assertion_budget.py new file mode 100644 index 00000000..eff39972 --- /dev/null +++ b/tests/test_source_assertion_budget.py @@ -0,0 +1,90 @@ +"""源码字符串断言预算（棘轮）。 + +仓库里存在一类“变更检测器”测试：读取前端/资源源码文本，然后 +``assert "xxx" in source``。这类断言在重构时必然变红，而真正的行为回归 +却测不出来（见 ``tests/test_webui_runtime_chat_frontend.py``）。 + +正确的写法是行为断言： + +- WebUI 脚本：用 node + ``vm`` 执行真实 JS 再断言行为， + 参考 ``tests/test_webui_config_form_frontend.py``； +- 原生 App：把断言迁到各 App 自己的 Vitest / cargo 测试里； +- Python 侧资源契约：尽量断言解析后的结构（如 JSON/TOML 字段），而不是原文子串。 + +历史存量一次性清不完，这里用预算棘轮收敛：**全仓源码字符串断言总数不得超过 +``_BUDGET``**。新增断言会让测试失败；把断言改成行为测试后，请顺带调低预算， +让这个数字只减不增。 +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +_TESTS_DIR = Path(__file__).resolve().parent + +# 当前存量约 800（其中 test_webui_runtime_chat_frontend.py 一个文件占 500+）。 +# 只允许下降：新增源码字符串断言会失败。 +_BUDGET = 800 + + +def _collect_source_vars(tree: ast.Module) -> set[str]: + source_vars: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + value = node.value + calls = ( + [value] + if isinstance(value, ast.Call) + else [c for c in ast.walk(value) if isinstance(c, ast.Call)] + ) + for call in calls: + func = call.func + if (isinstance(func, ast.Name) and func.id == "_read_source") or ( + isinstance(func, ast.Attribute) and func.attr == "read_text" + ): + for target in node.targets: + if isinstance(target, ast.Name): + source_vars.add(target.id) + # 源码变量的切片/拼接结果仍视为源码变量 + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Subscript): + base = node.value.value + while isinstance(base, ast.Subscript): + base = base.value + if isinstance(base, ast.Name) and base.id in source_vars: + for target in node.targets: + if isinstance(target, ast.Name): + source_vars.add(target.id) + return source_vars + + +def _count_source_string_comparisons(path: Path) -> int: + tree = ast.parse(path.read_text(encoding="utf-8")) + source_vars = _collect_source_vars(tree) + count = 0 + for node in ast.walk(tree): + if not isinstance(node, ast.Compare) or not isinstance(node.left, ast.Constant): + continue + for comparator in node.comparators: + if isinstance(comparator, ast.Name) and comparator.id in source_vars: + count += 1 + return count + + +def test_source_string_assertions_stay_within_budget() -> None: + per_file: dict[str, int] = {} + for path in sorted(_TESTS_DIR.glob("*.py")): + if path.name == Path(__file__).name: + continue + count = _count_source_string_comparisons(path) + if count: + per_file[path.relative_to(_TESTS_DIR).as_posix()] = count + + total = sum(per_file.values()) + top = sorted(per_file.items(), key=lambda kv: -kv[1])[:5] + assert total <= _BUDGET, ( + f"源码字符串断言总数 {total} 超出预算 {_BUDGET}。\n" + "请改为行为断言：node + vm 执行真实 JS（参考 test_webui_config_form_frontend.py）" + "或 App 内的 Vitest / cargo 测试；解析结构化资源时断言解析结果而非原文子串。\n" + "当前最多的文件：\n " + "\n ".join(f"{name}: {count}" for name, count in top) + ) diff --git a/tests/test_webui_runtime_chat_frontend.py b/tests/test_webui_runtime_chat_frontend.py index 3e9975a9..8ff07841 100644 --- a/tests/test_webui_runtime_chat_frontend.py +++ b/tests/test_webui_runtime_chat_frontend.py @@ -1,3 +1,16 @@ +"""WebUI / Chat 前端静态契约测试（历史遗留，勿继续扩张）。 + +本文件大量使用「读取源码文本 + assert 子串」的写法，属于变更检测器： +重构必然变红，真正的行为回归却测不出来（全仓 ~800 处此类断言里本文件占 500+， +由 ``tests/test_source_assertion_budget.py`` 的预算棘轮看住总量，只减不增）。 + +新增前端契约时请写行为断言： + +- 用 node + ``vm`` 执行真实 JS（参考 ``tests/test_webui_config_form_frontend.py``）； +- 原生 App 的断言迁移到各自 App 的 Vitest / cargo 测试； +- 结构化资源（JSON/TOML）断言解析后的字段而非原文子串。 +""" + from __future__ import annotations import asyncio From b5488b418b61252ed1e806bdacdfcd67674f4685 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 19:27:00 +0800 Subject: [PATCH 19/30] chore(version): bump version to 3.15.0 --- CHANGELOG.md | 19 +++++++++++++++++++ apps/undefined-chat/package-lock.json | 4 ++-- apps/undefined-chat/package.json | 2 +- apps/undefined-chat/src-tauri/Cargo.lock | 2 +- apps/undefined-chat/src-tauri/Cargo.toml | 2 +- apps/undefined-chat/src-tauri/tauri.conf.json | 2 +- apps/undefined-console/package-lock.json | 4 ++-- apps/undefined-console/package.json | 2 +- apps/undefined-console/src-tauri/Cargo.lock | 2 +- apps/undefined-console/src-tauri/Cargo.toml | 2 +- .../src-tauri/tauri.conf.json | 2 +- pyproject.toml | 2 +- src/Undefined/__init__.py | 2 +- uv.lock | 2 +- 14 files changed, 34 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43a52bb5..ae2dee66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## v3.15.0 按功能拆分 Embedding 配置与稳定性修复 + +本版本将嵌入模型配置按功能拆分，知识库、认知记忆与表情包可以共用默认模型或各自覆写连接、维度与指令；同时修复侧写并发合并丢失更新、Agent 技能无法加载、热更新静默失效、SIGTERM 停机缺失等多处影响长期运行的问题。 + +- `[models.embedding]` 保留为全局默认模型与默认参数，新增 `[models.embedding.features.]`（`knowledge` / `cognitive` / `memes`）按功能覆写：`use_default` 整体继承默认表，也可按字段覆写连接信息、`context_window_tokens`、`queue_interval_seconds`、`dimensions`、`query_instruction` / `document_instruction`，`request_params` 与默认表按键合并。WebUI 配置表单同步支持覆写段编辑，`use_proxy` 的三态语义（继承 / 启用 / 禁用）改为下拉选择。 +- 生效配置相同的功能复用同一个 Embedder 与发车队列，重排器全局共享，避免重复加载与重复排队。 +- 修复嵌入指令前缀被剥离的问题：query / document instruction 与文本直接拼接，`"passage: "` 这类带尾随空格的前缀现在原样保留。 +- 修复认知侧写并发合并丢失更新：同一实体的「读取 → LLM 改写 → 写入」整段互斥，后一个合并基于前一个已落盘的结果继续，不再互相覆盖。`ProfileStorage` 新增版本读取与恢复，恢复前自动把当前内容另存为新快照（恢复操作本身可回退）；新增 `scripts/restore_profile.py` 提供 list / show / restore 与 `--dry-run`，侧写回滚不再依赖手工复制文件。 +- 史官 worker 新增并发上限 `[cognitive.historian].max_concurrency`（默认 4），以信号量与在途计数双重约束，不再无上限并发。 +- 修复随包 Agent 的 handler 无法加载：handler 模块改按真实包路径导入，`code_delivery_agent` 等使用相对导入的 Agent 恢复可用；注册阶段即预导入全部 handler，失败项记录 `load_error` 并从 schema 中排除，主 AI 不再看到不可用的技能。 +- 配置热更新失败不再静默：更新改为步骤表逐项执行，单步异常不中断其余步骤，失败项与「未完全生效」汇总以 error 级日志输出；异步热更新任务与配置回调的异常均会被记录。嵌入 / 重排模型配置变更加入需重启提示，避免热重载静默无效。 +- 新增 SIGTERM 优雅停机：容器 / systemd / supervisor 停止时不再被直接终止并跳过落盘清理，SIGTERM 与 SIGINT 收敛到同一停机事件，取消连接任务并等待收敛。 +- 修复消息队列重试上限口径分叉：coordinator 与 QueueManager 各用一套重试上限，热更新后两者可分叉，导致等待方在仍会重试时被误判为失败、或重试已耗尽后干等到 480 秒超时；现统一由 `resolve_effective_retry_count` 计算并与等待超时预算同口径。 +- `scripts/reembed_cognitive.py` 支持维度变化迁移：检测到新旧向量维度不同时先读取全量记录、删除并重建同名 collection（沿用原索引元数据）后按新维度写回，记录不丢；`--dry-run` 不做任何写入。 +- 依赖与配置清理：crawl4ai 与 langchain-community 改为必需依赖，删除「未安装则降级」的静默回退（缺失时直接报错暴露环境问题）；移除零引用的死配置 `cognitive.historian.rewrite_max_retry` 与死依赖 imgkit、croniter。 +- CI 补齐治理：工作流收敛只读权限、并发取消与任务超时，前端行为测试真实执行而非静默跳过，新增 Python 3.11 / 3.13 兼容矩阵，测试开启 65% 覆盖率门禁。 + +--- + ## v3.14.0 OneBot 本地文件三模式传输 本版本为 Bot 本地文件新增统一传输层，支持 `local`、`url`、`stream` 三种发送方式并按投递快照热更新；默认保持 `local` 兼容旧部署，跨文件系统发送可显式启用 Runtime 临时链接或 NapCat Stream 分块上传。 diff --git a/apps/undefined-chat/package-lock.json b/apps/undefined-chat/package-lock.json index 4c938832..4ee6acd1 100644 --- a/apps/undefined-chat/package-lock.json +++ b/apps/undefined-chat/package-lock.json @@ -1,12 +1,12 @@ { "name": "undefined-chat", - "version": "3.14.0", + "version": "3.15.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "undefined-chat", - "version": "3.14.0", + "version": "3.15.0", "dependencies": { "@tauri-apps/api": "^2.3.0", "@tauri-apps/plugin-dialog": "^2.7.1", diff --git a/apps/undefined-chat/package.json b/apps/undefined-chat/package.json index d5cef4e4..e9c8e0ed 100644 --- a/apps/undefined-chat/package.json +++ b/apps/undefined-chat/package.json @@ -1,7 +1,7 @@ { "name": "undefined-chat", "private": true, - "version": "3.14.0", + "version": "3.15.0", "type": "module", "scripts": { "tauri": "tauri", diff --git a/apps/undefined-chat/src-tauri/Cargo.lock b/apps/undefined-chat/src-tauri/Cargo.lock index 4d67f414..572d809e 100644 --- a/apps/undefined-chat/src-tauri/Cargo.lock +++ b/apps/undefined-chat/src-tauri/Cargo.lock @@ -5431,7 +5431,7 @@ dependencies = [ [[package]] name = "undefined_chat" -version = "3.14.0" +version = "3.15.0" dependencies = [ "futures-util", "keyring", diff --git a/apps/undefined-chat/src-tauri/Cargo.toml b/apps/undefined-chat/src-tauri/Cargo.toml index d71d32d4..a3f3bc01 100644 --- a/apps/undefined-chat/src-tauri/Cargo.toml +++ b/apps/undefined-chat/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "undefined_chat" -version = "3.14.0" +version = "3.15.0" description = "Undefined native chat client" authors = ["Undefined contributors"] license = "MIT" diff --git a/apps/undefined-chat/src-tauri/tauri.conf.json b/apps/undefined-chat/src-tauri/tauri.conf.json index 780ce634..78684b91 100644 --- a/apps/undefined-chat/src-tauri/tauri.conf.json +++ b/apps/undefined-chat/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Undefined Chat", - "version": "3.14.0", + "version": "3.15.0", "identifier": "com.undefined.chat", "build": { "beforeDevCommand": "npm run dev", diff --git a/apps/undefined-console/package-lock.json b/apps/undefined-console/package-lock.json index d87ed904..1bfc09e7 100644 --- a/apps/undefined-console/package-lock.json +++ b/apps/undefined-console/package-lock.json @@ -1,12 +1,12 @@ { "name": "undefined-console", - "version": "3.14.0", + "version": "3.15.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "undefined-console", - "version": "3.14.0", + "version": "3.15.0", "dependencies": { "@tauri-apps/api": "^2.3.0", "@tauri-apps/plugin-http": "^2.3.0" diff --git a/apps/undefined-console/package.json b/apps/undefined-console/package.json index 177a760a..8dff5d9f 100644 --- a/apps/undefined-console/package.json +++ b/apps/undefined-console/package.json @@ -1,7 +1,7 @@ { "name": "undefined-console", "private": true, - "version": "3.14.0", + "version": "3.15.0", "type": "module", "scripts": { "tauri": "tauri", diff --git a/apps/undefined-console/src-tauri/Cargo.lock b/apps/undefined-console/src-tauri/Cargo.lock index 20cac311..7926eb47 100644 --- a/apps/undefined-console/src-tauri/Cargo.lock +++ b/apps/undefined-console/src-tauri/Cargo.lock @@ -4063,7 +4063,7 @@ checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "undefined_console" -version = "3.14.0" +version = "3.15.0" dependencies = [ "serde", "serde_json", diff --git a/apps/undefined-console/src-tauri/Cargo.toml b/apps/undefined-console/src-tauri/Cargo.toml index 22a09387..05988bb8 100644 --- a/apps/undefined-console/src-tauri/Cargo.toml +++ b/apps/undefined-console/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "undefined_console" -version = "3.14.0" +version = "3.15.0" description = "Undefined cross-platform management console" authors = ["Undefined contributors"] license = "MIT" diff --git a/apps/undefined-console/src-tauri/tauri.conf.json b/apps/undefined-console/src-tauri/tauri.conf.json index 7209eed8..1c7e6e8a 100644 --- a/apps/undefined-console/src-tauri/tauri.conf.json +++ b/apps/undefined-console/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Undefined Console", - "version": "3.14.0", + "version": "3.15.0", "identifier": "com.undefined.console", "build": { "beforeDevCommand": "npm run dev", diff --git a/pyproject.toml b/pyproject.toml index ba4cd282..2797b30e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "Undefined-bot" -version = "3.14.0" +version = "3.15.0" description = "QQ bot platform with cognitive memory architecture and multi-agent Skills, via OneBot V11." readme = "README.md" authors = [ diff --git a/src/Undefined/__init__.py b/src/Undefined/__init__.py index 86925bb5..984379c5 100644 --- a/src/Undefined/__init__.py +++ b/src/Undefined/__init__.py @@ -24,7 +24,7 @@ from .skills.registry import BaseRegistry as BaseRegistry from .skills.tools import ToolRegistry as ToolRegistry -__version__: str = "3.14.0" +__version__: str = "3.15.0" # symbol -> (module_path, attribute_name)；首次访问时才 importlib 加载 _LAZY_IMPORTS: dict[str, tuple[str, str]] = { diff --git a/uv.lock b/uv.lock index c3e54623..fe4028c3 100644 --- a/uv.lock +++ b/uv.lock @@ -4680,7 +4680,7 @@ wheels = [ [[package]] name = "undefined-bot" -version = "3.14.0" +version = "3.15.0" source = { editable = "." } dependencies = [ { name = "aiofiles" }, From c063e966ba7161a329eb5d687ac2a2ea36a2fc47 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 19:28:26 +0800 Subject: [PATCH 20/30] =?UTF-8?q?fix(chat):=20=E7=A7=BB=E9=99=A4=20Workspa?= =?UTF-8?q?ceLayout=20=E4=B8=AD=E5=A4=9A=E4=BD=99=E7=9A=84=20Fragment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/undefined-chat/src/App.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/undefined-chat/src/App.tsx b/apps/undefined-chat/src/App.tsx index 78f7a1a7..ceee074c 100644 --- a/apps/undefined-chat/src/App.tsx +++ b/apps/undefined-chat/src/App.tsx @@ -776,9 +776,5 @@ function WorkspaceLayout({ isDesktop: boolean; children: ReactNode; }) { - return isDesktop ? ( - {children} - ) : ( - <>{children} - ); + return isDesktop ? {children} : children; } From 4b4debfd61882bebba1f8e04daf338381f0ee2a6 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 19:55:03 +0800 Subject: [PATCH 21/30] =?UTF-8?q?fix(cognitive):=20=E4=BE=A7=E5=86=99?= =?UTF-8?q?=E9=94=81=E7=9A=84=E5=B9=B6=E5=8F=91=E9=A6=96=E6=AC=A1=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E6=94=B9=E4=B8=BA=20setdefault=20=E5=8E=9F=E5=AD=90?= =?UTF-8?q?=E6=8F=92=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit merge_guard / _get_lock 原先 if-not-in-then-create 在并发首次进入同一 新实体时会各自创建独立 Lock，后建覆盖先建，互斥形同虚设（读旧快照后 写覆盖）。改为 get + setdefault 双检：并发双方必然拿到同一把锁。 补并发首次进入的回归测试。 --- src/Undefined/cognitive/profile_storage.py | 19 ++++++++++------- tests/test_cognitive_profile_revision.py | 24 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/Undefined/cognitive/profile_storage.py b/src/Undefined/cognitive/profile_storage.py index 14f8e0ae..0722c9fe 100644 --- a/src/Undefined/cognitive/profile_storage.py +++ b/src/Undefined/cognitive/profile_storage.py @@ -27,21 +27,26 @@ def __init__(self, base_path: str | Path, revision_keep: int = 5) -> None: def _get_lock(self, entity_type: str, entity_id: str) -> asyncio.Lock: key = f"{entity_type}:{entity_id}" - if key not in self._locks: - self._locks[key] = asyncio.Lock() - return self._locks[key] + lock = self._locks.get(key) + if lock is None: + # dict.setdefault 原子插入：并发首次进入同一实体时双方拿到同一把锁， + # 不会出现各自建锁、后建覆盖先建导致互斥失效 + lock = self._locks.setdefault(key, asyncio.Lock()) + return lock def merge_guard(self, entity_type: str, entity_id: str) -> asyncio.Lock: """跨「读 → LLM → 写」整段侧写合并的互斥锁。 只串行化同一实体的合并周期，避免两个 job 各自基于旧快照改写后互相覆盖 （后写覆盖先写，先前的观察永久丢失）。与文件写入锁分开，避免与 - `write_profile` 的锁重入死锁。 + `write_profile` 的锁重入死锁。锁的创建经 `dict.setdefault` 原子完成， + 见 `_get_lock`。 """ key = f"{entity_type}:{entity_id}" - if key not in self._merge_locks: - self._merge_locks[key] = asyncio.Lock() - return self._merge_locks[key] + lock = self._merge_locks.get(key) + if lock is None: + lock = self._merge_locks.setdefault(key, asyncio.Lock()) + return lock def _profile_path(self, entity_type: str, entity_id: str) -> Path: return self._base / f"{entity_type}s" / f"{entity_id}.md" diff --git a/tests/test_cognitive_profile_revision.py b/tests/test_cognitive_profile_revision.py index a6ab577b..0a767c9a 100644 --- a/tests/test_cognitive_profile_revision.py +++ b/tests/test_cognitive_profile_revision.py @@ -88,6 +88,30 @@ async def merge(tag: str) -> None: assert await storage.read_profile("user", "10001") in {"a+b", "b+a"} +@pytest.mark.asyncio +async def test_merge_guard_concurrent_first_access_shares_one_lock( + tmp_path: Path, +) -> None: + """并发首次进入同一新实体时必须拿到同一把锁，不能各自建锁互相覆盖。""" + storage = ProfileStorage(tmp_path) + + async def grab(entity_id: str) -> asyncio.Lock: + return storage.merge_guard("user", entity_id) + + results = await asyncio.gather(*[grab(f"new-{i % 2}") for i in range(20)]) + # gather 保序：偶数位都是 new-0，奇数位都是 new-1 + assert all(lock is results[0] for lock in results[::2]) + assert all(lock is results[1] for lock in results[1::2]) + assert results[0] is not results[1] + assert len(storage._merge_locks) == 2 + assert len(storage._locks) == 0 + + write_locks = await asyncio.gather( + *[asyncio.to_thread(storage._get_lock, "user", "fresh") for _ in range(10)] + ) + assert all(lock is write_locks[0] for lock in write_locks) + + @pytest.mark.asyncio async def test_merge_profiles_holds_entity_merge_guard() -> None: events: list[str] = [] From 0a906834c17ddd404aaa6b543058def9866a88a7 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 19:56:30 +0800 Subject: [PATCH 22/30] =?UTF-8?q?fix(ai):=20crawl4ai=20=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E5=A4=B1=E8=B4=A5=E9=99=8D=E7=BA=A7=E4=B8=BA=E8=83=BD?= =?UTF-8?q?=E5=8A=9B=E7=BC=BA=E5=A4=B1=EF=BC=8C=E4=B8=8D=E5=86=8D=E8=AE=A9?= =?UTF-8?q?=20Bot=20=E5=90=AF=E5=8A=A8=E5=B4=A9=E6=BA=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit crawl4ai 仍是必需依赖，但安装损坏 / 版本不兼容时 ClientSetupMixin __init__ 直接调用能力探测会把 ImportError / RuntimeError 抛到启动 路径上。改为捕获后置 None 并以 error 日志暴露环境问题：网页获取 工具运行时自行报错，proxy 注入按不可用处理，其余功能照常启动。 --- src/Undefined/ai/client/ask_loop.py | 3 ++- src/Undefined/ai/client/setup.py | 27 +++++++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/Undefined/ai/client/ask_loop.py b/src/Undefined/ai/client/ask_loop.py index 66c0b8ae..3ef5f40d 100644 --- a/src/Undefined/ai/client/ask_loop.py +++ b/src/Undefined/ai/client/ask_loop.py @@ -324,7 +324,8 @@ async def emit_webchat_stage(stage: str, detail: Any | None = None) -> None: tool_context.setdefault("search_wrapper", self._search_wrapper) tool_context.setdefault( "crawl4ai_proxy_config_available", - self._crawl4ai_capabilities.proxy_config_available, + self._crawl4ai_capabilities is not None + and self._crawl4ai_capabilities.proxy_config_available, ) tool_context.setdefault("end_summary_storage", self._end_summary_storage) tool_context.setdefault("end_summaries", self._prompt_builder.end_summaries) diff --git a/src/Undefined/ai/client/setup.py b/src/Undefined/ai/client/setup.py index 99d0b863..985bb4cd 100644 --- a/src/Undefined/ai/client/setup.py +++ b/src/Undefined/ai/client/setup.py @@ -17,7 +17,10 @@ from Undefined.ai.model_selector import ModelSelector from Undefined.ai.multimodal import MultimodalAnalyzer from Undefined.ai.prompts import PromptBuilder -from Undefined.ai.crawl4ai_support import get_crawl4ai_capabilities +from Undefined.ai.crawl4ai_support import ( + Crawl4AICapabilities, + get_crawl4ai_capabilities, +) from Undefined.ai.summaries import SummaryService from Undefined.ai.tokens import TokenCounter from Undefined.ai.tool_search import TOOL_SEARCH_NAME @@ -162,7 +165,18 @@ def __init__( self.runtime_config = runtime_config self.memory_storage = memory_storage self._end_summary_storage = end_summary_storage or EndSummaryStorage() - self._crawl4ai_capabilities = get_crawl4ai_capabilities() + self._crawl4ai_capabilities: Crawl4AICapabilities | None + # crawl4ai 是必需依赖，但安装损坏 / 版本不兼容不应让整个 Bot 启动失败： + # 这里降级为能力缺失（网页获取工具运行时会明确报错），并以 error 日志暴露环境问题 + try: + self._crawl4ai_capabilities = get_crawl4ai_capabilities() + except Exception as exc: + self._crawl4ai_capabilities = None + logger.error( + "[初始化] crawl4ai 初始化失败，网页获取功能不可用，" + "请修复 crawl4ai 安装后重启: %s", + exc, + ) self._http_client = httpx.AsyncClient(timeout=480.0, trust_env=False) self._token_usage_storage = TokenUsageStorage() @@ -290,10 +304,11 @@ def __init__( else: logger.info("[初始化] SEARXNG_URL 未配置，搜索功能禁用") - logger.info( - "[初始化] crawl4ai 已就绪，网页获取功能已启用: proxy_config=%s", - self._crawl4ai_capabilities.proxy_config_available, - ) + if self._crawl4ai_capabilities is not None: + logger.info( + "[初始化] crawl4ai 已就绪，网页获取功能已启用: proxy_config=%s", + self._crawl4ai_capabilities.proxy_config_available, + ) self._prompt_builder = PromptBuilder( bot_qq=self.bot_qq, From 8698ef02eba2d9621cdaf6328e934a667513291d Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 19:57:17 +0800 Subject: [PATCH 23/30] =?UTF-8?q?docs(historian):=20=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E5=9C=A8=E9=80=94=E9=97=A8=E6=8E=A7=E4=B8=8E=E4=BF=A1=E5=8F=B7?= =?UTF-8?q?=E9=87=8F=E4=B8=A4=E5=B1=82=E5=B9=B6=E5=8F=91=E7=BA=A6=E6=9D=9F?= =?UTF-8?q?=E7=9A=84=E5=88=86=E5=B7=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _poll_loop 的在途计数限制任务对象数量（dequeue 暂停，防内存堆积）， _semaphore 约束实际并发处理；当前发车路径下两层上限一致、信号量 不会真正阻塞，保留它是为了约束未来绕过门控的调用。stop() 对两层 的收敛语义一并注明。 --- src/Undefined/cognitive/historian/worker.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Undefined/cognitive/historian/worker.py b/src/Undefined/cognitive/historian/worker.py index 3071b379..d9d87cb7 100644 --- a/src/Undefined/cognitive/historian/worker.py +++ b/src/Undefined/cognitive/historian/worker.py @@ -107,7 +107,10 @@ async def _poll_loop(self) -> None: float(config.poll_interval_seconds), ) if len(self._inflight_tasks) >= self._max_concurrency: - # 在途任务达到上限时先不取新任务，避免无界并发与内存堆积 + # 第一层约束（发车门控）：在途任务达到上限时先不取新任务。 + # 它限制的是「同时存在的任务对象数量」，让 dequeue 暂停， + # 避免任务堆积在内存里排队；与 _semaphore 互补，见 + # _process_job_with_retry 处的说明。 await asyncio.sleep(poll_interval) continue result = await self._job_queue.dequeue() @@ -150,7 +153,12 @@ async def _poll_loop(self) -> None: logger.info("[史官] 轮询循环已结束") async def _process_job_with_retry(self, job_id: str, job: dict[str, Any]) -> None: - # 并发上限由 _poll_loop 的在途计数与这里的信号量双重约束 + # 第二层约束（信号量）：与 _poll_loop 的在途计数门控互补。 + # 当前唯一发车路径是 _poll_loop，且两者上限同为 _max_concurrency， + # 因此正常情况下任务在信号量上不会真正阻塞；保留它是为了约束 + # 未来绕过发车门控的直接调用（如手动重放、管理接口触发）。 + # stop() 的收敛语义不受影响：两层上限一致，poll 退出后统一 + # gather 全部在途任务（含正在信号量上等待的任务）。 async with self._semaphore: await self._process_job_with_retry_inner(job_id, job) From 45b79c576bb59c088bc573c51fdb7544e5defa52 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 19:59:00 +0800 Subject: [PATCH 24/30] =?UTF-8?q?docs(config):=20=E6=98=8E=E7=A1=AE?= =?UTF-8?q?=E6=8C=87=E4=BB=A4=E5=89=8D=E7=BC=80=E7=9A=84=E7=A9=BA=E7=99=BD?= =?UTF-8?q?=E5=93=A8=E5=85=B5=E8=AF=AD=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 指令与文本直接拼接，_coerce_instruction 正确保留首尾空白，但纯空白 字符串与空串同样视为未设置、回落继承默认表。在 config.toml.example （默认表 + 三个功能覆写块）与 docs/configuration.md 中写明：空/纯空白 = 继承，覆写表无法表达显式清空，并给出「默认留空、按功能显式设置」 的替代写法。 --- config.toml.example | 44 ++++++++++++++++++++++++++++++------------- docs/configuration.md | 2 +- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/config.toml.example b/config.toml.example index 53d3c064..8b1abb97 100644 --- a/config.toml.example +++ b/config.toml.example @@ -710,8 +710,11 @@ queue_interval_seconds = 0.0 # zh: 向量维度（可选）。0 或留空表示使用模型默认维度。 # en: Embedding dimensions (optional). Use 0/empty to use model defaults. dimensions = 0 -# zh: 查询端指令前缀（可选，Qwen3/BGE 等模型常用）。 +# zh: 查询端指令前缀（可选，Qwen3/BGE 等模型常用）。指令与文本直接拼接， +# 首尾空白与换行原样保留；留空表示不使用指令。 # en: Query instruction prefix (optional, common for Qwen3/BGE-style models). +# The instruction is concatenated directly with the text, whitespace and +# newlines preserved; leave empty to disable. query_instruction = "" # zh: 文档端指令前缀（可选，E5 等模型常用，例如 "passage: "）。 # en: Document instruction prefix (optional, common for E5-style models, e.g. "passage: "). @@ -754,11 +757,16 @@ queue_interval_seconds = -1.0 # zh: 覆写向量维度；<0 表示继承默认配置，0 表示使用模型默认维度。 # en: Override embedding dimensions; <0 inherits the default, 0 uses the model default. dimensions = -1 -# zh: 覆写查询端指令前缀；空字符串表示继承默认配置。 -# en: Override the query instruction prefix; an empty string inherits the default. +# zh: 覆写查询端指令前缀；空字符串或纯空白都表示继承默认配置。覆写表无法表达 +# 「显式清空」——若默认表带前缀而本功能不需要，请把默认前缀留空、只在需要 +# 的功能上单独设置。 +# en: Override the query instruction prefix; an empty or whitespace-only string +# inherits the default. Overrides cannot express "explicitly empty" — if the +# default table has a prefix this feature does not need, keep the default +# empty and set the prefix only on the features that need it. query_instruction = "" -# zh: 覆写文档端指令前缀；空字符串表示继承默认配置。 -# en: Override the document instruction prefix; an empty string inherits the default. +# zh: 覆写文档端指令前缀；语义同 query_instruction（空/纯空白 = 继承）。 +# en: Override the document instruction prefix; same semantics as query_instruction (empty/whitespace = inherit). document_instruction = "" # zh: 覆写额外请求体参数，按 key 合并到默认 request_params 之上（同名以本表为准）。 @@ -791,11 +799,16 @@ queue_interval_seconds = -1.0 # zh: 覆写向量维度；<0 表示继承默认配置，0 表示使用模型默认维度。 # en: Override embedding dimensions; <0 inherits the default, 0 uses the model default. dimensions = -1 -# zh: 覆写查询端指令前缀；空字符串表示继承默认配置。 -# en: Override the query instruction prefix; an empty string inherits the default. +# zh: 覆写查询端指令前缀；空字符串或纯空白都表示继承默认配置。覆写表无法表达 +# 「显式清空」——若默认表带前缀而本功能不需要，请把默认前缀留空、只在需要 +# 的功能上单独设置。 +# en: Override the query instruction prefix; an empty or whitespace-only string +# inherits the default. Overrides cannot express "explicitly empty" — if the +# default table has a prefix this feature does not need, keep the default +# empty and set the prefix only on the features that need it. query_instruction = "" -# zh: 覆写文档端指令前缀；空字符串表示继承默认配置。 -# en: Override the document instruction prefix; an empty string inherits the default. +# zh: 覆写文档端指令前缀；语义同 query_instruction（空/纯空白 = 继承）。 +# en: Override the document instruction prefix; same semantics as query_instruction (empty/whitespace = inherit). document_instruction = "" # zh: 覆写额外请求体参数，按 key 合并到默认 request_params 之上（同名以本表为准）。 @@ -828,11 +841,16 @@ queue_interval_seconds = -1.0 # zh: 覆写向量维度；<0 表示继承默认配置，0 表示使用模型默认维度。 # en: Override embedding dimensions; <0 inherits the default, 0 uses the model default. dimensions = -1 -# zh: 覆写查询端指令前缀；空字符串表示继承默认配置。 -# en: Override the query instruction prefix; an empty string inherits the default. +# zh: 覆写查询端指令前缀；空字符串或纯空白都表示继承默认配置。覆写表无法表达 +# 「显式清空」——若默认表带前缀而本功能不需要，请把默认前缀留空、只在需要 +# 的功能上单独设置。 +# en: Override the query instruction prefix; an empty or whitespace-only string +# inherits the default. Overrides cannot express "explicitly empty" — if the +# default table has a prefix this feature does not need, keep the default +# empty and set the prefix only on the features that need it. query_instruction = "" -# zh: 覆写文档端指令前缀；空字符串表示继承默认配置。 -# en: Override the document instruction prefix; an empty string inherits the default. +# zh: 覆写文档端指令前缀；语义同 query_instruction（空/纯空白 = 继承）。 +# en: Override the document instruction prefix; same semantics as query_instruction (empty/whitespace = inherit). document_instruction = "" # zh: 覆写额外请求体参数，按 key 合并到默认 request_params 之上（同名以本表为准）。 diff --git a/docs/configuration.md b/docs/configuration.md index 834c046c..136b02bd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -531,7 +531,7 @@ document_instruction = "passage: " | `context_window_tokens` | `<=0` | 覆写上下文窗口上限 | | `queue_interval_seconds` | `<0` | `0` 表示请求到达立即发车 | | `dimensions` | `<0` | `0` 表示使用模型默认维度 | -| `query_instruction` / `document_instruction` | `""` | 覆写指令前缀；空字符串表示继承。如需“默认带前缀、个别功能不带”，请把默认前缀留空、只在需要的功能上单独设置 | +| `query_instruction` / `document_instruction` | `""` | 覆写指令前缀；空字符串**或纯空白**都表示继承。指令与文本直接拼接、首尾空白有意义；覆写表无法表达“显式清空”——如需“默认带前缀、个别功能不带”，请把默认前缀留空、只在需要的功能上单独设置 | | `[.request_params]` | 空表 | 按 key 合并到默认 `request_params` 之上，同名以本表为准 | 语义说明： From eb12d11ccfbe532665a4e9109f5195fb34b91d64 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 20:01:30 +0800 Subject: [PATCH 25/30] =?UTF-8?q?fix(skills):=20=E6=81=A2=E5=A4=8D?= =?UTF-8?q?=E5=90=84=E5=B7=A5=E5=85=B7=E8=AE=BF=E9=97=AE=E6=8E=A7=E5=88=B6?= =?UTF-8?q?=E6=96=87=E6=A1=88=E7=9A=84=E5=8E=9F=E5=A7=8B=E5=B7=AE=E5=BC=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit private_access_error 统一时把所有调用点都套上「发送失败：」前缀与 「已被访问控制拦截」后缀，改变了 LLM 看到的工具返回：表情反应本无 发送语义、文件类工具原不带拦截说明、code_delivery end 原为「上传 失败：」。为 helper 增加 access_note 参数并按调用点回传原参，恢复 各自合并前的文案。 --- .../agents/code_delivery_agent/tools/end/handler.py | 4 +++- src/Undefined/skills/shared.py | 12 ++++++++---- .../toolsets/messages/react_message_emoji/handler.py | 2 +- .../toolsets/messages/send_text_file/handler.py | 2 +- .../toolsets/messages/send_url_file/handler.py | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Undefined/skills/agents/code_delivery_agent/tools/end/handler.py b/src/Undefined/skills/agents/code_delivery_agent/tools/end/handler.py index 465b35f1..4e8f1b99 100644 --- a/src/Undefined/skills/agents/code_delivery_agent/tools/end/handler.py +++ b/src/Undefined/skills/agents/code_delivery_agent/tools/end/handler.py @@ -132,7 +132,9 @@ async def execute(args: dict[str, Any], context: dict[str, Any]) -> str: if target_type == "private" and not runtime_config.is_private_allowed( target_id ): - access_error = private_access_error(runtime_config, target_id) + access_error = private_access_error( + runtime_config, target_id, prefix="上传失败：", access_note="" + ) if access_error is not None: upload_status = access_error diff --git a/src/Undefined/skills/shared.py b/src/Undefined/skills/shared.py index a2df55e0..01e2eaad 100644 --- a/src/Undefined/skills/shared.py +++ b/src/Undefined/skills/shared.py @@ -15,23 +15,27 @@ def private_access_error( target_id: int, *, prefix: str = "发送失败：", + access_note: str = "，已被访问控制拦截", ) -> str: """按访问控制拒绝原因生成统一的用户可见说明。 读取 `runtime_config.private_access_denied_reason(target_id)`： - `blacklist` 表示命中 `access.blocked_private_ids`； - 其余情况（含 `allowlist` / 未配置）统一提示不在允许列表内。 + + `prefix` / `access_note` 用于保留各工具调用点的原有文案差异 + （如表情反应没有"发送失败"语义、文件类工具不带拦截说明）。 """ reason_getter = getattr(runtime_config, "private_access_denied_reason", None) reason = reason_getter(target_id) if callable(reason_getter) else None if reason == "blacklist": return ( - f"{prefix}目标用户 {target_id} 在黑名单内（access.blocked_private_ids），" - "已被访问控制拦截" + f"{prefix}目标用户 {target_id} 在黑名单内" + f"（access.blocked_private_ids）{access_note}" ) return ( - f"{prefix}目标用户 {target_id} 不在允许列表内（access.allowed_private_ids），" - "已被访问控制拦截" + f"{prefix}目标用户 {target_id} 不在允许列表内" + f"（access.allowed_private_ids）{access_note}" ) diff --git a/src/Undefined/skills/toolsets/messages/react_message_emoji/handler.py b/src/Undefined/skills/toolsets/messages/react_message_emoji/handler.py index 3447deda..d27370c2 100644 --- a/src/Undefined/skills/toolsets/messages/react_message_emoji/handler.py +++ b/src/Undefined/skills/toolsets/messages/react_message_emoji/handler.py @@ -299,7 +299,7 @@ def _validate_target_and_allowlist( if target_type == "group" and not runtime_config.is_group_allowed(target_id): return _group_access_error(runtime_config, target_id) if target_type == "private" and not runtime_config.is_private_allowed(target_id): - return private_access_error(runtime_config, target_id) + return private_access_error(runtime_config, target_id, prefix="") return None diff --git a/src/Undefined/skills/toolsets/messages/send_text_file/handler.py b/src/Undefined/skills/toolsets/messages/send_text_file/handler.py index fe76f016..30eee966 100644 --- a/src/Undefined/skills/toolsets/messages/send_text_file/handler.py +++ b/src/Undefined/skills/toolsets/messages/send_text_file/handler.py @@ -403,7 +403,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if target_type == "private" and not runtime_config.is_private_allowed( target_id ): - return private_access_error(runtime_config, target_id) + return private_access_error(runtime_config, target_id, access_note="") send_file_callable, history_recorded_by_sender, sender_error = ( _resolve_file_send_callable(context, target_type) diff --git a/src/Undefined/skills/toolsets/messages/send_url_file/handler.py b/src/Undefined/skills/toolsets/messages/send_url_file/handler.py index 6e55e52a..eb2e0044 100644 --- a/src/Undefined/skills/toolsets/messages/send_url_file/handler.py +++ b/src/Undefined/skills/toolsets/messages/send_url_file/handler.py @@ -398,7 +398,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if target_type == "private" and not runtime_config.is_private_allowed( target_id ): - return private_access_error(runtime_config, target_id) + return private_access_error(runtime_config, target_id, access_note="") send_file_callable, history_recorded_by_sender, sender_error = ( _resolve_file_send_callable(context, target_type) From dd16dbc4648a567b2f40484d7eee66736af6a207 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 20:04:00 +0800 Subject: [PATCH 26/30] =?UTF-8?q?fix(main):=20=E5=81=9C=E6=9C=BA=E4=BF=A1?= =?UTF-8?q?=E5=8F=B7=E5=A4=84=E7=90=86=E5=99=A8=E6=94=AF=E6=8C=81=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=EF=BC=8C=E9=81=BF=E5=85=8D=E6=B0=B8=E4=B9=85=E5=8A=AB?= =?UTF-8?q?=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit install_shutdown_signal_handlers 现返回 ShutdownSignalGuard（携带停机 事件），安装前保存原处理器，停机流程结束后调用 restore() 归还信号 控制权：loop 处理器先 remove_signal_handler，再 signal.signal 还原， 注册失败的信号不进入恢复名单。main() 在清理 finally 末尾恢复。 except KeyboardInterrupt 分支注明仅在处理器注册全部失败时可达。 --- src/Undefined/main.py | 52 ++++++++++++++++++++++++++++++++++--- tests/test_main_shutdown.py | 33 ++++++++++++++++++----- 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/src/Undefined/main.py b/src/Undefined/main.py index 55bbb1e2..6e51f96e 100644 --- a/src/Undefined/main.py +++ b/src/Undefined/main.py @@ -507,10 +507,12 @@ def _apply_config_updates( "/naga 命令和 /api/v1/naga/* 端点都不会可用" ) - shutdown_event = install_shutdown_signal_handlers(logger) + shutdown_guard = install_shutdown_signal_handlers(logger) try: - await _run_until_shutdown(onebot, shutdown_event, logger) + await _run_until_shutdown(onebot, shutdown_guard.event, logger) except KeyboardInterrupt: + # 仅在信号处理器注册全部失败（如非主线程运行）时才会走到这里； + # 正常安装后 SIGINT 会转为停机事件，不再抛 KeyboardInterrupt logger.info("[退出] 收到退出信号 (Ctrl+C)") except Exception as exc: logger.exception("[异常] 运行期间发生未捕获的错误: %s", exc) @@ -541,24 +543,64 @@ def _apply_config_updates( await config_manager.stop_hot_reload() await close_render_browser() await close_render_cache() + shutdown_guard.restore() logger.info("[退出] 机器人已停止运行") -def install_shutdown_signal_handlers(logger: logging.Logger) -> asyncio.Event: +class ShutdownSignalGuard: + """优雅停机信号注册的句柄：携带停机事件并支持恢复安装前的信号状态。""" + + def __init__( + self, + event: asyncio.Event, + previous: dict[int, Any], + loop_based: set[int], + loop: asyncio.AbstractEventLoop, + logger: logging.Logger, + ) -> None: + self.event = event + self._previous = previous + self._loop_based = loop_based + self._loop = loop + self._logger = logger + + def restore(self) -> None: + """恢复安装前的信号处理器；须从安装时的同一线程调用。""" + for signum, handler in self._previous.items(): + try: + if signum in self._loop_based: + self._loop.remove_signal_handler(signum) + signal.signal(signum, handler) + except (OSError, RuntimeError, ValueError): + self._logger.warning("[退出] 恢复信号 %s 的原处理器失败", signum) + self._previous.clear() + + +def install_shutdown_signal_handlers(logger: logging.Logger) -> ShutdownSignalGuard: """注册 SIGTERM / SIGINT 的优雅停机事件。 容器与服务管理器默认发送 SIGTERM（而非 Ctrl+C 的 SIGINT），此前未处理会直接 终止进程并跳过后面的落盘清理。这里把两个信号都收敛到同一个事件，由主循环在 被唤醒后走正常关闭流程。 + + 安装前会保存原有处理器，停机完成后调用 `ShutdownSignalGuard.restore()` + 归还信号控制权，避免作为库被导入时永久劫持调用方的信号处理。 """ stop_event = asyncio.Event() loop = asyncio.get_running_loop() + previous: dict[int, Any] = {} + loop_based: set[int] = set() for signame in ("SIGTERM", "SIGINT"): signum = getattr(signal, signame, None) if signum is None: continue + try: + previous[signum] = signal.getsignal(signum) + except (OSError, ValueError): + continue try: loop.add_signal_handler(signum, stop_event.set) + loop_based.add(signum) except (NotImplementedError, RuntimeError, ValueError): # Windows 的事件循环不支持 add_signal_handler，退回到 signal.signal try: @@ -570,7 +612,9 @@ def install_shutdown_signal_handlers(logger: logging.Logger) -> asyncio.Event: ) except (ValueError, OSError): logger.warning("[退出] 无法注册 %s 处理器", signame) - return stop_event + # 未安装成功就没有需要恢复的状态 + previous.pop(signum, None) + return ShutdownSignalGuard(stop_event, previous, loop_based, loop, logger) async def _run_until_shutdown( diff --git a/tests/test_main_shutdown.py b/tests/test_main_shutdown.py index 9577fdb2..4c45de6e 100644 --- a/tests/test_main_shutdown.py +++ b/tests/test_main_shutdown.py @@ -59,19 +59,40 @@ async def test_install_shutdown_signal_handlers_reacts_to_sigterm() -> None: pytest.skip("SIGTERM unavailable") original = signal.getsignal(signal.SIGTERM) - event = install_shutdown_signal_handlers(logger) + guard = install_shutdown_signal_handlers(logger) if signal.getsignal(signal.SIGTERM) is signal.SIG_DFL: pytest.skip("当前事件循环不支持信号处理器") try: os.kill(os.getpid(), signal.SIGTERM) - await asyncio.wait_for(event.wait(), timeout=2) - assert event.is_set() + await asyncio.wait_for(guard.event.wait(), timeout=2) + assert guard.event.is_set() + finally: + guard.restore() + assert signal.getsignal(signal.SIGTERM) is original + + +@pytest.mark.asyncio +async def test_install_shutdown_signal_handlers_restore_returns_previous_handler() -> ( + None +): + if not hasattr(signal, "SIGTERM"): # pragma: no cover - 非 POSIX 平台 + pytest.skip("SIGTERM unavailable") + + def _previous_handler(signum: object, frame: object) -> None: # pragma: no cover + return None + + original = signal.signal(signal.SIGTERM, _previous_handler) + try: + guard = install_shutdown_signal_handlers(logger) + guard.restore() + assert signal.getsignal(signal.SIGTERM) is _previous_handler finally: signal.signal(signal.SIGTERM, original) @pytest.mark.asyncio async def test_install_shutdown_signal_handlers_returns_fresh_event() -> None: - event = install_shutdown_signal_handlers(logger) - assert isinstance(event, asyncio.Event) - assert event.is_set() is False + guard = install_shutdown_signal_handlers(logger) + assert isinstance(guard.event, asyncio.Event) + assert guard.event.is_set() is False + guard.restore() From 77378214faa0b4e87a1257e0b83b1aa54f35768d Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 20:06:44 +0800 Subject: [PATCH 27/30] =?UTF-8?q?fix(scripts):=20=E9=87=8D=E5=B5=8C?= =?UTF-8?q?=E5=85=A5=E7=BB=B4=E5=BA=A6=E8=BF=81=E7=A7=BB=E6=94=B9=E4=B8=BA?= =?UTF-8?q?=E4=B8=B4=E6=97=B6=E5=BA=93=E5=86=99=E5=85=A5=20+=20=E5=8E=9F?= =?UTF-8?q?=E5=AD=90=E6=8D=A2=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原先 delete_collection 后逐批 upsert，中途被 SIGKILL 会留下残缺的 正式库。现改为：新向量全部写入 __rebuild_tmp 临时库（原库在 迁移完成前保持不动），写完校验记录数后删原库并 modify 换名，中断 窗口从「整个迁移过程」缩小到两次元数据操作之间；脚本启动时自动清 理遗留临时库，或在其持有全量数据时换名恢复。同时 client 改为必需 参数，删除不可达的 None 兜底分支。 --- scripts/reembed_cognitive.py | 106 +++++++++++++++++++------ tests/test_reembed_cognitive_script.py | 58 +++++++++++--- 2 files changed, 131 insertions(+), 33 deletions(-) diff --git a/scripts/reembed_cognitive.py b/scripts/reembed_cognitive.py index 83851b5a..5a4b5f11 100755 --- a/scripts/reembed_cognitive.py +++ b/scripts/reembed_cognitive.py @@ -8,8 +8,10 @@ 用新模型重新计算向量，然后通过 upsert 覆写回去。metadata 保持不变。 维度变化：ChromaDB 的 collection 在首次写入时定维，异维向量 upsert 会直接失败 -（InvalidArgumentError）。脚本会先比较新旧向量维度，检测到变化时删除并重建 -collection（先读全量记录再重建，不会丢数据），然后按新维度全量写回。 +（InvalidArgumentError）。脚本会先比较新旧向量维度，检测到变化时把新向量全部 +写入临时 collection（原库在迁移完成前保持不动），全部写完并校验记录数后删除 +原库、把临时 collection 原子换名回正式名称；中途被杀也不会丢库——下次运行会 +自动清理遗留的临时库，或在正式库缺失时从临时库恢复。 用法： # 先在 config.toml 中更新 [models.embedding] 为新模型配置 @@ -57,6 +59,13 @@ # ChromaDB get() 单次最大拉取量 _CHROMA_GET_LIMIT = 5000 +# 维度迁移时临时 collection 的后缀 +_STAGING_SUFFIX = "__rebuild_tmp" + + +def _staging_name(collection_name: str) -> str: + return f"{collection_name}{_STAGING_SUFFIX}" + def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -148,31 +157,72 @@ def _collection_dimension(collection: Any) -> int: return 0 -def _recreate_collection( - client: Any, - collection_name: str, - metadata: dict[str, Any] | None, -) -> Any: - """删除并重建 collection，用于向量维度变化后的全量写回。""" +def _recover_stale_staging(client: Any, collection_name: str) -> None: + """处理上次异常中断遗留的临时 collection，保证迁移可安全重跑。 + + - 正式库仍存在：临时库只是写了一半的残留，直接删除； + - 正式库缺失（中断发生在删原库之后、换名之前）：临时库持有全量数据， + 换名恢复成正式库。 + """ + staging_name = _staging_name(collection_name) + names = {c.name for c in client.list_collections()} + if staging_name not in names: + return + if collection_name in names: + logger.warning( + "发现上次运行遗留的临时 collection %s（正式库完好），已删除", + staging_name, + ) + client.delete_collection(staging_name) + else: + logger.warning( + "发现上次运行在换名前中断：正式库 %s 缺失但临时库完好，正在恢复", + collection_name, + ) + client.get_collection(staging_name).modify(name=collection_name) + logger.warning("恢复完成：%s 已从临时库换名回来", collection_name) + + +def _begin_dimension_migration(client: Any, collection_name: str, metadata: Any) -> Any: + """维度变化时创建临时 collection 承接新向量；原库在迁移完成前保持不动。""" + staging_name = _staging_name(collection_name) logger.warning( - "重建 collection %s（原维度与新模型不一致，ChromaDB 不支持原地改维）", - collection_name, + "向量维度变化，迁移写入临时 collection %s，全部写完并校验后原子换名", + staging_name, ) - client.delete_collection(collection_name) return client.get_or_create_collection( - collection_name, + staging_name, metadata=metadata or {"hnsw:space": "cosine"}, ) +def _finish_dimension_migration( + client: Any, + collection_name: str, + staging_collection: Any, + *, + expected_count: int, +) -> None: + """校验临时库记录数后删原库、换名；失败时保留原库，可安全重跑。""" + staged = staging_collection.count() + if staged != expected_count: + raise RuntimeError( + f"临时 collection {staging_collection.name} 记录数不符: " + f"{staged} != {expected_count}；已保留原库 {collection_name}，" + "请排查后重跑脚本" + ) + logger.info("临时 collection 校验通过（%d 条），换名为 %s", staged, collection_name) + client.delete_collection(collection_name) + staging_collection.modify(name=collection_name) + + async def _reembed_collection( collection: Any, collection_name: str, embedder: Embedder, batch_size: int, dry_run: bool, - *, - client: Any = None, + client: Any, ) -> int: """对单个 collection 执行全量重嵌入，返回处理的记录数。""" logger.info("正在读取 %s ...", collection_name) @@ -196,6 +246,8 @@ async def _reembed_collection( processed = 0 dimension_checked = False + write_target = collection + staging_collection: Any = None start_time = time.perf_counter() for i in range(0, total, batch_size): @@ -222,21 +274,20 @@ async def _reembed_collection( ) if dry_run: logger.info( - "[dry-run] 实际执行时会重建 collection %s 后写入新维度向量", + "[dry-run] 实际执行时会把新维度向量写入临时 collection，" + "全部写完后换名为 %s", collection_name, ) else: - if client is None: - raise RuntimeError( - "检测到向量维度变化，但缺少 ChromaDB client，无法重建 collection" - ) - collection = _recreate_collection( + write_target = _begin_dimension_migration( client, collection_name, collection.metadata ) + staging_collection = write_target if not dry_run: - # upsert 覆写：ID 不变，document 和 metadata 不变，仅更新 embedding - collection.upsert( + # upsert 覆写：ID 不变，document 和 metadata 不变，仅更新 embedding； + # 维度迁移时写入临时 collection，原库保持不动 + write_target.upsert( ids=batch_ids, documents=batch_docs, embeddings=new_embeddings, @@ -256,6 +307,11 @@ async def _reembed_collection( " (dry-run)" if dry_run else "", ) + if staging_collection is not None: + _finish_dimension_migration( + client, collection_name, staging_collection, expected_count=total + ) + elapsed_total = time.perf_counter() - start_time logger.info( "%s 完成：%d 条记录，耗时 %.1f 秒%s", @@ -307,6 +363,7 @@ async def _main(args: argparse.Namespace) -> None: try: if not args.profiles_only: + _recover_stale_staging(client, "cognitive_events") events_col = client.get_or_create_collection( "cognitive_events", metadata={"hnsw:space": "cosine"} ) @@ -316,10 +373,11 @@ async def _main(args: argparse.Namespace) -> None: embedder, args.batch_size, args.dry_run, - client=client, + client, ) if not args.events_only: + _recover_stale_staging(client, "cognitive_profiles") profiles_col = client.get_or_create_collection( "cognitive_profiles", metadata={"hnsw:space": "cosine"} ) @@ -329,7 +387,7 @@ async def _main(args: argparse.Namespace) -> None: embedder, args.batch_size, args.dry_run, - client=client, + client, ) finally: await embedder.stop() diff --git a/tests/test_reembed_cognitive_script.py b/tests/test_reembed_cognitive_script.py index 0f7931e0..75043448 100644 --- a/tests/test_reembed_cognitive_script.py +++ b/tests/test_reembed_cognitive_script.py @@ -125,21 +125,61 @@ async def test_reembed_dry_run_reports_dimension_change_without_writing( @pytest.mark.asyncio -async def test_reembed_requires_client_when_dimension_changes( +async def test_reembed_migration_leaves_no_staging_collection( tmp_path: Path, ) -> None: module = _load_script_module() client = chromadb.PersistentClient(path=str(tmp_path)) collection = _seed_collection(client, "cognitive_events", dimension=3) - with pytest.raises(RuntimeError, match="缺少 ChromaDB client"): - await module._reembed_collection( - collection, - "cognitive_events", - _FixedDimEmbedder(9), - batch_size=2, - dry_run=False, - ) + await module._reembed_collection( + collection, + "cognitive_events", + _FixedDimEmbedder(5), + batch_size=2, + dry_run=False, + client=client, + ) + + names = {c.name for c in client.list_collections()} + assert names == {"cognitive_events"} + migrated = client.get_collection("cognitive_events") + assert migrated.count() == 3 + sample = cast(Any, migrated.get(limit=1, include=["embeddings"])) + assert len(sample["embeddings"][0]) == 5 + + +def test_recover_stale_staging_deletes_residue_when_original_intact( + tmp_path: Path, +) -> None: + module = _load_script_module() + client = chromadb.PersistentClient(path=str(tmp_path)) + _seed_collection(client, "cognitive_events", dimension=3, count=2) + _seed_collection(client, "cognitive_events__rebuild_tmp", dimension=5, count=2) + + module._recover_stale_staging(client, "cognitive_events") + + names = {c.name for c in client.list_collections()} + assert names == {"cognitive_events"} + assert client.get_collection("cognitive_events").count() == 2 + + +def test_recover_stale_staging_renames_back_when_original_missing( + tmp_path: Path, +) -> None: + module = _load_script_module() + client = chromadb.PersistentClient(path=str(tmp_path)) + # 模拟「删了原库、还没换名」的中断现场：全量数据只在临时库里 + _seed_collection(client, "cognitive_events__rebuild_tmp", dimension=5, count=2) + + module._recover_stale_staging(client, "cognitive_events") + + names = {c.name for c in client.list_collections()} + assert names == {"cognitive_events"} + restored = client.get_collection("cognitive_events") + assert restored.count() == 2 + sample = cast(Any, restored.get(limit=1, include=["embeddings"])) + assert len(sample["embeddings"][0]) == 5 @pytest.mark.asyncio From fb9b54ac67349b0d47ac1e255c58141c7eae2991 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 20:08:41 +0800 Subject: [PATCH 28/30] =?UTF-8?q?fix(knowledge):=20=E6=A3=80=E7=B4=A2?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E6=97=B6=E5=88=9D=E5=A7=8B=E5=8C=96=E6=94=B9?= =?UTF-8?q?=E7=94=A8=20threading.Lock=20=E5=8F=8C=E6=A3=80=E5=AE=88?= =?UTF-8?q?=E5=8D=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 共享重排器的 _reranker_initialized 布尔守卫与 for_feature 的运行时 去重扫描在并发初始化下可能重复创建 Reranker / 运行时。单事件循环内 无 await 本就按协程粒度原子，加 threading.Lock 防御未来从线程池并发 调用的场景；stop() 锁内只做快照清理，await 移到锁外避免阻塞事件循环。 --- src/Undefined/knowledge/runtime.py | 66 +++++++++++++++++------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/src/Undefined/knowledge/runtime.py b/src/Undefined/knowledge/runtime.py index cfae277e..4d3d986a 100644 --- a/src/Undefined/knowledge/runtime.py +++ b/src/Undefined/knowledge/runtime.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import threading from collections.abc import Callable, Mapping from typing import TYPE_CHECKING @@ -143,23 +144,27 @@ def __init__( self._runtimes: list[RetrievalRuntime] = [] self._reranker: Reranker | None = None self._reranker_initialized = False + # 常规运行只跑单个事件循环，方法内部无 await，本就按协程粒度原子； + # 锁用于防御未来从多线程（如 to_thread / 线程池）并发初始化 + self._init_lock = threading.Lock() def for_feature(self, feature: str) -> RetrievalRuntime: """返回功能对应的检索运行时；相同生效配置复用同一实例。""" model = self._embedding_models.get(feature) if model is None: raise KeyError(f"unknown embedding feature: {feature}") - for runtime in self._runtimes: - if runtime.embedding_model == model: - return runtime - runtime = RetrievalRuntime( - self._requester, - model, - self._rerank_model, - embed_batch_size=self._embed_batch_size, - reranker_provider=self.ensure_reranker, - ) - self._runtimes.append(runtime) + with self._init_lock: + for runtime in self._runtimes: + if runtime.embedding_model == model: + return runtime + runtime = RetrievalRuntime( + self._requester, + model, + self._rerank_model, + embed_batch_size=self._embed_batch_size, + reranker_provider=self.ensure_reranker, + ) + self._runtimes.append(runtime) logger.info( "[检索运行时] 功能已绑定 embedding 配置: feature=%s model=%s interval=%.2fs", feature, @@ -174,24 +179,29 @@ def runtimes(self) -> tuple[RetrievalRuntime, ...]: def ensure_reranker(self) -> Reranker | None: """共享重排器；未配置完整时返回 None。""" - if not self._reranker_initialized: - self._reranker_initialized = True - if self._rerank_model.api_url and self._rerank_model.model_name: - reranker = Reranker(self._requester, self._rerank_model) - reranker.start() - self._reranker = reranker - logger.info( - "[检索运行时] 重排发车器已启动: interval=%.2fs model=%s", - reranker.interval, - self._rerank_model.model_name, - ) - return self._reranker + with self._init_lock: + if not self._reranker_initialized: + self._reranker_initialized = True + if self._rerank_model.api_url and self._rerank_model.model_name: + reranker = Reranker(self._requester, self._rerank_model) + reranker.start() + self._reranker = reranker + logger.info( + "[检索运行时] 重排发车器已启动: interval=%.2fs model=%s", + reranker.interval, + self._rerank_model.model_name, + ) + return self._reranker async def stop(self) -> None: - if self._reranker is not None: - await self._reranker.stop() + # 锁内只做状态快照与清理，await 放在锁外，避免持有线程锁阻塞事件循环 + with self._init_lock: + reranker = self._reranker self._reranker = None - self._reranker_initialized = False - for runtime in self._runtimes: + self._reranker_initialized = False + runtimes = list(self._runtimes) + self._runtimes.clear() + if reranker is not None: + await reranker.stop() + for runtime in runtimes: await runtime.stop() - self._runtimes.clear() From 62b612edf888b14c19bd1375e328dd09a432f97e Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 20:09:51 +0800 Subject: [PATCH 29/30] =?UTF-8?q?chore(config):=20=E6=9C=AA=E7=9F=A5=20emb?= =?UTF-8?q?edding=20=E5=8A=9F=E8=83=BD=E5=90=8D=E8=AD=A6=E5=91=8A=E5=88=97?= =?UTF-8?q?=E5=87=BA=E5=90=88=E6=B3=95=E5=80=BC=EF=BC=9BCHANGELOG=20?= =?UTF-8?q?=E8=A1=A5=E6=B8=85=E7=90=86=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 未知功能名警告现在标注「疑似拼写错误」并列出全部可用功能名，降低 静默回落默认配置被忽视的概率；CHANGELOG 补记 SecurityService 限流 方法移除、crawl4ai 启动降级与 reembed 原子迁移的行为口径。 --- CHANGELOG.md | 6 +++--- src/Undefined/config/parsers/embedding.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae2dee66..a021ef8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,10 @@ - 史官 worker 新增并发上限 `[cognitive.historian].max_concurrency`（默认 4），以信号量与在途计数双重约束，不再无上限并发。 - 修复随包 Agent 的 handler 无法加载：handler 模块改按真实包路径导入，`code_delivery_agent` 等使用相对导入的 Agent 恢复可用；注册阶段即预导入全部 handler，失败项记录 `load_error` 并从 schema 中排除，主 AI 不再看到不可用的技能。 - 配置热更新失败不再静默：更新改为步骤表逐项执行，单步异常不中断其余步骤，失败项与「未完全生效」汇总以 error 级日志输出；异步热更新任务与配置回调的异常均会被记录。嵌入 / 重排模型配置变更加入需重启提示，避免热重载静默无效。 -- 新增 SIGTERM 优雅停机：容器 / systemd / supervisor 停止时不再被直接终止并跳过落盘清理，SIGTERM 与 SIGINT 收敛到同一停机事件，取消连接任务并等待收敛。 +- 新增 SIGTERM 优雅停机：容器 / systemd / supervisor 停止时不再被直接终止并跳过落盘清理，SIGTERM 与 SIGINT 收敛到同一停机事件，取消连接任务并等待收敛；停机完成后恢复系统原有信号处理器。 - 修复消息队列重试上限口径分叉：coordinator 与 QueueManager 各用一套重试上限，热更新后两者可分叉，导致等待方在仍会重试时被误判为失败、或重试已耗尽后干等到 480 秒超时；现统一由 `resolve_effective_retry_count` 计算并与等待超时预算同口径。 -- `scripts/reembed_cognitive.py` 支持维度变化迁移：检测到新旧向量维度不同时先读取全量记录、删除并重建同名 collection（沿用原索引元数据）后按新维度写回，记录不丢；`--dry-run` 不做任何写入。 -- 依赖与配置清理：crawl4ai 与 langchain-community 改为必需依赖，删除「未安装则降级」的静默回退（缺失时直接报错暴露环境问题）；移除零引用的死配置 `cognitive.historian.rewrite_max_retry` 与死依赖 imgkit、croniter。 +- `scripts/reembed_cognitive.py` 支持维度变化迁移：检测到新旧向量维度不同时把新向量全部写入临时 collection，校验记录数后删除原库并原子换名，原库在迁移完成前保持不动，中途被杀可自动清理或从临时库恢复，记录不丢；`--dry-run` 不做任何写入。 +- 依赖与配置清理：crawl4ai 与 langchain-community 改为必需依赖，删除「未安装则降级」的静默回退（crawl4ai 环境异常时启动降级为网页获取不可用并以 error 日志提示修复，不再让 Bot 启动崩溃）；移除零引用的死配置 `cognitive.historian.rewrite_max_retry`、无调用方的 `SecurityService.check_rate_limit` / `record_rate_limit`（限流由 rate_limiter 承担）与死依赖 imgkit、croniter。 - CI 补齐治理：工作流收敛只读权限、并发取消与任务超时，前端行为测试真实执行而非静默跳过，新增 Python 3.11 / 3.13 兼容矩阵，测试开启 65% 覆盖率门禁。 --- diff --git a/src/Undefined/config/parsers/embedding.py b/src/Undefined/config/parsers/embedding.py index 7014b5a4..10340293 100644 --- a/src/Undefined/config/parsers/embedding.py +++ b/src/Undefined/config/parsers/embedding.py @@ -117,8 +117,10 @@ def _parse_embedding_feature_overrides( unknown = sorted(str(name) for name in raw if name not in EMBEDDING_FEATURES) if unknown: logger.warning( - "[配置] models.embedding.features 中存在未知功能名，已忽略: %s", + "[配置] models.embedding.features 中存在未知功能名（疑似拼写错误），" + "已忽略、对应功能继续使用默认配置: %s（可用功能名: %s）", ", ".join(unknown), + ", ".join(EMBEDDING_FEATURES), ) overrides: dict[str, EmbeddingFeatureOverride] = {} From f6e0f51ceebbaf49a9b6a9b10f5578b2c3d7289b Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Sat, 19 Sep 2026 20:29:23 +0800 Subject: [PATCH 30/30] =?UTF-8?q?docs:=20=E8=90=BD=E5=AE=9E=E5=A4=8D?= =?UTF-8?q?=E5=AE=A1=E6=94=B6=E5=B0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 侧写锁按复审建议简化为单次 setdefault 原子创建； - background.py 注明约定：新增 _execute_queued_* 一律使用 resolve_effective_retry_count，禁止直读 config.ai_request_max_retries； - 史官 max_concurrency 注明仅启动时读取，若开放热更新须同步重建 在途门控与信号量； - configuration.md 强调 dimensions 三态语义（-1 继承 / 0 模型默认 / >0 显式），并说明继承与模型默认的区别及定维后果。 --- docs/configuration.md | 1 + src/Undefined/cognitive/historian/worker.py | 2 ++ src/Undefined/cognitive/profile_storage.py | 21 +++++++------------ .../services/coordinator/background.py | 2 ++ 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 136b02bd..cfa58793 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -537,6 +537,7 @@ document_instruction = "passage: " 语义说明： - 未出现在本表中的字段，以及取哨兵值的字段，都表示继续继承 `[models.embedding]`； +- **`dimensions` 的三态区别**：`-1`（或留空默认值）= 继承默认表的 `dimensions`，`0` = 不传维度、使用模型自带默认维度，`>0` = 显式指定维度。继承与“用模型默认”是两回事：默认表配了具体维度（如 `2560`）时，覆写表写 `0` 得到的是模型默认维度而不是默认表的值；ChromaDB 首次写入即定维，改动维度需要跑重嵌入脚本； - 生效配置完全相同（含指令前缀）的功能共用同一个 Embedder 与发车队列；任一字段不同则该功能拥有独立的 Embedder 与队列，重排器在所有功能间共享； - 功能名拼写错误或写成未知功能名时会被忽略并记录警告，该功能回落到默认配置； - 嵌入配置（含 `features` 子表）与 `[models.rerank]` 都在启动时构造运行时，热更新只提示“需要重启生效”，不会改变已运行实例。 diff --git a/src/Undefined/cognitive/historian/worker.py b/src/Undefined/cognitive/historian/worker.py index d9d87cb7..00e9578e 100644 --- a/src/Undefined/cognitive/historian/worker.py +++ b/src/Undefined/cognitive/historian/worker.py @@ -59,6 +59,8 @@ def __init__( self._config_getter = config_getter self._model_config = model_config self._max_concurrency = max(1, int(max_concurrency)) + # max_concurrency 仅在启动时读取一次，热更新不生效；若将来开放热更新， + # 必须同步重建在途门控（_poll_loop 比较的 _max_concurrency）与 _semaphore self._stop_event = asyncio.Event() self._task: asyncio.Task[None] | None = None self._inflight_tasks: set[asyncio.Task[None]] = set() diff --git a/src/Undefined/cognitive/profile_storage.py b/src/Undefined/cognitive/profile_storage.py index 0722c9fe..12920012 100644 --- a/src/Undefined/cognitive/profile_storage.py +++ b/src/Undefined/cognitive/profile_storage.py @@ -26,27 +26,20 @@ def __init__(self, base_path: str | Path, revision_keep: int = 5) -> None: ) def _get_lock(self, entity_type: str, entity_id: str) -> asyncio.Lock: - key = f"{entity_type}:{entity_id}" - lock = self._locks.get(key) - if lock is None: - # dict.setdefault 原子插入：并发首次进入同一实体时双方拿到同一把锁， - # 不会出现各自建锁、后建覆盖先建导致互斥失效 - lock = self._locks.setdefault(key, asyncio.Lock()) - return lock + # setdefault 原子插入：并发首次进入同一实体时双方拿到同一把锁， + # 不会出现各自建锁、后建覆盖先建导致互斥失效 + return self._locks.setdefault(f"{entity_type}:{entity_id}", asyncio.Lock()) def merge_guard(self, entity_type: str, entity_id: str) -> asyncio.Lock: """跨「读 → LLM → 写」整段侧写合并的互斥锁。 只串行化同一实体的合并周期，避免两个 job 各自基于旧快照改写后互相覆盖 （后写覆盖先写，先前的观察永久丢失）。与文件写入锁分开，避免与 - `write_profile` 的锁重入死锁。锁的创建经 `dict.setdefault` 原子完成， - 见 `_get_lock`。 + `write_profile` 的锁重入死锁。锁的创建经 `dict.setdefault` 原子完成。 """ - key = f"{entity_type}:{entity_id}" - lock = self._merge_locks.get(key) - if lock is None: - lock = self._merge_locks.setdefault(key, asyncio.Lock()) - return lock + return self._merge_locks.setdefault( + f"{entity_type}:{entity_id}", asyncio.Lock() + ) def _profile_path(self, entity_type: str, entity_id: str) -> Path: return self._base / f"{entity_type}s" / f"{entity_id}.md" diff --git a/src/Undefined/services/coordinator/background.py b/src/Undefined/services/coordinator/background.py index 2aa993d7..34e8b956 100644 --- a/src/Undefined/services/coordinator/background.py +++ b/src/Undefined/services/coordinator/background.py @@ -178,6 +178,8 @@ async def _execute_queued_llm_call(self, request: dict[str, Any]) -> None: # 重试上限以 QueueManager 为准（与队列实际重试逻辑、等待超时同源）， # 否则热更新后两边分叉：等待方可能在仍会重试时就收到失败，或在重试 # 已耗尽时一直挂到 480s 超时。 + # 约定：后续新增 _execute_queued_* 一律使用 resolve_effective_retry_count + # 判断重试耗尽，禁止直接读 config.ai_request_max_retries。 retry_count = int(request.get("_retry_count", 0) or 0) max_retries = resolve_effective_retry_count( self.config, getattr(self, "queue_manager", None)