Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ LANGSMITH_TRACING=false
# 默认建议创建独立数据库,如 ctb;若直接使用 postgres 默认库,也可改为 /postgres。
# APP_DATABASE_URL=postgresql+psycopg://postgres:root@localhost:15432/ctb
# APP_DATABASE_ECHO=false
# APP_POSTGRES_VECTOR_DIMENSIONS 必须和所选 embedding 模型维度一致。
# 例如 text-embedding-3-small 默认是 1536 维。
# APP_POSTGRES_VECTOR_DIMENSIONS=1536

# ============================================================================
Expand Down
3 changes: 2 additions & 1 deletion backend/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ class Settings(BaseSettings):
db_path: Path | None = None
database_url: str = ""
database_echo: bool = False
postgres_vector_dimensions: int = 1536

# 各类子目录(由 validator 从 runtime_dir 派生,可独立覆盖以便测试)
upload_dir: Path | None = None
Expand All @@ -225,7 +226,7 @@ class Settings(BaseSettings):
True # 是否信任系统代理环境变量,Windows 下设为 False 可解 WinError 10054
)

rag_embedding_model: str = "text-embedding-v3"
rag_embedding_model: str = "text-embedding-3-small"
rag_embedding_api_key: str = ""
rag_embedding_base_url: str = ""

Expand Down
8 changes: 5 additions & 3 deletions backend/core/model_selection.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@ class LLMSelectionError(Exception):


def split_models(model_name: str | None) -> list[str]:
"""返回模型名称列表。现在只支持单个模型,但保留列表形式以兼容调用方。"""
name = (model_name or "").strip()
return [name] if name else []
"""返回模型名称列表,兼容逗号分隔的历史存储格式。"""
raw = str(model_name or "").strip()
if not raw:
return []
return [item.strip() for item in raw.split(",") if item.strip()]


def build_managed_provider_context(db):
Expand Down
60 changes: 26 additions & 34 deletions backend/core/rag.py
Original file line number Diff line number Diff line change
Expand Up @@ -317,56 +317,48 @@ def index_question(db: Session, question_id: int) -> bool:

vector_json = _serialize_vector(vector)

if existing:
existing.content = chunk_data["content"]
existing.metadata_json = json.dumps(chunk_data["metadata"], ensure_ascii=False)
existing.content_hash = chunk_data["content_hash"]
existing.embedding_model = settings.rag_embedding_model if vector else None
existing.vector_json = vector_json
else:
chunk = RagDocumentChunk(
user_id=question.user_id,
project_id=question.project_id,
source_type="question",
source_id=question_id,
chunk_index=0,
content=chunk_data["content"],
metadata_json=json.dumps(chunk_data["metadata"], ensure_ascii=False),
content_hash=chunk_data["content_hash"],
embedding_model=settings.rag_embedding_model if vector else None,
vector_json=vector_json,
)
db.add(chunk)

try:
db.flush()
target_chunk = existing or chunk
_write_postgres_vector(db, target_chunk.id, vector)
db.commit()
with db.begin_nested():
if existing:
existing.content = chunk_data["content"]
existing.metadata_json = json.dumps(chunk_data["metadata"], ensure_ascii=False)
existing.content_hash = chunk_data["content_hash"]
existing.embedding_model = settings.rag_embedding_model if vector else None
existing.vector_json = vector_json
target_chunk = existing
else:
target_chunk = RagDocumentChunk(
user_id=question.user_id,
project_id=question.project_id,
source_type="question",
source_id=question_id,
chunk_index=0,
content=chunk_data["content"],
metadata_json=json.dumps(chunk_data["metadata"], ensure_ascii=False),
content_hash=chunk_data["content_hash"],
embedding_model=settings.rag_embedding_model if vector else None,
vector_json=vector_json,
)
db.add(target_chunk)

db.flush()
_write_postgres_vector(db, target_chunk.id, vector)
return True
except Exception as e:
db.rollback()
logger.error("索引题目 %d 失败: %s", question_id, e)
return False


def delete_question_chunks(db: Session, question_id: int) -> int:
"""删除错题关联的所有 RAG chunk"""
count = (
return (
db.query(RagDocumentChunk)
.filter(
RagDocumentChunk.source_type == "question",
RagDocumentChunk.source_id == question_id,
)
.delete(synchronize_session=False)
)
try:
db.commit()
except Exception as e:
db.rollback()
logger.error("删除题目 %d 的 chunk 失败: %s", question_id, e)
return 0
return count


# ---------------------------------------------------------------------------
Expand Down
41 changes: 40 additions & 1 deletion backend/db/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
数据库模块:引擎创建、Session 工厂、初始化函数
"""

from sqlalchemy import create_engine, event
from sqlalchemy import create_engine, event, inspect, text
from sqlalchemy.orm import sessionmaker
import os
import sys
Expand Down Expand Up @@ -154,7 +154,46 @@ def _migrate_schema():
conn.close()


def _prepare_postgresql_extensions():
"""在 PostgreSQL 建表前确保 pgvector 扩展可用。"""
if not is_postgresql_backend(engine):
return
with engine.begin() as conn:
conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))


def _ensure_postgresql_schema():
"""为 PostgreSQL 现有表补齐 pgvector 列和索引。"""
if not is_postgresql_backend(engine):
return

inspector = inspect(engine)
table_names = set(inspector.get_table_names())
if "rag_document_chunks" not in table_names:
return

columns = {column["name"] for column in inspector.get_columns("rag_document_chunks")}
with engine.begin() as conn:
conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
if "embedding_vector" not in columns:
conn.execute(
text(
"ALTER TABLE rag_document_chunks "
f"ADD COLUMN embedding_vector vector({settings.postgres_vector_dimensions})"
)
)
conn.execute(
text(
"CREATE INDEX IF NOT EXISTS idx_rag_document_chunks_embedding_vector "
"ON rag_document_chunks USING ivfflat (embedding_vector vector_cosine_ops) "
"WITH (lists = 100)"
)
)


def init_db():
"""初始化数据库:建表并执行轻量级自动迁移"""
_prepare_postgresql_extensions()
Base.metadata.create_all(bind=engine)
_migrate_schema()
_ensure_postgresql_schema()
6 changes: 5 additions & 1 deletion backend/db/crud/projects.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from db.models import (
ChatSession, Note, NoteTagMapping, Project, Question,
QuestionEmbedding, QuestionTagMapping, UploadBatch,
QuestionEmbedding, QuestionTagMapping, RagDocumentChunk, UploadBatch,
)


Expand Down Expand Up @@ -141,6 +141,10 @@ def delete_project(db: Session, project_id: int, user_id=None) -> bool:
# 先删除题目和笔记的关联子表,再删除题目/笔记本身
question_ids = [q.id for q in db.query(Question.id).filter(Question.project_id == project.id).all()]
if question_ids:
db.query(RagDocumentChunk).filter(
RagDocumentChunk.source_type == "question",
RagDocumentChunk.source_id.in_(question_ids),
).delete(synchronize_session=False)
db.query(QuestionEmbedding).filter(QuestionEmbedding.question_id.in_(question_ids)).delete(synchronize_session=False)
db.query(ChatSession).filter(ChatSession.question_id.in_(question_ids)).delete(synchronize_session=False)
db.query(QuestionTagMapping).filter(QuestionTagMapping.question_id.in_(question_ids)).delete(synchronize_session=False)
Expand Down
6 changes: 4 additions & 2 deletions backend/db/crud/questions.py
Original file line number Diff line number Diff line change
Expand Up @@ -358,8 +358,6 @@ def save_questions_to_db(
# 科目由编排智能体识别,不再使用关键词匹配
subject = batch_info.get("subject") or "未知"
project_id = project_id or batch_info.get("project_id")
if not project_id:
raise ValueError("PROJECT_REQUIRED")

# 创建批次记录
batch = UploadBatch(
Expand Down Expand Up @@ -693,6 +691,10 @@ def delete_question(db: Session, question_id: int, user_id=None) -> bool:

try:
batch_id = question.batch_id
from core.rag import delete_question_chunks

delete_question_chunks(db, question_id)

# 删除关联的标签映射
db.query(QuestionTagMapping).filter(QuestionTagMapping.question_id == question_id).delete()

Expand Down
24 changes: 22 additions & 2 deletions backend/db/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,32 @@

import uuid
from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, ForeignKey, UniqueConstraint
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.types import TypeDecorator
from sqlalchemy.orm import declarative_base, relationship
from datetime import datetime

from core.config import settings

try:
from pgvector.sqlalchemy import Vector
except ImportError: # pragma: no cover - optional on SQLite-only setups
Vector = None

Base = declarative_base()


class EmbeddingVectorType(TypeDecorator):
"""Use pgvector on PostgreSQL and plain text elsewhere."""

impl = Text
cache_ok = True

def load_dialect_impl(self, dialect):
if dialect.name == "postgresql" and Vector is not None:
return dialect.type_descriptor(Vector(settings.postgres_vector_dimensions))
return dialect.type_descriptor(Text())


class User(Base):
"""用户表"""
__tablename__ = "users"
Expand Down Expand Up @@ -330,5 +349,6 @@ class RagDocumentChunk(Base):
content_hash = Column(String(64), default="", index=True)
embedding_model = Column(String(100), nullable=True)
vector_json = Column(Text, nullable=True)
embedding_vector = Column(EmbeddingVectorType(), nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, index=True)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
30 changes: 15 additions & 15 deletions backend/routes/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -412,21 +412,6 @@ def stream_chat(session_id):

user_id = session.get("user_id")
with SessionLocal() as db:
try:
selection = resolve_llm_selection(
db,
user_id=user_id,
category=model_provider,
model_name=model_name,
provider_source=provider_source,
provider_id=provider_id,
)
except LLMSelectionError as e:
return (
jsonify({"success": False, "code": e.code, "error": e.message}),
e.status_code,
)

uid = _effective_user_id()
cs_query = (
db.query(ChatSessionModel)
Expand All @@ -446,6 +431,21 @@ def stream_chat(session_id):
if not chat_session:
return jsonify({"success": False, "error": "对话不存在"}), 404

try:
selection = resolve_llm_selection(
db,
user_id=user_id,
category=model_provider,
model_name=model_name,
provider_source=provider_source,
provider_id=provider_id,
)
except LLMSelectionError as e:
return (
jsonify({"success": False, "code": e.code, "error": e.message}),
e.status_code,
)

should_consume_quota = bool(user_id) and uses_server_llm_selection(
selection["source"],
db=db,
Expand Down
33 changes: 21 additions & 12 deletions backend/routes/questions.py
Original file line number Diff line number Diff line change
Expand Up @@ -456,8 +456,10 @@ def rag_reindex():
with SessionLocal() as db:
success = index_question(db, qid)
if success:
db.commit()
indexed += 1
else:
db.rollback()
skipped += 1
except Exception as e:
logger.warning("索引题目 %d 失败: %s", qid, e)
Expand Down Expand Up @@ -531,7 +533,10 @@ def update_question(question_id):
try:
from core.rag import index_question
with SessionLocal() as db:
index_question(db, question_id)
if index_question(db, question_id):
db.commit()
else:
db.rollback()
except Exception as e:
logger.warning(f"更新题目 {question_id} 的 RAG 索引失败: {e}")

Expand Down Expand Up @@ -560,7 +565,10 @@ def update_question_answer(question_id):
try:
from core.rag import index_question
with SessionLocal() as db:
index_question(db, question_id)
if index_question(db, question_id):
db.commit()
else:
db.rollback()
except Exception as e:
logger.warning(f"更新题目 {question_id} 的用户作答索引失败: {e}")

Expand Down Expand Up @@ -624,6 +632,8 @@ def save_to_db():

if not isinstance(selected_uids, list) or not selected_uids:
return jsonify({'success': False, 'error': '请选择至少一道题目'}), 400
if project_id is None:
return jsonify({'success': False, 'error': '请先创建并选择一个错题库'}), 400

run_id = data.get('run_id')
record_id = data.get('record_id')
Expand Down Expand Up @@ -677,15 +687,11 @@ def save_to_db():

with SessionLocal() as db:
try:
project_id = (
crud.require_project_id(
db,
project_id,
user_id=session.get('user_id'),
project_type="question",
)
if project_id
else None
project_id = crud.resolve_project_id(
db,
project_id,
user_id=session.get('user_id'),
project_type="question",
)
except ValueError as exc:
if str(exc) == "PROJECT_REQUIRED":
Expand Down Expand Up @@ -878,7 +884,10 @@ def save_question_answer(question_id):
try:
from core.rag import index_question
with SessionLocal() as db:
index_question(db, question_id)
if index_question(db, question_id):
db.commit()
else:
db.rollback()
except Exception as e:
logger.warning(f"更新题目 {question_id} 的 RAG 索引失败: {e}")

Expand Down
Loading