diff --git a/.changeset/secure-workbench-release.md b/.changeset/secure-workbench-release.md new file mode 100644 index 0000000..9d1aa2e --- /dev/null +++ b/.changeset/secure-workbench-release.md @@ -0,0 +1,16 @@ +--- +"opendocuments": major +"@opendocuments/client": major +"opendocuments-core": patch +"opendocuments-server": minor +"@opendocuments/connector-github": minor +"@opendocuments/connector-gdrive": minor +"@opendocuments/connector-s3": minor +"@opendocuments/connector-web-crawler": minor +"opendocuments-model-google": minor +"opendocuments-model-ollama": patch +"opendocuments-parser-pptx": minor +"opendocuments-parser-xlsx": minor +--- + +Ship the simplified multilingual web workbench, persisted connector management, safer team-mode transport controls, resilient document reindexing, expanded chunking and parser support, and refreshed TypeScript client APIs. Remove unsupported model-management commands and provider packages from the supported distribution. diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..610fd9c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +node_modules +packages/*/node_modules +plugins/*/node_modules +docs-site/node_modules +.turbo +packages/*/dist +plugins/*/dist +docs-site/.vitepress/cache +docs-site/.vitepress/dist +.git +.github +.changeset +tmp +*.log +.env +.env.* +!.env.example +opendocuments-data +*.db +*.sqlite +Dockerfile +docker-compose*.yml +portfolio-opendocuments.html diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..55d77da --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +GOOGLE_API_KEY=your-google-api-key + +# Interview demo defaults +OPENDOCUMENTS_MODEL_PROVIDER=google +OPENDOCUMENTS_MODEL_LLM=gemini-3.5-flash +OPENDOCUMENTS_MODEL_EMBEDDING=gemini-embedding-001 +OPENDOCUMENTS_MODEL_EMBEDDING_DIMENSIONS=3072 +OPENDOCUMENTS_MODEL_THINKING_BUDGET=0 +OPENDOCUMENTS_RAG_PROFILE=fast diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da393f3..7be2417 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,9 +18,9 @@ jobs: node-version: [20, 22] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: ${{ matrix.node-version }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7dd54a7..b9b2199 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,8 +18,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 with: node-version: 20 - run: cd docs-site && npm install && npm run build diff --git a/Dockerfile b/Dockerfile index 889cb80..160d9c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,22 @@ # ── Stage 1: build ────────────────────────────────────────────────────────── -FROM node:20-alpine AS builder +FROM node:20-bookworm-slim AS builder WORKDIR /app COPY package*.json turbo.json tsconfig.base.json ./ COPY packages/ packages/ COPY plugins/ plugins/ -RUN npm ci +RUN npm install --include=optional +RUN rollup_version="$(node -p "require('./node_modules/rollup/package.json').version")" \ + && lancedb_version="$(node -p "require('./node_modules/@lancedb/lancedb/package.json').version")" \ + && node_arch="$(node -p "process.arch")" \ + && case "$node_arch" in \ + arm64) npm install --no-save "@rollup/rollup-linux-arm64-gnu@$rollup_version" "@lancedb/lancedb-linux-arm64-gnu@$lancedb_version" ;; \ + x64) npm install --no-save "@rollup/rollup-linux-x64-gnu@$rollup_version" "@lancedb/lancedb-linux-x64-gnu@$lancedb_version" ;; \ + *) echo "No explicit native package install for arch: $node_arch" ;; \ + esac RUN npx turbo build # ── Stage 2: runtime ──────────────────────────────────────────────────────── -FROM node:20-alpine +FROM node:20-bookworm-slim # OCI image labels LABEL org.opencontainers.image.title="OpenDocuments" \ @@ -16,8 +24,14 @@ LABEL org.opencontainers.image.title="OpenDocuments" \ org.opencontainers.image.source="https://github.com/joungminsung/OpenDocuments" \ org.opencontainers.image.licenses="MIT" +# Runtime utilities for health checks and TLS. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates wget \ + && rm -rf /var/lib/apt/lists/* + # Create non-root user and group -RUN addgroup -S opendocs && adduser -S opendocs -G opendocs +RUN groupadd --system opendocs \ + && useradd --system --gid opendocs --home-dir /app --shell /usr/sbin/nologin opendocs WORKDIR /app @@ -28,6 +42,7 @@ COPY --from=builder /app/packages/server/dist packages/server/dist COPY --from=builder /app/packages/server/package.json packages/server/ COPY --from=builder /app/packages/cli/dist packages/cli/dist COPY --from=builder /app/packages/cli/package.json packages/cli/ +COPY --from=builder /app/packages/cli/node_modules packages/cli/node_modules COPY --from=builder /app/packages/web/dist packages/web/dist COPY --from=builder /app/plugins/ plugins/ COPY --from=builder /app/node_modules node_modules @@ -46,6 +61,6 @@ USER opendocs EXPOSE 3000 HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ - CMD wget -qO- http://localhost:3000/health || exit 1 + CMD wget -qO- http://localhost:3000/healthz || exit 1 CMD ["node", "packages/cli/dist/index.js", "start", "--port", "3000"] diff --git a/README.ko.md b/README.ko.md index 92d4c95..d32bcc8 100644 --- a/README.ko.md +++ b/README.ko.md @@ -308,32 +308,28 @@ parserFallbacks: { ### 클라우드 제공자 -| 제공자 | 모델 | 임베딩 | 적합한 용도 | -| --------------------- | ------------------------------------------------------------ | ---------------------------- | ------------------------------------------------------------------------------ | -| **OpenAI** | GPT-5.4, GPT-5.4-mini, GPT-4.1, o3, o4-mini | text-embedding-3-small/large | 범용, 비전, 추론 | -| **Anthropic** | Claude Opus 4.6, Claude Sonnet 4.6, Claude Haiku 4.5 | -- (별도 제공자 사용) | 긴 컨텍스트(1M), 코딩, 분석 | -| **Google** | Gemini 3.1 Pro, Gemini 3.1 Flash Lite, Gemini 3.0 Deep Think | text-embedding-005 | 멀티모달, 다국어 | -| **xAI** | Grok 4, Grok 4 Heavy, Grok 4.1 Fast | Grok embedding | 실시간 지식, 코드 | -| **DeepSeek** | DeepSeek-V3.2, DeepSeek-R1, DeepSeek-V4 (upcoming) | -- (별도 제공자 사용) | 비용 효율적 추론, 164K 컨텍스트 | -| **Mistral** | Mistral Small 4 (MoE), Large 2.1, Codestral, Pixtral | mistral-embed (1024) | 유럽 데이터 레지던시, 코딩, 비전 | -| **OpenAI-compatible** | 모든 OpenAI 호환 엔드포인트 | 엔드포인트에 따라 다름 | vLLM, LM Studio, Together, Fireworks, Groq, DeepInfra, SiliconFlow, OpenRouter | +| 제공자 | 모델 | 임베딩 | 적합한 용도 | +| ------------- | ------------------------------------------------------------ | ---------------------------- | --------------------------- | +| **OpenAI** | GPT-5.4, GPT-5.4-mini, GPT-4.1, o3, o4-mini | text-embedding-3-small/large | 범용, 비전, 추론 | +| **Anthropic** | Claude Opus 4.6, Claude Sonnet 4.6, Claude Haiku 4.5 | -- (별도 제공자 사용) | 긴 컨텍스트(1M), 코딩, 분석 | +| **Google** | Gemini 3.1 Pro, Gemini 3.1 Flash Lite, Gemini 3.0 Deep Think | text-embedding-005 | 멀티모달, 다국어 | +| **xAI** | Grok 4, Grok 4 Heavy, Grok 4.1 Fast | Grok embedding | 실시간 지식, 코드 | ### 로컬 모델(Ollama) -| 모델 | 활성 파라미터 | 전체 파라미터 | 비전 | 한국어 | 적합한 용도 | -| -------------------------- | ------------------- | ------------- | ---- | --------- | ----------------------------------------------- | -| **Qwen 3.5 27B** | 27B (dense) | 27B | Yes | Excellent | 범용(32GB+ RAM) | -| **Qwen 3.5 9B** | 9B (dense) | 9B | Yes | Excellent | 중급 사양(16GB RAM) | -| **Qwen 3.5-122B-A10B** | 10B (MoE) | 122B | Yes | Excellent | 고품질, 효율적 | -| **Llama 4 Scout** | 17B (MoE) | 109B | Yes | Good | 10M 컨텍스트 윈도우 | -| **Llama 4 Maverick** | 17B (MoE) | 400B | Yes | Good | 최고 수준 오픈소스 품질 | -| **DeepSeek V3.2** | 37B (MoE) | 671B | No | Good | 코딩, 추론 | -| **Gemma 4** | 27B / 12B / 4B / 1B | dense | Yes | Good | 최신 Google 오픈 모델, 128K 컨텍스트, 140+ 언어 | -| **Gemma 3 27B** | 27B | 27B | Yes | Good | 경량, 140+ 언어 | -| **Gemma 3 4B** | 4B | 4B | Yes | Good | 저사양 머신(8GB RAM) | -| **K-EXAONE** | 23B (MoE) | 236B | No | Best | 한국어 특화 | -| **EXAONE Deep 32B** | 32B | 32B | No | Best | 한국어 추론 | -| **Phi-4 Reasoning Vision** | 15B | 15B | Yes | Fair | 컴팩트 멀티모달 | +| 모델 | 활성 파라미터 | 전체 파라미터 | 비전 | 한국어 | 적합한 용도 | +| -------------------------- | ------------- | ------------- | ---- | --------- | ----------------------- | +| **Qwen 3.5 27B** | 27B (dense) | 27B | Yes | Excellent | 범용(32GB+ RAM) | +| **Qwen 3.5 9B** | 9B (dense) | 9B | Yes | Excellent | 중급 사양(16GB RAM) | +| **Qwen 3.5-122B-A10B** | 10B (MoE) | 122B | Yes | Excellent | 고품질, 효율적 | +| **Llama 4 Scout** | 17B (MoE) | 109B | Yes | Good | 10M 컨텍스트 윈도우 | +| **Llama 4 Maverick** | 17B (MoE) | 400B | Yes | Good | 최고 수준 오픈소스 품질 | +| **DeepSeek V3.2** | 37B (MoE) | 671B | No | Good | 코딩, 추론 | +| **Gemma 3 27B** | 27B | 27B | Yes | Good | 경량, 140+ 언어 | +| **Gemma 3 4B** | 4B | 4B | Yes | Good | 저사양 머신(8GB RAM) | +| **K-EXAONE** | 23B (MoE) | 236B | No | Best | 한국어 특화 | +| **EXAONE Deep 32B** | 32B | 32B | No | Best | 한국어 추론 | +| **Phi-4 Reasoning Vision** | 15B | 15B | Yes | Fair | 컴팩트 멀티모달 | ### 임베딩 모델 @@ -367,8 +363,11 @@ parserFallbacks: { | -------------- | ------------------------------------------------------------------------------------------------------------------------ | | **Chat** | 스트리밍 답변, 출처 인용, 신뢰도 점수, 피드백 버튼과 함께 질문합니다. fast/balanced/precise 프로필을 전환할 수 있습니다. | | **Documents** | 인덱싱된 문서 탐색, 드래그 앤 드롭 업로드, 문서 상세 보기, 휴지통/복원 방식의 soft-delete. | +| **Collections** | 문서를 목적별 지식 모음으로 구성하고 컬렉션 멤버십을 관리합니다. | | **Connectors** | 커넥터 동기화 상태와 마지막 동기화 시간 확인. | +| **Activity** | 런타임 상태, 모델 준비 상태, 인덱싱 활동을 확인합니다. | | **Plugins** | health indicator와 함께 설치된 플러그인 확인. | +| **Workspaces** | 워크스페이스 격리와 팀 모드 설정을 확인합니다. | | **Settings** | 다크/라이트 테마 전환, RAG 프로필 변경, 서버 버전 확인. | | **Admin** | 통계 대시보드, 검색 품질 메트릭, 페이지네이션된 쿼리 로그, 플러그인 상태, 커넥터 상태, 감사 로그. | @@ -376,7 +375,7 @@ parserFallbacks: { ### 2. CLI -파워 유저와 자동화를 위한 17개 명령어를 제공합니다. +파워 유저와 자동화를 위한 19개 명령어를 제공합니다. ```bash # 질문하기 @@ -398,17 +397,9 @@ cat README.md | opendocuments ask "Summarize this" --stdin opendocuments ask "List endpoints" --json | jq '.sources[].sourcePath' # 관리 -opendocuments doctor # Health check (provider별 API ping) +opendocuments doctor # Health check opendocuments auth create-key --name "ci-bot" --role member opendocuments export --output ./backup - -# 모델 관리 -opendocuments model list --suggestions # 설치된 모델 + 추천 모델 표시 -opendocuments model install-ollama # Ollama 원샷 설치(macOS/Linux) -opendocuments model pull gemma3:27b bge-m3 # 디스크 공간 확인 후 batch pull -opendocuments model set-key deepseek # API 키 입력 + .env 저장 -opendocuments model test # 설정된 LLM에 round-trip test -opendocuments model switch # 설정 파일 수정 없이 provider 변경 ``` ### 3. MCP 서버 @@ -616,22 +607,22 @@ console.log(result.confidence); // { level: 'high', score: 0.87 } git clone https://github.com/joungminsung/OpenDocuments.git cd OpenDocuments npm run setup # Install + build (one command) -npm run test # 51 test suites, ~300 tests +npm run test # 51개 테스트 작업, 600개 이상 테스트 npm run dev # Watch mode ``` ### 아키텍처 -| 패키지 | 역할 | 테스트 | -| ----------------------- | ----------------------------------------------------------------------------- | ------ | -| `@opendocuments/core` | 플러그인 시스템, RAG 엔진, ingest pipeline, storage, auth, security | 159 | -| `@opendocuments/server` | HTTP API(Hono), MCP 서버, auth middleware, widget | 27 | -| `@opendocuments/cli` | 17개 CLI 명령어(Commander.js) | 3 | -| `@opendocuments/web` | 7개 페이지 React SPA(Vite + Tailwind) | -- | -| `@opendocuments/client` | TypeScript SDK | 3 | -| 8개 모델 플러그인 | Ollama, OpenAI, Anthropic, Google, Grok, DeepSeek, Mistral, OpenAI-compatible | 41 | -| 9개 파서 플러그인 | PDF, DOCX, XLSX, HTML, Jupyter, Email, Code, PPTX, Structured | 37 | -| 8개 커넥터 플러그인 | GitHub, Notion, GDrive, S3, Confluence, Swagger, WebCrawler, WebSearch | 38 | +| 패키지 | 역할 | 테스트 | +| ----------------------- | ---------------------------------------------------------------------- | ------ | +| `@opendocuments/core` | 플러그인 시스템, RAG 엔진, ingest pipeline, storage, auth, security | 424 | +| `@opendocuments/server` | HTTP API(Hono), MCP 서버, auth middleware, widget | 63 | +| `@opendocuments/cli` | 19개 CLI 명령어(Commander.js) | 3 | +| `@opendocuments/web` | 9개 페이지 React SPA(Vite + Tailwind) | -- | +| `@opendocuments/client` | TypeScript SDK | 5 | +| 5개 모델 플러그인 | Ollama, OpenAI, Anthropic, Google, Grok | 42 | +| 9개 파서 플러그인 | PDF, DOCX, XLSX, HTML, Jupyter, Email, Code, PPTX, Structured | 37 | +| 8개 커넥터 플러그인 | GitHub, Notion, GDrive, S3, Confluence, Swagger, WebCrawler, WebSearch | 42 | 규칙, 테스트 패턴, 플러그인 개발 가이드는 [CONTRIBUTING.md](CONTRIBUTING.md)를 참고하세요. @@ -642,7 +633,7 @@ npm run dev # Watch mode | 가이드 | 설명 | | ------------------------------------------------------------ | ------------------------------------- | | [빠른 시작](#빠른-시작) | 5분 안에 설치 및 실행 | -| [아키텍처](docs/architecture.ko.md) | 패키지 구조, 데이터 흐름, 설계 결정 | +| [아키텍처](docs/architecture.ko.md) | 패키지 구조, 데이터 흐름, 설계 결정 | | [Plugin API: Parsers](docs-site/plugins/parser-api.md) | 커스텀 문서 파서 만들기 | | [Plugin API: Connectors](docs-site/plugins/connector-api.md) | 외부 데이터 소스 연결 | | [Plugin API: Models](docs-site/plugins/model-api.md) | 커스텀 AI 제공자 추가 | diff --git a/README.md b/README.md index 2ad50f1..d38bc80 100644 --- a/README.md +++ b/README.md @@ -312,9 +312,6 @@ parserFallbacks: { | **Anthropic** | Claude Opus 4.6, Claude Sonnet 4.6, Claude Haiku 4.5 | -- (use separate provider) | Long context (1M), coding, analysis | | **Google** | Gemini 3.1 Pro, Gemini 3.1 Flash Lite, Gemini 3.0 Deep Think | text-embedding-005 | Multimodal, multilingual | | **xAI** | Grok 4, Grok 4 Heavy, Grok 4.1 Fast | Grok embedding | Real-time knowledge, code | -| **DeepSeek** | DeepSeek-V3.2, DeepSeek-R1, DeepSeek-V4 (upcoming) | -- (use separate provider) | Cost-efficient reasoning, 164K context | -| **Mistral** | Mistral Small 4 (MoE), Large 2.1, Codestral, Pixtral | mistral-embed (1024) | European data residency, coding, vision | -| **OpenAI-compatible** | Any OpenAI-compatible endpoint | Depends on endpoint | vLLM, LM Studio, Together, Fireworks, Groq, DeepInfra, SiliconFlow, OpenRouter | ### Local Models (via Ollama) @@ -326,7 +323,6 @@ parserFallbacks: { | **Llama 4 Scout** | 17B (MoE) | 109B | Yes | Good | 10M context window | | **Llama 4 Maverick** | 17B (MoE) | 400B | Yes | Good | Top open-source quality | | **DeepSeek V3.2** | 37B (MoE) | 671B | No | Good | Coding, reasoning | -| **Gemma 4** | 27B / 12B / 4B / 1B | dense | Yes | Good | Latest Google open model, 128K context, 140+ languages | | **Gemma 3 27B** | 27B | 27B | Yes | Good | Lightweight, 140+ languages | | **Gemma 3 4B** | 4B | 4B | Yes | Good | Low-spec machines (8GB RAM) | | **K-EXAONE** | 23B (MoE) | 236B | No | Best | Korean-specialized | @@ -365,8 +361,11 @@ Full-featured dashboard at `http://localhost:3000`: |------|-----------------| | **Chat** | Ask questions with streaming answers, source citations, confidence scores, feedback buttons. Switch between fast/balanced/precise profiles. | | **Documents** | Browse indexed documents, drag-and-drop upload, view document details, soft-delete with trash/restore. | +| **Collections** | Group documents into focused knowledge sets and manage collection membership. | | **Connectors** | See connector sync status and last sync times. | +| **Activity** | Inspect runtime health, model readiness, and indexing activity. | | **Plugins** | View installed plugins with health indicators. | +| **Workspaces** | Review workspace isolation and team-mode configuration. | | **Settings** | Toggle dark/light theme, change RAG profile, view server version. | | **Admin** | Stats dashboard, search quality metrics, paginated query logs, plugin health, connector status, audit logs. | @@ -374,7 +373,7 @@ Full-featured dashboard at `http://localhost:3000`: ### 2. CLI -17 commands for power users and automation: +19 commands for power users and automation: ```bash # Ask questions @@ -396,17 +395,9 @@ cat README.md | opendocuments ask "Summarize this" --stdin opendocuments ask "List endpoints" --json | jq '.sources[].sourcePath' # Administration -opendocuments doctor # Health check (per-provider API ping) +opendocuments doctor # Health check opendocuments auth create-key --name "ci-bot" --role member opendocuments export --output ./backup - -# Model management -opendocuments model list --suggestions # Show installed + curated models -opendocuments model install-ollama # One-shot Ollama install (macOS/Linux) -opendocuments model pull gemma3:27b bge-m3 # Batch pull with disk-space check -opendocuments model set-key deepseek # Prompt + save API key to .env -opendocuments model test # Round-trip test against configured LLM -opendocuments model switch # Change provider without editing config ``` ### 3. MCP Server @@ -610,7 +601,7 @@ Add a chat widget to your internal tools: git clone https://github.com/joungminsung/OpenDocuments.git cd OpenDocuments npm run setup # Install + build (one command) -npm run test # 51 test suites, ~300 tests +npm run test # 51 test tasks, 600+ tests npm run dev # Watch mode ``` @@ -618,14 +609,14 @@ npm run dev # Watch mode | Package | Role | Tests | |---------|------|-------| -| `@opendocuments/core` | Plugin system, RAG engine, ingest pipeline, storage, auth, security | 159 | -| `@opendocuments/server` | HTTP API (Hono), MCP server, auth middleware, widget | 27 | -| `@opendocuments/cli` | 17 CLI commands (Commander.js) | 3 | -| `@opendocuments/web` | React SPA with 7 pages (Vite + Tailwind) | -- | -| `@opendocuments/client` | TypeScript SDK | 3 | -| 8 model plugins | Ollama, OpenAI, Anthropic, Google, Grok, DeepSeek, Mistral, OpenAI-compatible | 41 | +| `@opendocuments/core` | Plugin system, RAG engine, ingest pipeline, storage, auth, security | 424 | +| `@opendocuments/server` | HTTP API (Hono), MCP server, auth middleware, widget | 63 | +| `@opendocuments/cli` | 19 CLI commands (Commander.js) | 3 | +| `@opendocuments/web` | React SPA with 9 pages (Vite + Tailwind) | -- | +| `@opendocuments/client` | TypeScript SDK | 5 | +| 5 model plugins | Ollama, OpenAI, Anthropic, Google, Grok | 42 | | 9 parser plugins | PDF, DOCX, XLSX, HTML, Jupyter, Email, Code, PPTX, Structured | 37 | -| 8 connector plugins | GitHub, Notion, GDrive, S3, Confluence, Swagger, WebCrawler, WebSearch | 38 | +| 8 connector plugins | GitHub, Notion, GDrive, S3, Confluence, Swagger, WebCrawler, WebSearch | 42 | See [CONTRIBUTING.md](CONTRIBUTING.md) for conventions, test patterns, and plugin development guide. diff --git a/assets/demo.gif b/assets/demo.gif index e9fab6d..9048dbd 100644 Binary files a/assets/demo.gif and b/assets/demo.gif differ diff --git a/assets/demo.tape b/assets/demo.tape index f3febdf..8c6ec7a 100644 --- a/assets/demo.tape +++ b/assets/demo.tape @@ -1,4 +1,4 @@ -# OpenDocuments CLI Demo +# OpenDocuments Demo Output assets/demo.gif Set Shell "bash" @@ -10,7 +10,7 @@ Set Padding 15 Set TypingSpeed 35ms Set CursorBlink false -# Pre-setup: docs and a local demo model endpoint already exist in /tmp/od-demo +# Pre-setup: docs already exist in /tmp/od-demo Require opendocuments Sleep 500ms @@ -22,16 +22,16 @@ Type "opendocuments index ." Enter Sleep 6s -Type `opendocuments ask "What is the AI team budget?" --profile fast` +Type `opendocuments ask "How does authentication work?"` Enter -Sleep 6s +Sleep 10s -Type `opendocuments ask "How does authentication work?" --profile fast` +Type `opendocuments ask "What is the AI team budget?" --profile fast` Enter -Sleep 7s +Sleep 8s Type "opendocuments doctor" Enter -Sleep 4s +Sleep 5s Sleep 2s diff --git a/assets/external-link-icon.svg b/assets/external-link-icon.svg new file mode 100644 index 0000000..8442473 --- /dev/null +++ b/assets/external-link-icon.svg @@ -0,0 +1,5 @@ + diff --git a/docker-compose.demo.yml b/docker-compose.demo.yml new file mode 100644 index 0000000..7c8d384 --- /dev/null +++ b/docker-compose.demo.yml @@ -0,0 +1,60 @@ +services: + opendocuments: + build: + context: . + dockerfile: Dockerfile + ports: + - "${OPENDOCUMENTS_PORT:-3333}:3000" + volumes: + - opendocuments-demo-data:/data + - ./docker/demo/opendocuments.config.ts:/app/opendocuments.config.ts:ro + - ./docker/demo/docs:/demo-docs:ro + - ./docker/demo/entrypoint.sh:/app/docker-entrypoint-demo.sh:ro + environment: + NODE_ENV: production + PORT: "3000" + OPENDOCUMENTS_DATA_DIR: /data + OPENDOCUMENTS_DEMO_INDEX: "${OPENDOCUMENTS_DEMO_INDEX:-true}" + OPENDOCUMENTS_DEMO_REINDEX: "${OPENDOCUMENTS_DEMO_REINDEX:-false}" + OPENDOCUMENTS_MODEL_PROVIDER: "${OPENDOCUMENTS_MODEL_PROVIDER:-google}" + OPENDOCUMENTS_MODEL_LLM: "${OPENDOCUMENTS_MODEL_LLM:-gemini-3.5-flash}" + OPENDOCUMENTS_MODEL_EMBEDDING: "${OPENDOCUMENTS_MODEL_EMBEDDING:-gemini-embedding-001}" + OPENDOCUMENTS_MODEL_BASE_URL: "${OPENDOCUMENTS_MODEL_BASE_URL:-}" + OPENDOCUMENTS_MODEL_EMBEDDING_DIMENSIONS: "${OPENDOCUMENTS_MODEL_EMBEDDING_DIMENSIONS:-3072}" + OPENDOCUMENTS_MODEL_THINKING_BUDGET: "${OPENDOCUMENTS_MODEL_THINKING_BUDGET:-0}" + OPENDOCUMENTS_MODEL_API_KEY: "${OPENDOCUMENTS_MODEL_API_KEY:-}" + GOOGLE_API_KEY: "${GOOGLE_API_KEY:-}" + OPENAI_API_KEY: "${OPENAI_API_KEY:-}" + OPENDOCUMENTS_RAG_PROFILE: "${OPENDOCUMENTS_RAG_PROFILE:-fast}" + command: ["sh", "/app/docker-entrypoint-demo.sh"] + restart: unless-stopped + depends_on: + ollama: + condition: service_healthy + required: false + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/healthz"] + interval: 30s + timeout: 10s + start_period: 20s + retries: 3 + + ollama: + image: ollama/ollama:latest + ports: + - "${OLLAMA_PORT:-11434}:11434" + volumes: + - opendocuments-demo-ollama:/root/.ollama + profiles: + - ollama + restart: unless-stopped + healthcheck: + test: ["CMD", "ollama", "list"] + interval: 30s + timeout: 10s + start_period: 30s + retries: 5 + +volumes: + opendocuments-demo-data: + opendocuments-demo-ollama: diff --git a/docker-compose.yml b/docker-compose.yml index a7a00de..41efe69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: condition: service_healthy required: false healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/healthz"] interval: 30s timeout: 10s start_period: 15s diff --git a/docker/demo/README.md b/docker/demo/README.md new file mode 100644 index 0000000..dc4ee43 --- /dev/null +++ b/docker/demo/README.md @@ -0,0 +1,92 @@ +# OpenDocuments Interview Demo + +This folder contains a Docker Compose demo profile for interviews. + +## Google Gemini demo + +Put your API key in the repository root `.env` file: + +```bash +GOOGLE_API_KEY=your-google-api-key +``` + +Then start the demo: + +```bash +docker compose -f docker-compose.demo.yml up --build +open http://localhost:3333 +``` + +Default models: + +- LLM: `gemini-3.5-flash` +- Embedding: `gemini-embedding-001` +- Thinking budget: `0` + +The demo disables Gemini thinking by default so streamed RAG answers stay concise +and do not lose the beginning of the response while presenting. + +The compose service indexes the bundled documents in `docker/demo/docs` on first start. + +You can also use OpenDocuments' generic model key in `.env`: + +```bash +OPENDOCUMENTS_MODEL_API_KEY=your-google-api-key +``` + +## Local Ollama demo + +```bash +OPENDOCUMENTS_MODEL_PROVIDER=ollama \ +OPENDOCUMENTS_MODEL_LLM=gemma4:12b \ +OPENDOCUMENTS_MODEL_EMBEDDING=nomic-embed-text:latest \ +OPENDOCUMENTS_MODEL_BASE_URL=http://host.docker.internal:11434 \ +docker compose -f docker-compose.demo.yml up --build +open http://localhost:3333 +``` + +This connects the app container to Ollama running on the Mac through +`http://host.docker.internal:11434`. + +## Override Ollama models + +```bash +OPENDOCUMENTS_MODEL_PROVIDER=ollama \ +OPENDOCUMENTS_MODEL_LLM=qwen3.6:27b \ +OPENDOCUMENTS_MODEL_EMBEDDING=nomic-embed-text:latest \ +docker compose -f docker-compose.demo.yml up --build +``` + +## Internal Ollama container + +If you want Compose to run its own Ollama container instead of using the Mac host: + +```bash +OPENDOCUMENTS_MODEL_PROVIDER=ollama \ +OPENDOCUMENTS_MODEL_LLM=gemma4:12b \ +OPENDOCUMENTS_MODEL_EMBEDDING=nomic-embed-text:latest \ +OPENDOCUMENTS_MODEL_BASE_URL=http://ollama:11434 \ +docker compose -f docker-compose.demo.yml --profile ollama up --build +``` + +The first internal Ollama run can take several minutes because models must be pulled. + +## Cloud-backed demo + +```bash +OPENDOCUMENTS_MODEL_PROVIDER=openai \ +OPENDOCUMENTS_MODEL_LLM=gpt-4o-mini \ +OPENDOCUMENTS_MODEL_EMBEDDING=text-embedding-3-small \ +OPENDOCUMENTS_MODEL_EMBEDDING_DIMENSIONS=1536 \ +OPENAI_API_KEY=sk-... \ +docker compose -f docker-compose.demo.yml up --build +``` + +## Useful reset + +```bash +docker compose -f docker-compose.demo.yml down -v +``` + +Run the reset when you switch embedding models or dimensions. The demo entrypoint +also resets its demo volume automatically when those settings change. diff --git a/docker/demo/docs/01-project-overview.md b/docker/demo/docs/01-project-overview.md new file mode 100644 index 0000000..6cdb022 --- /dev/null +++ b/docker/demo/docs/01-project-overview.md @@ -0,0 +1,17 @@ +# OpenDocuments Project Overview + +OpenDocuments is a self-hosted RAG platform for private document search. It connects to document sources, parses files, chunks content, creates embeddings, stores metadata in SQLite, stores vectors in LanceDB, and answers natural-language questions with source citations. + +The project is useful when an organization has knowledge spread across GitHub, Notion, Google Drive, Confluence, S3, API documents, spreadsheets, PDFs, and local files. + +Core product surfaces: + +- Web UI for chat, document management, plugins, connectors, settings, health, and admin views. +- CLI named `opendocuments` for init, start, index, ask, doctor, backup, restore, plugin, connector, auth, workspace, and document commands. +- HTTP API from the server package. +- TypeScript SDK from `@opendocuments/client`. +- MCP server so AI coding assistants can search the same internal knowledge base. + +Interview positioning: + +OpenDocuments is not just a chatbot UI. It is an application layer around RAG that includes ingestion, parsing, retrieval, answer generation, source citations, storage, security, operations tooling, and plugin extensibility. diff --git a/docker/demo/docs/02-architecture.md b/docker/demo/docs/02-architecture.md new file mode 100644 index 0000000..d7baa30 --- /dev/null +++ b/docker/demo/docs/02-architecture.md @@ -0,0 +1,25 @@ +# Architecture + +OpenDocuments is a TypeScript monorepo. + +Main packages: + +- `packages/core`: business logic, RAG pipeline, document ingest, storage, auth, plugin registry, connectors, parsers, event bus, and security utilities. +- `packages/server`: Hono HTTP API, MCP server, auth middleware, rate limiting, and static web serving. +- `packages/cli`: Commander.js CLI with operational commands. +- `packages/web`: React and Vite single-page application. +- `packages/client`: TypeScript SDK. + +Plugin families: + +- Model plugins: Ollama, OpenAI, Anthropic, Google, and Grok. +- Parser plugins: PDF, DOCX, XLSX, HTML, Jupyter, email, code, PPTX, and built-in text formats. +- Connector plugins: GitHub, Notion, Google Drive, S3, Confluence, Swagger, web crawler, and web search. + +Storage: + +- SQLite stores metadata, workspaces, documents, chunks, conversations, API keys, audit data, collections, and tags. +- LanceDB stores vector embeddings for semantic retrieval. +- FTS5 provides keyword search over indexed chunks. + +The server is intentionally thin. Most behavior lives in core so the CLI, HTTP API, Web UI, SDK, and MCP server use the same underlying logic. diff --git a/docker/demo/docs/03-rag-pipeline.md b/docker/demo/docs/03-rag-pipeline.md new file mode 100644 index 0000000..3612e87 --- /dev/null +++ b/docker/demo/docs/03-rag-pipeline.md @@ -0,0 +1,26 @@ +# RAG Pipeline + +The RAG flow starts with ingestion. + +## 한국어 데모 요약 + +OpenDocuments의 RAG 파이프라인은 문서 수집, 파싱, 청킹, 임베딩, 저장, 검색, 답변 생성 순서로 동작합니다. 커넥터나 로컬 파일에서 원본 문서를 가져오고, 파서가 문서를 청크로 변환한 뒤, 파일 유형에 맞는 청킹 전략을 적용합니다. 이후 모델 플러그인이 임베딩을 만들고, 메타데이터는 SQLite에, 벡터는 LanceDB에 저장합니다. + +사용자가 질문하면 semantic vector search, FTS5 keyword search, reranking, HyDE, multi-query retrieval, cross-lingual query expansion, parent-document recall 같은 검색 기술을 선택한 프로필에 따라 조합합니다. 마지막으로 답변 생성기가 검색된 청크를 근거로 출처가 포함된 답변을 만듭니다. + +1. A connector or local file provides raw content. +2. A parser turns the raw document into parsed chunks. +3. The chunking layer uses file-aware strategies for prose, code, tables, and structured data. +4. Optional contextual retrieval can add situating context to chunks before embedding. +5. Embeddings are created through a model plugin. +6. Metadata is written to SQLite and vectors are written to LanceDB. +7. Search combines semantic retrieval, FTS5 keyword search, reranking, HyDE, multi-query retrieval, cross-lingual query expansion, and parent-document recall depending on the selected profile. +8. The answer generator produces a grounded response with source citations. + +Retrieval profiles: + +- `fast`: lower latency, fewer retrieval features. +- `balanced`: default profile for good quality and reasonable speed. +- `precise`: deeper retrieval, reranking, and broader recall for harder questions. + +The important engineering decision is that retrieval is not a single vector search call. The system uses multiple recall and ranking techniques so answers remain useful across code, documentation, spreadsheets, policies, and mixed-language content. diff --git a/docker/demo/docs/04-security-and-operations.md b/docker/demo/docs/04-security-and-operations.md new file mode 100644 index 0000000..97bb6ba --- /dev/null +++ b/docker/demo/docs/04-security-and-operations.md @@ -0,0 +1,23 @@ +# Security And Operations + +Security choices: + +- API keys and secrets are read from environment variables, not hardcoded. +- SQL access uses parameterized statements. +- FTS5 user input is escaped before query execution. +- LanceDB filters are built through a helper instead of raw string interpolation. +- Team mode protects HTTP endpoints with auth middleware. +- Workspace isolation keeps documents, conversations, and admin data scoped to the active workspace. +- Error responses avoid exposing stack traces or internal paths in production-facing paths. + +Operational choices: + +- `opendocuments doctor` checks configuration, model connectivity, and Ollama availability. +- Backup and restore commands snapshot SQLite and LanceDB data. +- Health endpoints support container orchestration. +- The Docker image runs as a non-root user. +- The project supports local Ollama for private deployments and cloud providers when latency or quality is more important. + +Interview point: + +The project was designed as a product-shaped system rather than a script. It includes diagnostics, health checks, CLI workflows, persistent storage, plugin boundaries, and deployment paths. diff --git a/docker/demo/docs/05-demo-questions.md b/docker/demo/docs/05-demo-questions.md new file mode 100644 index 0000000..294435a --- /dev/null +++ b/docker/demo/docs/05-demo-questions.md @@ -0,0 +1,31 @@ +# Demo Questions + +Use these questions during an interview demo. + +Product questions: + +- What problem does OpenDocuments solve? +- What are the main surfaces of the product? +- How is this different from a simple chatbot wrapper? + +Architecture questions: + +- Explain the monorepo structure. +- Why does most business logic live in `packages/core`? +- How do the CLI, Web UI, HTTP API, SDK, and MCP server share behavior? + +RAG questions: + +- How does the ingestion pipeline work? +- What retrieval techniques does OpenDocuments use? +- What is the difference between the fast, balanced, and precise profiles? + +Security and operations questions: + +- How are API keys handled? +- What security protections exist around SQL, FTS5, and workspace isolation? +- How can this be deployed for private documents? + +Suggested opening question: + +"What is OpenDocuments, and how is it architected?" diff --git a/docker/demo/docs/06-github-connector.md b/docker/demo/docs/06-github-connector.md new file mode 100644 index 0000000..013dee7 --- /dev/null +++ b/docker/demo/docs/06-github-connector.md @@ -0,0 +1,45 @@ +# GitHub Connector + +The GitHub connector lets OpenDocuments ingest repository documentation and selected source files into the same RAG index as local files. + +## 한국어 데모 요약 + +GitHub 자동 수집 기능은 연결된 저장소를 주기적으로 읽어 변경된 문서를 다시 인덱싱하는 기능입니다. OpenDocuments는 GitHub 저장소의 파일 목록을 읽고, path filter에 맞는 파일만 가져와 chunking, embedding, indexing을 수행합니다. 자동 수집은 `syncInterval` 값에 따라 polling 방식으로 실행되며, 변경되지 않은 파일은 content hash 비교로 skip합니다. + +면접에서 "GitHub 검색이 어떻게 되나요?", "GitHub 자동 수집이 되나요?", "GitHub 문서는 어떻게 인덱싱되나요?"라고 물으면 이 문서를 근거로 답하면 됩니다. + +Current demo configuration: + +- Repository: `joungminsung/OpenDocuments` +- Branch: `main` +- Sync mode: automatic polling plus manual sync +- Default polling interval: 60 seconds in the running demo +- Authentication: public repositories work without a token; private repositories require a GitHub Personal Access Token +- Path filters can restrict ingestion to specific files or folders such as `README.md`, `docs/`, `packages/core/src/rag/`, or `plugins/connector-github/src/` + +GitHub ingestion flow: + +1. The admin UI sends repository, branch, token, path filters, and sync interval to the server. +2. The server creates the `@opendocuments/connector-github` plugin and stores the connector config in SQLite. +3. The connector checks GitHub connectivity with the repository API. +4. During sync, it reads the recursive Git tree for the selected branch. +5. It filters files by path prefix and supported extension. +6. It fetches matching file contents through the GitHub contents API. +7. The ingest pipeline parses, chunks, embeds, and stores the documents. +8. The connector compares content hashes on later syncs, so unchanged files are skipped. + +Automatic collection: + +When a GitHub connector is registered, the server starts a background timer using the saved `syncInterval`. On every interval, OpenDocuments reads GitHub again and updates `lastSyncedAt`. If GitHub reports the same content hash, the document is not re-embedded. If the hash changes, the document is re-fetched and re-indexed. + +What this proves in a demo: + +- OpenDocuments is not only answering from bundled sample text. +- It can connect to a live GitHub repository. +- It can use repository content as cited answer sources. +- It can keep GitHub-backed documents fresh without pressing the sync button every time. +- The same RAG pipeline handles local documents and GitHub documents. + +Known next step: + +The current automatic collection uses polling. A production-grade GitHub integration could add GitHub webhooks or a GitHub App so pushes trigger sync immediately instead of waiting for the next interval. diff --git a/docker/demo/docs/07-search-quality.md b/docker/demo/docs/07-search-quality.md new file mode 100644 index 0000000..5692903 --- /dev/null +++ b/docker/demo/docs/07-search-quality.md @@ -0,0 +1,41 @@ +# Search Quality And Accuracy + +OpenDocuments is designed to improve answer accuracy by grounding each response in retrieved source chunks instead of relying only on the language model. + +## 한국어 데모 요약 + +OpenDocuments의 정확도는 모델 자체의 기억이 아니라 인덱싱된 문서 검색 결과에 기반합니다. 질문이 들어오면 semantic vector search, keyword search, retrieval profile, reranking, source citation, confidence score를 통해 답변 근거를 구성합니다. 답변에는 출처가 붙기 때문에 사용자는 어떤 문서와 어떤 섹션을 근거로 답했는지 확인할 수 있습니다. + +정확도를 높이려면 질문과 관련된 문서가 실제로 인덱싱되어 있어야 합니다. GitHub connector의 경우 README만 인덱싱하면 답변 근거가 README에 제한됩니다. GitHub connector 구현을 설명하려면 `plugins/connector-github/src/index.ts`와 `packages/core/src/connector/manager.ts` 같은 핵심 파일도 인덱싱해야 합니다. + +면접에서 "정확도는 어떻게 보장하나요?", "검색 품질은 어떻게 높이나요?", "출처 기반 답변은 어떻게 동작하나요?"라고 물으면 이 문서를 근거로 답하면 됩니다. + +Accuracy controls: + +- Source-grounded generation: answers include citations from indexed documents. +- Hybrid retrieval: semantic vector search is combined with keyword search when the selected profile enables it. +- Reranking: retrieved chunks can be rescored before they are passed to the model. +- Parent-document recall: the system can recover broader context around a matched chunk. +- Retrieval profiles: `fast`, `balanced`, and `precise` trade latency for deeper recall. +- Confidence scoring: each answer receives a confidence level and reason. +- Hash-based re-indexing: unchanged GitHub files are skipped, while changed files are re-indexed. + +Why citations matter: + +The answer should be treated as useful only when the cited sources support it. In the web UI, sources show the document name, match score, heading, and source path. GitHub-backed documents use paths such as `github://owner/repo/path`, while bundled demo documents use `/demo-docs/...`. + +How to improve retrieval during a demo: + +- Ask specific questions such as "GitHub connector는 자동 수집을 어떻게 하나요?" +- Use the `fast` profile during the bundled interview demo because the corpus is small and the direct source match is easier to explain. +- Use the `precise` profile for larger corpora or harder architecture questions where broader recall matters more than latency. +- Connect narrower GitHub path filters first, such as `docs/` and key source folders, instead of indexing an entire repository. +- Add documents that explain the product's current implemented behavior, because RAG quality depends on what has been indexed. + +Important limitation: + +RAG accuracy is not magic. If the indexed corpus does not contain the detail, the model can only infer or answer weakly. For example, if GitHub indexing only includes `README.md`, then questions about connector internals will mostly cite README sections. Indexing `plugins/connector-github/src/`, `packages/core/src/connector/`, and server admin routes gives the system better evidence for GitHub implementation questions. + +Interview positioning: + +The strongest accuracy story is that OpenDocuments exposes the full chain: source connector, parser, chunking, embedding model, vector store, keyword search, retrieval profile, cited generation, and confidence score. This makes the system debuggable. When an answer is weak, you can inspect whether the missing piece is the corpus, retrieval, reranking, or generation. diff --git a/docker/demo/entrypoint.sh b/docker/demo/entrypoint.sh new file mode 100755 index 0000000..c2cc93f --- /dev/null +++ b/docker/demo/entrypoint.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +cd /app + +if [ "${OPENDOCUMENTS_MODEL_PROVIDER:-openai}" = "ollama" ] && [ -z "${OPENDOCUMENTS_MODEL_BASE_URL:-}" ]; then + export OPENDOCUMENTS_MODEL_BASE_URL="http://ollama:11434" +fi + +if [ "${OPENDOCUMENTS_DEMO_INDEX:-true}" = "true" ]; then + key_state="nokey" + if [ -n "${OPENDOCUMENTS_MODEL_API_KEY:-}" ] || [ -n "${GOOGLE_API_KEY:-}" ] || [ -n "${OPENAI_API_KEY:-}" ]; then + key_state="key" + fi + + model_state="${OPENDOCUMENTS_MODEL_PROVIDER:-openai}-${OPENDOCUMENTS_MODEL_EMBEDDING:-default}-${OPENDOCUMENTS_MODEL_EMBEDDING_DIMENSIONS:-default}-${key_state}" + state_file="/data/.demo-model-state" + if [ -f "$state_file" ] && [ "$(cat "$state_file")" != "$model_state" ]; then + echo "[demo] Model embedding settings changed; resetting demo data volume." + find /data -mindepth 1 -maxdepth 1 -exec rm -rf {} + + fi + + marker="/data/.demo-indexed-${model_state}" + if [ "${OPENDOCUMENTS_DEMO_REINDEX:-false}" = "true" ] || [ ! -f "$marker" ]; then + echo "[demo] Indexing bundled interview documents..." + if node packages/cli/dist/index.js index /demo-docs --reindex; then + echo "$model_state" > "$state_file" + date > "$marker" + else + echo "[demo] Demo indexing failed; starting the server anyway." + fi + else + echo "[demo] Bundled interview documents already indexed. Set OPENDOCUMENTS_DEMO_REINDEX=true to refresh." + fi +fi + +echo "[demo] Starting OpenDocuments at http://localhost:${PORT:-3000}" +exec node packages/cli/dist/index.js start --port "${PORT:-3000}" diff --git a/docs-site/guide/configuration.md b/docs-site/guide/configuration.md index f8c9019..b98bc8c 100644 --- a/docs-site/guide/configuration.md +++ b/docs-site/guide/configuration.md @@ -12,7 +12,7 @@ export default defineConfig({ mode: 'personal', // 'personal' | 'team' model: { - provider: 'ollama', // 'ollama' | 'openai' | 'anthropic' | 'google' | 'grok' | 'deepseek' | 'mistral' | 'openai-compatible' + provider: 'ollama', // 'ollama' | 'openai' | 'anthropic' | 'google' | 'grok' llm: 'qwen2.5:14b', embedding: 'bge-m3', // embeddingProvider: 'openai', // Use different provider for embeddings @@ -51,6 +51,12 @@ export default defineConfig({ method: 'replace', // 'replace' | 'hash' | 'remove' }, }, + transport: { + enforceHTTPS: true, + // Comma-separated proxy IPs/CIDRs; forwarded headers from other peers are ignored. + proxy: '10.0.0.10,172.18.0.0/16', + allowedOrigins: ['https://docs.example.com'], + }, audit: { enabled: true }, }, @@ -74,34 +80,12 @@ export default defineConfig({ ```typescript model: { provider: 'ollama', - llm: 'qwen2.5:14b', // Any Ollama model: gemma3, gemma2, llama3.3, qwen2.5, phi4, deepseek-r1... + llm: 'qwen2.5:14b', // Any Ollama model embedding: 'bge-m3', baseUrl: 'http://localhost:11434', // Default } ``` -Popular Ollama models (April 2026): - -```bash -# Google Gemma 4 (128K context, 140+ languages, multimodal) -ollama pull gemma3:27b # Gemma 3 flagship -ollama pull gemma3:12b # Balanced -ollama pull gemma3:4b # Low-spec -ollama pull gemma3n # Selective activation for laptops/phones -ollama pull gemma4 # Newest release (if available) - -# Alibaba Qwen 3.5 -ollama pull qwen2.5:14b -ollama pull qwen3.5:9b - -# Meta Llama 4 -ollama pull llama4:scout -ollama pull llama4:maverick - -# DeepSeek R1 distilled -ollama pull deepseek-r1:14b -``` - ### Cloud Models ```typescript @@ -129,43 +113,6 @@ model: { embedding: 'text-embedding-004', apiKey: process.env.GOOGLE_API_KEY, } - -// DeepSeek (V3.2 / R1 — no embedding, cheapest reasoning) -model: { - provider: 'deepseek', - llm: 'deepseek-chat', // or 'deepseek-reasoner' for R1 - embedding: 'bge-m3', - embeddingProvider: 'ollama', - apiKey: process.env.DEEPSEEK_API_KEY, -} - -// Mistral (Small 4 / Large / Codestral / Pixtral) -model: { - provider: 'mistral', - llm: 'mistral-small-latest', // MoE w/ reasoning + vision + code - embedding: 'mistral-embed', - apiKey: process.env.MISTRAL_API_KEY, -} - -// Generic OpenAI-compatible — works with vLLM, LM Studio, Together, Fireworks, Groq, DeepInfra, SiliconFlow, OpenRouter -model: { - provider: 'openai-compatible', - baseUrl: 'https://api.groq.com/openai/v1', // required - llm: 'llama-4-70b-instruct', - embedding: 'bge-m3', - embeddingProvider: 'ollama', // Groq has no embeddings - apiKey: process.env.GROQ_API_KEY, - // extraHeaders: { 'HTTP-Referer': 'https://myapp.com' }, // e.g. for OpenRouter -} - -// Self-hosted vLLM -model: { - provider: 'openai-compatible', - baseUrl: 'http://vllm.internal:8000/v1', - llm: 'meta-llama/Llama-4-70B-Instruct', - embedding: 'BAAI/bge-m3', - apiKey: '', // vLLM accepts empty -} ``` ## Environment Variables @@ -176,15 +123,9 @@ API keys should be stored in `.env`, never in the config file: # .env OPENAI_API_KEY=sk-... ANTHROPIC_API_KEY=sk-ant-... -GOOGLE_API_KEY=... -GROK_API_KEY=xai-... -DEEPSEEK_API_KEY=sk-... -MISTRAL_API_KEY=... -OPENAI_COMPATIBLE_API_KEY=... # For generic openai-compatible provider -OPENAI_COMPATIBLE_BASE_URL=https://... # Optional default for openai-compatible baseUrl GITHUB_TOKEN=ghp_... NOTION_TOKEN=ntn_... -TAVILY_API_KEY=tvly-... # For web search integration +TAVILY_API_KEY=tvly-... # For web search integration ``` The `.env` file is automatically loaded before config resolution. @@ -206,10 +147,22 @@ The `.env` file is automatically loaded before config resolution. ```typescript export default defineConfig({ mode: 'team', + security: { + transport: { + enforceHTTPS: true, + // Set this when TLS terminates at a reverse proxy. Use only IPs/CIDRs + // controlled by your deployment; untrusted X-Forwarded-* headers are ignored. + proxy: '10.0.0.10,172.18.0.0/16', + }, + }, // Enables: API key auth, RBAC, workspace isolation, audit logging }) ``` +Container liveness probes use the unauthenticated `/healthz` endpoint. Detailed +health and readiness endpoints under `/api/v1` continue to require authentication +in team mode. + Create API keys: ```bash diff --git a/docs-site/sdk/guide.md b/docs-site/sdk/guide.md index ec0cb7b..d00d4ef 100644 --- a/docs-site/sdk/guide.md +++ b/docs-site/sdk/guide.md @@ -5,13 +5,13 @@ The OpenDocuments SDK provides a simple TypeScript client for the REST API. ## Installation ```bash -npm install opendocuments-client +npm install @opendocuments/client ``` ## Quick Start ```typescript -import { OpenDocumentsClient } from 'opendocuments-client' +import { OpenDocumentsClient } from '@opendocuments/client' const client = new OpenDocumentsClient({ baseUrl: 'http://localhost:3000', @@ -19,9 +19,10 @@ const client = new OpenDocumentsClient({ }) // Ask a question -const result = await client.ask('How does authentication work?') +const result = await client.ask('How does authentication work?', { profile: 'balanced' }) console.log(result.answer) console.log(result.sources) +console.log(result.confidence.level) // List indexed documents const { documents } = await client.listDocuments() @@ -32,6 +33,13 @@ await client.uploadDocument(file) // Delete a document await client.deleteDocument('doc-id') + +// Continue a saved conversation +const conversation = await client.createConversation('Auth review') +await client.ask('What changed in the auth docs?', { + profile: 'precise', + conversationId: conversation.id, +}) ``` ## API Reference @@ -43,8 +51,19 @@ Ask a question about indexed documents. |-----------|------|---------|-------------| | query | string | required | The question to ask | | options.profile | string | 'balanced' | RAG profile: fast, balanced, precise | +| options.conversationId | string | undefined | Existing conversation to append to | + +Returns: `Promise` with `answer`, `sources`, and `confidence`. -Returns: `Promise` with `answer`, `sources`, `confidence` +`confidence` is an object: + +```typescript +{ + score: number + level: 'high' | 'medium' | 'low' | 'none' + reason: string +} +``` ### `listDocuments()` Returns all indexed documents. @@ -52,9 +71,20 @@ Returns all indexed documents. ### `uploadDocument(file)` Upload and index a document file. +Returns: `{ documentId, chunks, status }`. + ### `deleteDocument(id)` Soft-delete a document by ID. +### Conversation Methods + +- `listConversations({ limit, offset })` +- `createConversation(title?)` +- `listConversationMessages(id)` +- `updateConversation(id, { title })` +- `deleteConversation(id)` +- `shareConversation(id)` + ## Error Handling ```typescript diff --git a/docs/architecture.ko.md b/docs/architecture.ko.md index f00fdba..4c0a95e 100644 --- a/docs/architecture.ko.md +++ b/docs/architecture.ko.md @@ -4,6 +4,8 @@ OpenDocuments는 GitHub, Notion, Google Drive, 로컬 파일 등 여러 소스에 흩어진 문서를 검색 가능한 지식 베이스로 만들고, 자연어 질문에 출처 기반 답변을 제공하는 자체 호스팅 RAG 플랫폼입니다. +세부 실행 흐름은 [OpenDocuments 세부 파이프라인](pipeline.ko.md)을 참고하세요. + 설계의 중심 원칙은 세 가지입니다. - **Core-first 설계**: 비즈니스 로직은 `@opendocuments/core`에 두고 CLI, server, web, SDK는 이를 재사용합니다. @@ -267,4 +269,3 @@ flowchart LR | Core-first monorepo | CLI, server, web, SDK, MCP가 동일한 business logic을 재사용하기 위해 선택 | | Hono server layer | core service를 감싸는 가벼운 TypeScript 친화 HTTP layer가 필요해 선택 | | Retrieval profiles | fast, balanced, precise 모드로 속도와 검색 품질을 조절하기 위해 선택 | - diff --git a/docs/pipeline-list.ko.md b/docs/pipeline-list.ko.md new file mode 100644 index 0000000..96065e6 --- /dev/null +++ b/docs/pipeline-list.ko.md @@ -0,0 +1,214 @@ +# OpenDocuments 파이프라인 목록 + +## 0. 사전 준비 파이프라인 + +- Runtime config 로드 +- Environment variable 로드 +- API key 환경변수 확인 +- Secret 하드코딩 여부 확인 +- Workspace context 확인 +- Team mode 여부 확인 +- Auth middleware 적용 +- Workspace isolation 확인 +- Plugin registry 조회 +- Plugin manifest 확인 +- `coreVersion` 호환성 확인 +- Connector plugin 등록 +- Parser plugin 등록 +- Model plugin 등록 +- SQLite 연결 +- LanceDB 연결 + +## 1. 문서 수집 파이프라인 + +- Source config 입력 +- Credential 확인 +- 문서 소스 선택 +- Connector plugin 선택 +- Connector 초기화 +- Source별 API 호출 +- 외부 문서 목록 탐색 +- 문서 목록 수집 +- 삭제 문서 감지 +- 변경 문서 감지 +- 원본 문서 가져오기 +- 원본 metadata 수집 +- Raw document 생성 +- MIME type 확인 + +## 2. 문서 파싱 파이프라인 + +- Parser plugin 선택 +- Parser 매칭 +- Parser 실행 +- 원본 콘텐츠 파싱 +- 텍스트 추출 +- 문서 구조 추출 +- Title 추출 +- Body text 추출 +- Section 정보 추출 +- Source path 연결 +- External id 연결 +- Modified time 연결 +- Normalized document 생성 +- 공통 document model로 정규화 +- Content hash 계산 +- 변경 여부 확인 + +## 3. Chunking 파이프라인 + +- Chunk boundary 결정 +- 문서 본문 chunking +- Chunk 생성 +- Chunk order 부여 +- Chunk 위치 정보 연결 +- Citation metadata 부여 +- Chunk metadata 생성 +- Embedding text 준비 + +## 4. Embedding 파이프라인 + +- Embedding provider 선택 +- Model plugin 초기화 +- Provider credential 주입 +- Chunk embedding 요청 +- Embedding API 호출 +- Embedding vector 수신 +- Embedding dimension 확인 +- Embedding result 정규화 + +## 5. 저장 파이프라인 + +- SQLite transaction 시작 +- Workspace table 조회 +- Document row upsert +- Chunk row upsert +- Metadata payload 저장 +- FTS5 row upsert +- SQLite FTS5 index 갱신 +- LanceDB vector upsert +- Job table 갱신 +- SQLite transaction commit +- Indexing result 기록 +- Ingest event 발행 +- Ingest job 상태 갱신 +- Connection 정리 + +## 6. 질문 입력 파이프라인 + +- 사용자 질문 입력 +- 진입점 확인 +- API key 또는 session 확인 +- 인증 확인 +- Workspace context 확인 +- 권한 범위 확인 +- Source filter 확인 +- 사용자 대상 error message 준비 + +## 7. Query 처리 파이프라인 + +- Query normalization 수행 +- Keyword query 준비 +- FTS5 `escapeFTS5Query()` 적용 +- Embedding query 준비 +- Question embedding 생성 +- LanceDB `buildWhereClause()` 적용 +- SQL parameterized statement 적용 + +## 8. Retrieval 파이프라인 + +- LanceDB dense search 실행 +- SQLite FTS5 sparse search 실행 +- Dense search 결과 수집 +- Sparse search 결과 수집 +- 검색 결과 병합 +- 중복 chunk 제거 +- Workspace filter 적용 +- Permission filter 적용 +- Source filter 적용 +- Score 계산 +- Rank 보정 +- Parent context 확장 +- Context candidate 선택 + +## 9. 답변 생성 파이프라인 + +- Context window 크기 확인 +- Context fitting 수행 +- Prompt 구성 +- Model plugin 선택 +- Answer generation 실행 +- 답변 초안 생성 +- Grounding 확인 +- Citation mapping 수행 +- Source reference 생성 +- Stack trace 응답 노출 차단 +- 내부 filesystem path 응답 노출 차단 +- Answer payload 생성 +- 사용자에게 답변 반환 + +## 10. 장애 확인 파이프라인 + +- 실패 단계 식별 +- Connector 로그 확인 +- Parser 로그 확인 +- Chunk 저장 여부 확인 +- FTS5 index 확인 +- LanceDB vector 확인 +- Embedding dimension 확인 +- Workspace filter 확인 +- Permission filter 확인 +- Citation metadata 확인 +- Model provider 응답 확인 +- 사용자 응답 error message 확인 +- 내부 로그 보존 +- 민감 정보 노출 여부 확인 + +## 11. 릴리즈 파이프라인 + +- Feature branch 작업 완료 +- `npm run build` 실행 +- `npm run typecheck` 실행 +- `npm run test` 실행 +- 1차 기능 리뷰 +- Edge case 확인 +- Error handling 확인 +- Test coverage 확인 +- 2차 보안 리뷰 +- SQL injection 확인 +- Secret 노출 확인 +- Raw query 사용 확인 +- `any` 타입 사용 확인 +- 성능 이슈 확인 +- 3차 스타일 리뷰 +- Naming convention 확인 +- ESM `.js` import 확인 +- CLI `log` 유틸리티 확인 +- Public API JSDoc 확인 +- Debug code 제거 확인 +- `npx changeset` 실행 +- 영향 package 선택 +- Bump type 선택 +- Changeset summary 작성 +- `.changeset/*.md` 확인 +- `npx changeset version` 실행 +- Package version 확인 +- Workspace dependency 확인 +- `CHANGELOG.md` 확인 +- `README.md` 확인 +- `README.ko.md` 확인 +- `docs-site` 관련 문서 확인 +- `npm run build && npm run typecheck && npm run test` 실행 +- Release commit 생성 +- Branch push +- PR 생성 +- GitHub Actions 확인 +- PR review 승인 +- Main branch merge +- `npx changeset publish` 실행 +- npm package version 확인 +- Global install 테스트 +- Git tag 생성 +- Git tag push +- GitHub Release 작성 +- Docs site 배포 확인 diff --git a/docs/pipeline.ko.md b/docs/pipeline.ko.md new file mode 100644 index 0000000..bdf6b3a --- /dev/null +++ b/docs/pipeline.ko.md @@ -0,0 +1,584 @@ +# OpenDocuments 세부 파이프라인 + +이 문서는 OpenDocuments의 핵심 실행 흐름을 운영 관점에서 정리한다. 범위는 문서 인덱싱, 질문 응답 RAG, 릴리즈 파이프라인이다. + +관련 문서: + +- [아키텍처 개요](architecture.ko.md) +- [Architecture](architecture.md) + +## 전체 흐름 + +OpenDocuments의 런타임은 외부 문서를 검색 가능한 지식 베이스로 변환한 뒤, 사용자 질문에 대해 검색된 근거를 바탕으로 답변을 생성한다. + +```mermaid +flowchart LR + SRC["문서 소스\nGitHub / Notion / Google Drive / Local / Web"] + CON["Connector Plugin"] + RAW["Raw Document"] + PAR["Parser Plugin"] + DOC["Normalized Document"] + CHK["Chunks"] + EMB["Embeddings"] + SQL[("SQLite\nmetadata + FTS5")] + VEC[("LanceDB\nvectors")] + ASK["User Question"] + RET["Retrieval"] + CTX["Context Assembly"] + LLM["Model Plugin"] + ANS["Answer + Sources"] + + SRC --> CON --> RAW --> PAR --> DOC --> CHK --> EMB + DOC --> SQL + CHK --> SQL + EMB --> VEC + ASK --> RET + SQL --> RET + VEC --> RET + RET --> CTX --> LLM --> ANS +``` + +핵심 원칙은 다음과 같다. + +- 비즈니스 로직은 `packages/core`에 둔다. +- 외부 시스템, 파일 형식, 모델 제공자는 플러그인으로 분리한다. +- 답변은 검색된 chunk와 source metadata를 근거로 생성한다. +- 사용자 입력 기반 검색 조건은 반드시 안전한 helper를 거친다. + +## 1. 문서 인덱싱 파이프라인 + +문서 인덱싱 파이프라인은 외부 소스의 문서를 가져와 SQLite와 LanceDB에 저장 가능한 검색 단위로 변환한다. + +```mermaid +sequenceDiagram + participant Job as Ingest Job + participant Connector as Connector Plugin + participant Parser as Parser Plugin + participant Core as Core Ingest Pipeline + participant Model as Model Provider + participant SQLite as SQLite / FTS5 + participant LanceDB as LanceDB + participant Bus as Typed Event Bus + + Job->>Bus: ingest.started + Job->>Connector: source config 전달 + Connector->>Connector: 문서 목록 탐색 + Connector->>Core: raw document + metadata 반환 + Core->>Parser: content type에 맞는 parser 선택 + Parser->>Core: normalized document 반환 + Core->>Core: chunk 생성 + Core->>Model: embedding 생성 + Core->>SQLite: document, chunk, metadata, FTS5 저장 + Core->>LanceDB: vector payload 저장 + Core->>Bus: ingest.completed 또는 ingest.failed +``` + +### 1.1 입력 + +인덱싱의 입력은 source configuration과 인증 정보다. + +| 입력 | 예시 | 담당 | +| --- | --- | --- | +| Source config | repository, Drive folder, Notion database, local path | CLI / server / config loader | +| Credentials | API key, OAuth token, service account | auth / connector config | +| Workspace context | `workspace_id`, team mode state | core / server | +| Plugin registry | connector, parser, model provider 목록 | core plugin system | + +시크릿은 코드에 하드코딩하지 않고 환경변수나 안전한 설정 채널로 주입한다. + +### 1.2 Connector 단계 + +Connector는 외부 소스별 차이를 숨기고 공통 raw document 형태로 변환한다. + +담당 패키지: + +- `plugins/connector-github` +- `plugins/connector-notion` +- `plugins/connector-gdrive` +- `plugins/connector-s3` +- `plugins/connector-confluence` +- `plugins/connector-web-crawler` +- `plugins/connector-web-search` +- `plugins/connector-swagger` + +주요 책임: + +- 문서 목록 탐색 +- 원본 콘텐츠 가져오기 +- source path, external id, updated time, MIME type 수집 +- 삭제되었거나 변경된 문서 감지 +- rate limit과 외부 API 실패 처리 + +출력은 parser가 처리할 수 있는 raw content와 metadata다. + +### 1.3 Parser 단계 + +Parser는 파일 형식별 원본 콘텐츠를 텍스트와 구조 정보로 변환한다. + +담당 패키지: + +- `plugins/parser-pdf` +- `plugins/parser-docx` +- `plugins/parser-xlsx` +- `plugins/parser-html` +- `plugins/parser-jupyter` +- `plugins/parser-email` +- `plugins/parser-code` +- `plugins/parser-pptx` + +주요 책임: + +- 텍스트 추출 +- 제목, heading, page, slide, sheet, code block 같은 구조 보존 +- 검색에 불필요한 binary payload 제거 +- 파싱 실패 시 actionable error 반환 + +출력은 normalized document다. + +```text +Raw document + -> Parser + -> title + -> body text + -> structured sections + -> source metadata + -> content hash +``` + +### 1.4 정규화 단계 + +Core ingest pipeline은 connector와 parser 출력물을 내부 document model로 맞춘다. + +정규화 시 유지해야 하는 정보: + +- `workspace_id` +- source type +- source id 또는 external id +- source path +- title +- content hash +- modified time +- parser metadata +- access scope + +이 단계에서 같은 문서의 중복 인덱싱을 막고, 변경 여부를 판단할 수 있어야 한다. + +### 1.5 Chunking 단계 + +Chunking은 긴 문서를 검색과 LLM context에 적합한 단위로 나눈다. + +주요 기준: + +- heading, section, paragraph 같은 구조 경계를 우선한다. +- code block은 가능한 한 의미 단위로 유지한다. +- chunk에는 원본 문서와 위치 정보를 남긴다. +- citation에 필요한 source path와 section 정보를 보존한다. + +출력 예: + +```text +Document + -> Chunk 1: title + intro + -> Chunk 2: heading A body + -> Chunk 3: heading B code block + -> Chunk 4: appendix +``` + +### 1.6 Embedding 단계 + +Embedding 단계는 chunk text를 vector로 변환한다. + +담당 패키지: + +- `plugins/model-ollama` +- `plugins/model-openai` +- `plugins/model-anthropic` +- `plugins/model-google` +- `plugins/model-grok` + +주의사항: + +- 모델 provider가 없어도 개발 모드에서는 stub model fallback으로 동작할 수 있어야 한다. +- embedding 실패는 조용히 삼키지 말고 job failure 또는 partial failure로 기록한다. +- provider별 API key는 환경변수 또는 안전한 config를 통해 전달한다. + +### 1.7 저장 단계 + +OpenDocuments는 SQLite와 LanceDB를 역할별로 나누어 사용한다. + +| 저장소 | 데이터 | 목적 | +| --- | --- | --- | +| SQLite | workspace, document, chunk, job, auth metadata | 구조화 데이터와 상태 관리 | +| SQLite FTS5 | chunk text, title, source path | keyword search | +| LanceDB | embedding vector, chunk payload | semantic search | + +보안 규칙: + +- SQL은 parameterized statement를 사용한다. +- FTS5 검색에는 raw user input을 직접 넣지 않고 `escapeFTS5Query()`를 사용한다. +- LanceDB 필터는 raw string interpolation 대신 `buildWhereClause()`를 사용한다. + +### 1.8 이벤트와 관측성 + +Core 컴포넌트 간 통신은 TypedEventBus를 통해 이루어진다. + +권장 이벤트 흐름: + +```text +ingest.started + -> document.discovered + -> document.parsed + -> chunk.created + -> embedding.created + -> document.indexed + -> ingest.completed +``` + +실패 시에는 최소한 다음 정보를 남긴다. + +- workspace +- source type +- document id 또는 source path +- 실패 단계 +- 사용자에게 줄 수 있는 복구 방법 + +프로덕션 사용자 응답에는 stack trace나 내부 filesystem path를 노출하지 않는다. + +## 2. 질문 응답 RAG 파이프라인 + +RAG 파이프라인은 사용자 질문을 검색 쿼리로 변환하고, 관련 chunk를 찾아 LLM 답변과 출처를 생성한다. + +```mermaid +flowchart TD + Q["User Question"] + AUTH["Auth / Workspace Context"] + NORM["Query Normalization"] + DENSE["Dense Search\nLanceDB"] + SPARSE["Sparse Search\nSQLite FTS5"] + MERGE["Merge / Rank"] + FILTER["Permission + Source Filter"] + EXPAND["Parent Context Expansion"] + FIT["Context Fitting"] + GEN["Answer Generation"] + CITE["Citation Mapping"] + OUT["Answer + Sources"] + + Q --> AUTH --> NORM + NORM --> DENSE + NORM --> SPARSE + DENSE --> MERGE + SPARSE --> MERGE + MERGE --> FILTER --> EXPAND --> FIT --> GEN --> CITE --> OUT +``` + +### 2.1 진입점 + +질문은 여러 인터페이스에서 들어올 수 있다. + +| 진입점 | 담당 패키지 | 역할 | +| --- | --- | --- | +| CLI | `packages/cli` | 명령어 기반 질의 | +| HTTP API | `packages/server` | Hono API endpoint | +| Web UI | `packages/web` | React SPA | +| SDK | `packages/client` | TypeScript client | +| MCP Server | `packages/server` | AI assistant 연동 | + +server는 프로토콜 변환 레이어이며, 검색과 답변 생성의 핵심 로직은 core에 둔다. + +### 2.2 인증과 workspace context + +team mode에서는 HTTP endpoint가 반드시 auth middleware로 보호되어야 한다. + +확인해야 할 정보: + +- API key 또는 session +- workspace 권한 +- 요청자가 접근 가능한 source 범위 +- rate limit 또는 quota + +인증 실패는 내부 정보를 노출하지 않는 명확한 사용자 메시지로 반환한다. + +### 2.3 Query normalization + +질문은 검색에 적합한 형태로 정규화된다. + +처리 예: + +- 불필요한 공백 정리 +- workspace/source filter 결합 +- keyword search용 escape 처리 +- embedding query 생성 +- 필요한 경우 multi-query 또는 HyDE 생성 + +FTS5에는 반드시 `escapeFTS5Query()`를 거친 문자열만 전달한다. + +### 2.4 Retrieval + +Retrieval은 dense search와 sparse search를 함께 사용할 수 있다. + +Dense search: + +- LanceDB에서 embedding vector similarity 검색 +- 의미적으로 유사한 chunk 탐색 +- natural language 질문에 강함 + +Sparse search: + +- SQLite FTS5 기반 keyword search +- 정확한 용어, 파일명, symbol, error text 검색에 강함 + +Hybrid search: + +- dense 결과와 sparse 결과를 병합한다. +- 중복 chunk는 합친다. +- score와 rank를 기준으로 상위 후보를 만든다. + +### 2.5 Filtering과 ranking + +검색 결과는 답변 생성 전에 필터링된다. + +필터 기준: + +- workspace 일치 여부 +- 권한 범위 +- source type +- document freshness +- duplicate chunk +- 낮은 score + +Ranking 단계에서는 결과 순서를 보정한다. 필요한 경우 RRF, reranker, metadata boost를 적용할 수 있다. + +### 2.6 Context assembly + +선택된 chunk는 LLM prompt에 들어갈 context로 조립된다. + +context에 포함해야 할 정보: + +- chunk text +- document title +- source path +- section 또는 page 정보 +- updated time +- citation id + +Context fitting은 모델 context window를 넘지 않도록 chunk를 줄이거나 우선순위를 조정한다. + +### 2.7 Answer generation + +Model plugin이 context와 질문을 받아 답변을 생성한다. + +답변 생성 규칙: + +- 검색된 context에 근거해서 답한다. +- 근거가 부족하면 부족하다고 말한다. +- 출처를 함께 반환한다. +- 내부 stack trace, API key, local path 같은 민감 정보는 포함하지 않는다. + +### 2.8 Citation mapping + +답변에 포함되는 출처는 chunk metadata에서 생성한다. + +출처에 포함할 수 있는 정보: + +- source type +- document title +- source path 또는 URL +- page, slide, section +- chunk id + +Citation은 사용자가 원문을 찾아갈 수 있을 정도로 구체적이어야 한다. + +## 3. 플러그인 실행 파이프라인 + +OpenDocuments의 확장 지점은 connector, parser, model provider다. + +```mermaid +flowchart TB + REG["Plugin Registry"] + CORE["Core Contracts"] + CON["ConnectorPlugin"] + PAR["ParserPlugin"] + MOD["ModelPlugin"] + ING["Ingest Pipeline"] + RAG["RAG Pipeline"] + + REG --> CORE + CORE --> CON + CORE --> PAR + CORE --> MOD + CON --> ING + PAR --> ING + MOD --> ING + MOD --> RAG +``` + +플러그인 추가 시 확인할 항목: + +- `coreVersion: '^0.1.0'` 호환성 유지 +- public API에 JSDoc 작성 +- ESM import에 `.js` 확장자 포함 +- `any` 대신 `unknown` 또는 명시적 타입 사용 +- happy path와 error path 테스트 추가 +- 외부 API 호출은 테스트에서 `vi.stubGlobal('fetch', ...)`로 mock + +## 4. 릴리즈 파이프라인 + +릴리즈는 changesets 기반으로 수행한다. + +```mermaid +flowchart TD + READY["Code Complete"] + BUILD1["Build / Typecheck / Test"] + REVIEW1["Review 1\nFunctionality"] + REVIEW2["Review 2\nSecurity / Performance"] + REVIEW3["Review 3\nStyle / Consistency"] + CHANGESET["npx changeset"] + VERSION["npx changeset version"] + DOCS["CHANGELOG / README / docs-site"] + FINAL["Final Build / Typecheck / Test"] + PR["Commit / Push / PR"] + CI["GitHub Actions\nNode 20 + 22"] + PUBLISH["npx changeset publish"] + RELEASE["Git Tag / GitHub Release"] + VERIFY["npm install / docs-site 확인"] + + READY --> BUILD1 --> REVIEW1 --> REVIEW2 --> REVIEW3 --> CHANGESET + CHANGESET --> VERSION --> DOCS --> FINAL --> PR --> CI --> PUBLISH --> RELEASE --> VERIFY +``` + +### 4.1 코드 준비 + +릴리즈 전에 전체 검증을 실행한다. + +```bash +npm run build +npm run typecheck +npm run test +``` + +### 4.2 3회 코드 리뷰 + +1차 리뷰는 기능과 정확성을 확인한다. + +- 변경된 파일의 로직 확인 +- edge case 확인 +- error handling 확인 +- 새 기능 테스트 확인 + +2차 리뷰는 보안과 성능을 확인한다. + +- SQL injection 가능성 +- raw FTS5 query 사용 여부 +- raw LanceDB filter interpolation 여부 +- API key 또는 secret 하드코딩 여부 +- 불필요한 `any` 사용 여부 +- N+1 query 또는 불필요한 loop + +3차 리뷰는 스타일과 일관성을 확인한다. + +- naming convention +- ESM import의 `.js` 확장자 +- CLI output의 `log` 유틸리티 사용 +- public API JSDoc +- 불필요한 `console.log`, TODO, debugging code 제거 + +### 4.3 Changeset 생성 + +```bash +npx changeset +``` + +선택 항목: + +- 영향 받는 package +- bump type: patch, minor, major +- 변경 요약 + +생성된 `.changeset/*.md` 파일은 커밋에 포함한다. + +### 4.4 버전 범프 + +```bash +npx changeset version +``` + +이 명령은 package version, workspace dependency version, `CHANGELOG.md`를 업데이트한다. + +### 4.5 문서 보완 + +릴리즈 내용에 따라 다음 문서를 확인한다. + +- `README.md` +- `README.ko.md` +- `CHANGELOG.md` +- `docs-site/guide/*` +- `docs-site/plugins/*` +- `docs-site/api/*` + +특히 새 플러그인, CLI 명령어, MCP 도구 수, 테스트 수가 바뀌면 문서에 반영한다. + +### 4.6 최종 검증 + +```bash +npm run build && npm run typecheck && npm run test +``` + +모든 검증이 통과한 뒤 release commit을 만든다. + +```bash +git add -A +git commit -m "chore: release v{VERSION}" +git push origin {branch} +``` + +### 4.7 배포와 릴리즈 + +CI가 통과하고 PR이 main에 merge된 뒤 npm 배포를 실행한다. + +```bash +npx changeset publish +``` + +배포 후 확인: + +```bash +npm info opendocuments +npm info opendocuments-core +npm install -g opendocuments@latest +``` + +마지막으로 태그와 GitHub Release를 만든다. + +```bash +git tag v{VERSION} +git push origin v{VERSION} +``` + +## 5. 장애 처리 체크리스트 + +문제가 발생하면 먼저 어느 단계에서 실패했는지 분리한다. + +| 증상 | 우선 확인 단계 | +| --- | --- | +| 문서가 검색되지 않음 | connector, parser, chunk 저장, FTS5 index | +| 의미 검색 결과가 이상함 | embedding provider, LanceDB payload, vector dimension | +| 키워드 검색이 실패함 | FTS5 query escaping, tokenization, chunk text | +| 답변에 출처가 없음 | citation mapping, chunk metadata | +| 권한 없는 문서가 보임 | auth middleware, workspace filter, source filter | +| 배포 후 package 버전 불일치 | changeset version, workspace dependency, npm publish 결과 | + +장애 로그에는 원인 분석에 필요한 단계 정보는 남기되, 사용자 응답에는 내부 경로나 stack trace를 노출하지 않는다. + +## 6. 구현 변경 시 체크포인트 + +새 기능이나 수정이 파이프라인에 영향을 주는 경우 다음을 확인한다. + +- core 로직에 들어갈 변경인지, plugin에 들어갈 변경인지 분리한다. +- HTTP endpoint를 추가하면 team mode auth middleware를 적용한다. +- SQL은 parameterized statement를 사용한다. +- FTS5 query는 `escapeFTS5Query()`를 사용한다. +- LanceDB filter는 `buildWhereClause()`를 사용한다. +- CLI output에는 이모지 대신 `log.ok`, `log.fail`, `log.info`, `log.arrow`, `log.wait`를 사용한다. +- TypeScript import에는 `.js` 확장자를 포함한다. +- 테스트는 happy path와 error path를 모두 포함한다. +- 외부 API는 Vitest에서 mock 처리한다. diff --git a/docs/superpowers/plans/2026-06-25-platform-hardening-and-web-workbench.md b/docs/superpowers/plans/2026-06-25-platform-hardening-and-web-workbench.md new file mode 100644 index 0000000..77f5888 --- /dev/null +++ b/docs/superpowers/plans/2026-06-25-platform-hardening-and-web-workbench.md @@ -0,0 +1,78 @@ +# Platform Hardening and Web Workbench Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make OpenDocuments behave like a working self-hosted RAG platform across web, CLI, SDK, and server security contracts. + +**Architecture:** Keep the server as the API contract owner, align the SDK and web client to those contracts, and make the web app's first screen a chat workbench with conversation state instead of an admin dashboard. Fix CLI reliability and shell execution risks without changing its local-first architecture. + +**Tech Stack:** TypeScript, Hono, React, Zustand, Vitest, Turborepo, Commander.js. + +--- + +### Task 1: Server Contract and Security Scope + +**Files:** +- Modify: `packages/server/src/http/routes/documents.ts` +- Modify: `packages/server/src/http/routes/conversations.ts` +- Test: `packages/server/tests/http/documents.test.ts` +- Test: `packages/server/tests/http/conversations.test.ts` + +- [ ] Add failing tests that verify read-only team API keys cannot upload, delete, restore, or share conversations. +- [ ] Apply `requireScope('document:write')` to write document routes. +- [ ] Apply `requireScope('ask')` to conversation sharing. +- [ ] Run `npm run test --workspace opendocuments-server`. + +### Task 2: SDK Contract Alignment + +**Files:** +- Modify: `packages/client/src/index.ts` +- Modify: `packages/client/tests/client.test.ts` +- Modify: `docs-site/sdk/guide.md` + +- [ ] Add failing tests for `ask(query, { profile })`, confidence object typing, upload response shape, and conversation APIs. +- [ ] Update SDK types and methods to match server responses. +- [ ] Update docs to use `@opendocuments/client`. +- [ ] Run `npm run test --workspace @opendocuments/client`. + +### Task 3: CLI Reliability and Command Safety + +**Files:** +- Modify: `packages/cli/tests/commands/ask.test.ts` +- Modify: `packages/cli/tests/commands/doctor.test.ts` +- Modify: `packages/cli/src/commands/plugin.ts` +- Modify: `packages/cli/src/commands/start.ts` +- Modify: `packages/cli/src/commands/doctor.ts` + +- [ ] Make CLI tests bootstrap with stub models and clean up safely. +- [ ] Replace shell-interpolated npm commands with `execFileSync`. +- [ ] Handle `SIGTERM` with the same graceful shutdown path as `SIGINT`. +- [ ] Report the package version in `doctor`. +- [ ] Run `npm run test --workspace opendocuments`. + +### Task 4: Web Conversation Workbench + +**Files:** +- Modify: `packages/web/src/lib/types.ts` +- Modify: `packages/web/src/lib/api.ts` +- Modify: `packages/web/src/stores/chatStore.ts` +- Modify: `packages/web/src/components/chat/ChatPage.tsx` +- Modify: `packages/web/src/components/layout/Layout.tsx` +- Modify: `packages/web/src/components/layout/Sidebar.tsx` +- Modify: `packages/web/src/App.tsx` + +- [ ] Add conversation list/message/share API client methods. +- [ ] Add Zustand actions for loading, selecting, creating, deleting, renaming, and sharing sessions. +- [ ] Redesign `ChatPage` as a three-region RAG workbench: sessions, message stream, evidence/health rail. +- [ ] Hide admin-first navigation from the primary path and make the header show product state instead. +- [ ] Prevent streaming failures from leaving the app stuck. +- [ ] Run `npm run build --workspace @opendocuments/web`. + +### Task 5: Deployment and Final Verification + +**Files:** +- Modify: `Dockerfile` +- Modify: `docker-compose.yml` + +- [ ] Point container healthchecks at `/api/v1/health`. +- [ ] Run `npm run build`, `npm run typecheck`, and targeted tests. diff --git a/docs/superpowers/plans/2026-06-25-remove-unsupported-connectors.md b/docs/superpowers/plans/2026-06-25-remove-unsupported-connectors.md new file mode 100644 index 0000000..73beda8 --- /dev/null +++ b/docs/superpowers/plans/2026-06-25-remove-unsupported-connectors.md @@ -0,0 +1,87 @@ +# Remove Unsupported Connectors Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove connector packages that exist in the workspace but are not supported by the product bootstrap, CLI presets, or official docs. + +**Architecture:** Keep the supported connector surface aligned with `packages/server/src/bootstrap.ts`, CLI presets, and documented connectors. Remove unsupported workspace packages completely so they are not built, tested, published, or importable. + +**Tech Stack:** TypeScript, npm workspaces, Turborepo, Vitest. + +--- + +### Task 1: Add Removal Guard Test + +**Files:** +- Create: `packages/server/tests/unsupported-connectors.test.ts` + +- [ ] **Step 1: Write the failing test** + +```typescript +const unsupportedConnectorPackages = [ + '@opendocuments/connector-discord', + '@opendocuments/connector-jira', + '@opendocuments/connector-linear', + '@opendocuments/connector-slack', +] + +describe('unsupported connector packages', () => { + it('are not importable from the workspace', async () => { + for (const packageName of unsupportedConnectorPackages) { + await expect(import(packageName)).rejects.toThrow() + } + }) +}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run packages/server/tests/unsupported-connectors.test.ts` +Expected: FAIL because the packages still exist and import successfully. + +### Task 2: Remove Unsupported Connector Packages + +**Files:** +- Delete directory: `plugins/connector-discord` +- Delete directory: `plugins/connector-jira` +- Delete directory: `plugins/connector-linear` +- Delete directory: `plugins/connector-slack` + +- [ ] **Step 1: Remove the four plugin package directories** + +Run: `rm -rf plugins/connector-discord plugins/connector-jira plugins/connector-linear plugins/connector-slack` + +- [ ] **Step 2: Confirm directories are gone** + +Run: `test ! -d plugins/connector-discord && test ! -d plugins/connector-jira && test ! -d plugins/connector-linear && test ! -d plugins/connector-slack` +Expected: exit code 0. + +### Task 3: Refresh Workspace Metadata + +**Files:** +- Modify: `package-lock.json` + +- [ ] **Step 1: Regenerate package lock without installing package code changes** + +Run: `npm install --package-lock-only` +Expected: package lock no longer contains the four removed workspace packages. + +- [ ] **Step 2: Search for stale references** + +Run: `rg "connector-(discord|jira|linear|slack)|@opendocuments/connector-(discord|jira|linear|slack)" . -g '!node_modules'` +Expected: only historical references in changelogs or no references. If product docs/code references remain, remove them. + +### Task 4: Verify + +**Files:** +- Test: `packages/server/tests/unsupported-connectors.test.ts` + +- [ ] **Step 1: Run the guard test** + +Run: `npx vitest run packages/server/tests/unsupported-connectors.test.ts` +Expected: PASS. + +- [ ] **Step 2: Run typecheck** + +Run: `npm run typecheck` +Expected: PASS. diff --git a/docs/superpowers/plans/2026-06-26-simplified-web-workbench.md b/docs/superpowers/plans/2026-06-26-simplified-web-workbench.md new file mode 100644 index 0000000..19e0c9e --- /dev/null +++ b/docs/superpowers/plans/2026-06-26-simplified-web-workbench.md @@ -0,0 +1,114 @@ +# Simplified Web Workbench Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a simpler, more trustworthy OpenDocuments web workbench and the backend API needed to support it. + +**Architecture:** Add a server-side workbench route that aggregates workspace-scoped health, corpus, connector, quality, and recent query data. Update the React web app to consume that summary and present a focused ask/read/cite experience instead of a dense operational console. + +**Tech Stack:** Hono, TypeScript, SQLite via existing DB wrapper, React 19, Zustand, Tailwind CSS, Vitest. + +--- + +### Task 1: Backend Workbench API + +**Files:** +- Create: `packages/server/src/http/routes/workbench.ts` +- Modify: `packages/server/src/http/app.ts` +- Test: `packages/server/tests/http/workbench.test.ts` + +- [ ] **Step 1: Write failing tests** + +Create tests that call `GET /api/v1/workbench`, assert response shape, and assert team mode auth still protects the endpoint. + +- [ ] **Step 2: Run test to verify failure** + +Run: `npx vitest run packages/server/tests/http/workbench.test.ts` +Expected: fail because the route does not exist. + +- [ ] **Step 3: Implement route** + +Create `workbenchRoutes(ctx)` with parameterized SQL for document, query, and recent query summaries. Register it from `createApp`. + +- [ ] **Step 4: Run targeted tests** + +Run: `npx vitest run packages/server/tests/http/workbench.test.ts` +Expected: pass. + +### Task 2: Frontend API Types + +**Files:** +- Modify: `packages/web/src/lib/types.ts` +- Modify: `packages/web/src/lib/api.ts` + +- [ ] **Step 1: Add types** + +Add `WorkbenchResponse` with `health`, `corpus`, `quality`, `connectors`, `recentQueries`, and `suggestedQuestions`. + +- [ ] **Step 2: Add client function** + +Add `getWorkbench()` that requests `/workbench`. + +- [ ] **Step 3: Run typecheck** + +Run: `npx tsc -p packages/web/tsconfig.json --noEmit` +Expected: pass after UI usage is implemented. + +### Task 3: Simplified Chat Workbench + +**Files:** +- Modify: `packages/web/src/components/chat/ChatPage.tsx` +- Modify: `packages/web/src/components/chat/ChatInput.tsx` +- Modify: `packages/web/src/components/chat/ChatMessage.tsx` +- Modify: `packages/web/src/components/chat/SourceCard.tsx` + +- [ ] **Step 1: Replace dense layout** + +Remove the permanent right evidence rail and use a focused central layout with compact source chips. + +- [ ] **Step 2: Add workbench summary** + +Load `getWorkbench()` on mount. Show corpus count, health, average confidence, and suggested prompts in quiet UI. + +- [ ] **Step 3: Improve answer details** + +Render confidence and citations under assistant answers without overwhelming the page. + +- [ ] **Step 4: Run web build** + +Run: `npm run build --workspace=@opendocuments/web` +Expected: TypeScript and Vite build pass. + +### Task 4: Dashboard Simplification + +**Files:** +- Modify: `packages/web/src/components/dashboard/UnifiedDashboard.tsx` + +- [ ] **Step 1: Reframe dashboard** + +Keep operational details available but reduce visual noise, using clear sections for corpus, quality, connectors, and plugin health. + +- [ ] **Step 2: Run web build** + +Run: `npm run build --workspace=@opendocuments/web` +Expected: pass. + +### Task 5: Verification + +**Files:** +- No new files. + +- [ ] **Step 1: Run targeted server tests** + +Run: `npx vitest run packages/server/tests/http/workbench.test.ts packages/server/tests/http/chat.test.ts` +Expected: pass. + +- [ ] **Step 2: Run web build** + +Run: `npm run build --workspace=@opendocuments/web` +Expected: pass. + +- [ ] **Step 3: Start dev server** + +Run: `npm run dev --workspace=@opendocuments/web -- --host 127.0.0.1` +Expected: Vite starts and prints a local URL. diff --git a/docs/superpowers/specs/2026-06-26-simplified-web-workbench-design.md b/docs/superpowers/specs/2026-06-26-simplified-web-workbench-design.md new file mode 100644 index 0000000..764fddb --- /dev/null +++ b/docs/superpowers/specs/2026-06-26-simplified-web-workbench-design.md @@ -0,0 +1,61 @@ +# Simplified Web Workbench Design + +## Goal + +Make the OpenDocuments web app feel like a calm, credible AI document search product for daily office use, while adding a backend workbench summary API that supports the simplified first screen without several separate requests. + +## Product Direction + +OpenDocuments is a self-hosted RAG app. The web UI should emphasize asking documents, reading cited answers, and understanding whether the corpus is usable. It should not feel like a dense admin dashboard, a marketing landing page, or a technical demo. + +The approved visual direction is minimal: a quiet left navigation, a large central ask/search surface, cited answers, compact source chips, and only essential status. Advanced operational details remain available on dashboard/admin pages but should not dominate the main chat surface. + +## Design Thesis + +This interface should feel like a focused document search desk because the product is for trust-sensitive internal knowledge work, so the UI should emphasize a single question path, visible citations, and low-noise system status. + +## Scope + +- Redesign the main chat screen around a single central work surface. +- Replace the always-visible evidence inspector with compact citations under the answer. +- Add simple, useful first-screen context: corpus count, health, recent prompts, and retrieval profile. +- Add a backend `GET /api/v1/workbench` endpoint that returns web-oriented overview data. +- Keep existing routes and pages working. +- Avoid backend changes to RAG retrieval behavior in this pass. + +## Backend API + +`GET /api/v1/workbench` returns: + +- `health`: server status and version. +- `corpus`: document count, chunk count, source distribution, status distribution. +- `quality`: total query count, average confidence, average response time, feedback counts. +- `connectors`: connector count, active count, and recent connector summaries. +- `recentQueries`: recent query log rows trimmed for the home screen. +- `suggestedQuestions`: deterministic fallback prompts based on whether documents exist. + +The endpoint is protected by the existing `/api/*` auth middleware. It uses the resolved request workspace and parameterized SQL. + +## Frontend + +The chat page becomes the primary product surface: + +- Top area: small corpus/health strip. +- Empty state: large `Ask your documents` composer and concise suggested prompts. +- Answer state: readable message stream with compact citations and confidence. +- Session list: retained but simplified visually. +- Evidence: shown as source chips and expandable snippets rather than a permanent right rail. +- Dashboard: refreshed to a simpler overview using the same workbench summary API where useful. + +## Error Handling + +- Workbench API failures should not block chat. +- Missing admin-only workbench fields should show quiet fallback text instead of noisy errors. +- Offline health should disable sending and show a compact status. + +## Testing + +- Add server tests for `GET /api/v1/workbench`. +- Verify workspace scoping and response shape. +- Add frontend typecheck/build verification. +- Run targeted server tests and web build before completion. diff --git a/package-lock.json b/package-lock.json index a5f9a70..897ed65 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1070,10 +1070,6 @@ "resolved": "plugins/connector-confluence", "link": true }, - "node_modules/@opendocuments/connector-discord": { - "resolved": "plugins/connector-discord", - "link": true - }, "node_modules/@opendocuments/connector-gdrive": { "resolved": "plugins/connector-gdrive", "link": true @@ -1082,14 +1078,6 @@ "resolved": "plugins/connector-github", "link": true }, - "node_modules/@opendocuments/connector-jira": { - "resolved": "plugins/connector-jira", - "link": true - }, - "node_modules/@opendocuments/connector-linear": { - "resolved": "plugins/connector-linear", - "link": true - }, "node_modules/@opendocuments/connector-notion": { "resolved": "plugins/connector-notion", "link": true @@ -1098,10 +1086,6 @@ "resolved": "plugins/connector-s3", "link": true }, - "node_modules/@opendocuments/connector-slack": { - "resolved": "plugins/connector-slack", - "link": true - }, "node_modules/@opendocuments/connector-swagger": { "resolved": "plugins/connector-swagger", "link": true @@ -1143,6 +1127,20 @@ "tslib": "^2.8.0" } }, + "node_modules/@turbo/darwin-64": { + "version": "2.8.21", + "resolved": "https://registry.npmjs.org/@turbo/darwin-64/-/darwin-64-2.8.21.tgz", + "integrity": "sha512-kfGoM0Iw8ZNZpbds+4IzOe0hjvHldqJwUPRAjXJi3KBxg/QOZL95N893SRoMtf2aJ+jJ3dk32yPkp8rvcIjP9g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, "node_modules/@turbo/darwin-arm64": { "version": "2.8.21", "cpu": [ @@ -1155,6 +1153,62 @@ "darwin" ] }, + "node_modules/@turbo/linux-64": { + "version": "2.8.21", + "resolved": "https://registry.npmjs.org/@turbo/linux-64/-/linux-64-2.8.21.tgz", + "integrity": "sha512-uTxlCcXWy5h1fSSymP8XSJ+AudzEHMDV3IDfKX7+DGB8kgJ+SLoTUAH7z4OFA7I/l2sznz0upPdbNNZs91YMag==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@turbo/linux-arm64": { + "version": "2.8.21", + "resolved": "https://registry.npmjs.org/@turbo/linux-arm64/-/linux-arm64-2.8.21.tgz", + "integrity": "sha512-cdHIcxNcihHHkCHp0Y4Zb60K4Qz+CK4xw1gb6s/t/9o4SMeMj+hTBCtoW6QpPnl9xPYmxuTou8Zw6+cylTnREg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@turbo/windows-64": { + "version": "2.8.21", + "resolved": "https://registry.npmjs.org/@turbo/windows-64/-/windows-64-2.8.21.tgz", + "integrity": "sha512-/iBj4OzbqEY8CX+eaeKbBTMZv2CLXNrt0692F7HnK7LcyYwyDecaAiSET6ZzL4opT7sbwkKvzAC/fhqT3Quu1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@turbo/windows-arm64": { + "version": "2.8.21", + "resolved": "https://registry.npmjs.org/@turbo/windows-arm64/-/windows-arm64-2.8.21.tgz", + "integrity": "sha512-95tMA/ZbIidJFUUtkmqioQ1gf3n3I1YbRP3ZgVdWTVn2qVbkodcIdGXBKRHHrIbRsLRl99SiHi/L7IxhpZDagQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@types/babel__core": { "version": "7.20.5", "dev": true, @@ -3561,6 +3615,15 @@ "yallist": "^3.0.2" } }, + "node_modules/lucide-react": { + "version": "1.22.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.22.0.tgz", + "integrity": "sha512-c9o3l0PiNcgOQDW4F31BEYHudE7kgxVt3o30qMl36ZPwTxXlGB4QnLilhERvVM4uh/pl5MDyY1/gzZSYcHDtBg==", + "license": "ISC", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/magic-string": { "version": "0.30.21", "dev": true, @@ -4369,10 +4432,6 @@ "resolved": "plugins/model-anthropic", "link": true }, - "node_modules/opendocuments-model-deepseek": { - "resolved": "plugins/model-deepseek", - "link": true - }, "node_modules/opendocuments-model-google": { "resolved": "plugins/model-google", "link": true @@ -4381,10 +4440,6 @@ "resolved": "plugins/model-grok", "link": true }, - "node_modules/opendocuments-model-mistral": { - "resolved": "plugins/model-mistral", - "link": true - }, "node_modules/opendocuments-model-ollama": { "resolved": "plugins/model-ollama", "link": true @@ -4393,10 +4448,6 @@ "resolved": "plugins/model-openai", "link": true }, - "node_modules/opendocuments-model-openai-compatible": { - "resolved": "plugins/model-openai-compatible", - "link": true - }, "node_modules/opendocuments-parser-code": { "resolved": "plugins/parser-code", "link": true @@ -6532,6 +6583,7 @@ "version": "0.2.0", "license": "MIT", "dependencies": { + "lucide-react": "^1.22.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-markdown": "^9.0.0", @@ -6716,27 +6768,6 @@ "vitest": "^2.1.0" } }, - "plugins/connector-discord": { - "name": "@opendocuments/connector-discord", - "version": "0.1.1", - "license": "MIT", - "dependencies": { - "opendocuments-core": "0.3.0" - }, - "devDependencies": { - "typescript": "^5.5.0", - "vitest": "^2.1.0" - } - }, - "plugins/connector-discord/node_modules/jiti": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", - "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", - "license": "MIT", - "bin": { - "jiti": "lib/jiti-cli.mjs" - } - }, "plugins/connector-gdrive": { "name": "@opendocuments/connector-gdrive", "version": "0.1.1", @@ -6761,30 +6792,6 @@ "vitest": "^2.1.0" } }, - "plugins/connector-jira": { - "name": "@opendocuments/connector-jira", - "version": "0.1.1", - "license": "MIT", - "dependencies": { - "opendocuments-core": "0.3.0" - }, - "devDependencies": { - "typescript": "^5.5.0", - "vitest": "^2.1.0" - } - }, - "plugins/connector-linear": { - "name": "@opendocuments/connector-linear", - "version": "0.1.1", - "license": "MIT", - "dependencies": { - "opendocuments-core": "0.3.0" - }, - "devDependencies": { - "typescript": "^5.5.0", - "vitest": "^2.1.0" - } - }, "plugins/connector-notion": { "name": "@opendocuments/connector-notion", "version": "0.1.1", @@ -6809,21 +6816,6 @@ "vitest": "^2.1.0" } }, - "plugins/connector-slack": { - "name": "@opendocuments/connector-slack", - "version": "0.1.1", - "license": "MIT", - "dependencies": { - "opendocuments-core": "0.3.0" - }, - "devDependencies": { - "typescript": "^5.5.0", - "vitest": "^2.1.0" - }, - "peerDependencies": { - "opendocuments-core": "^0.3.0" - } - }, "plugins/connector-swagger": { "name": "@opendocuments/connector-swagger", "version": "0.1.1", @@ -6879,20 +6871,6 @@ "opendocuments-core": "^0.3.0" } }, - "plugins/model-deepseek": { - "version": "0.1.0", - "license": "MIT", - "dependencies": { - "opendocuments-core": "0.3.0" - }, - "devDependencies": { - "typescript": "^5.5.0", - "vitest": "^2.1.0" - }, - "peerDependencies": { - "opendocuments-core": "^0.3.0" - } - }, "plugins/model-google": { "name": "opendocuments-model-google", "version": "0.1.1", @@ -6923,20 +6901,6 @@ "opendocuments-core": "^0.3.0" } }, - "plugins/model-mistral": { - "version": "0.1.0", - "license": "MIT", - "dependencies": { - "opendocuments-core": "0.3.0" - }, - "devDependencies": { - "typescript": "^5.5.0", - "vitest": "^2.1.0" - }, - "peerDependencies": { - "opendocuments-core": "^0.3.0" - } - }, "plugins/model-ollama": { "name": "opendocuments-model-ollama", "version": "0.1.3", @@ -6967,20 +6931,6 @@ "opendocuments-core": "^0.3.0" } }, - "plugins/model-openai-compatible": { - "version": "0.1.0", - "license": "MIT", - "dependencies": { - "opendocuments-core": "0.3.0" - }, - "devDependencies": { - "typescript": "^5.5.0", - "vitest": "^2.1.0" - }, - "peerDependencies": { - "opendocuments-core": "^0.3.0" - } - }, "plugins/parser-code": { "name": "opendocuments-parser-code", "version": "0.1.1", diff --git a/package.json b/package.json index 7536407..96cefe3 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "setup": "npm install && npm run build && echo 'Setup complete. Run: npm run test'", "build": "turbo build", "test": "turbo test", - "lint": "turbo lint", + "lint": "turbo typecheck", "typecheck": "turbo typecheck", "dev": "turbo dev", "clean": "turbo clean" diff --git a/packages/cli/src/commands/doctor.ts b/packages/cli/src/commands/doctor.ts index d2ce7b7..f5baf64 100644 --- a/packages/cli/src/commands/doctor.ts +++ b/packages/cli/src/commands/doctor.ts @@ -1,93 +1,7 @@ import { Command } from 'commander' -import { log } from 'opendocuments-core' +import { VERSION, log } from 'opendocuments-core' import { getContext, shutdownContext } from '../utils/bootstrap.js' -/** - * Provider-specific diagnostic endpoints. Each entry is a GET that returns 200 - * when the API key is valid and reachable. - */ -const PROVIDER_DIAGNOSTICS: Record string) - headers: (key: string) => Record - envVar: string - docs: string -}> = { - openai: { - url: 'https://api.openai.com/v1/models', - headers: (k) => ({ Authorization: `Bearer ${k}` }), - envVar: 'OPENAI_API_KEY', - docs: 'https://platform.openai.com/api-keys', - }, - anthropic: { - url: 'https://api.anthropic.com/v1/models', - headers: (k) => ({ 'x-api-key': k, 'anthropic-version': '2023-06-01' }), - envVar: 'ANTHROPIC_API_KEY', - docs: 'https://console.anthropic.com/settings/keys', - }, - google: { - url: (cfg) => `https://generativelanguage.googleapis.com/v1beta/models?key=${cfg.apiKey ?? ''}`, - headers: () => ({}), - envVar: 'GOOGLE_API_KEY', - docs: 'https://aistudio.google.com/apikey', - }, - grok: { - url: 'https://api.x.ai/v1/models', - headers: (k) => ({ Authorization: `Bearer ${k}` }), - envVar: 'XAI_API_KEY', - docs: 'https://console.x.ai', - }, - deepseek: { - url: 'https://api.deepseek.com/v1/models', - headers: (k) => ({ Authorization: `Bearer ${k}` }), - envVar: 'DEEPSEEK_API_KEY', - docs: 'https://platform.deepseek.com/api_keys', - }, - mistral: { - url: 'https://api.mistral.ai/v1/models', - headers: (k) => ({ Authorization: `Bearer ${k}` }), - envVar: 'MISTRAL_API_KEY', - docs: 'https://console.mistral.ai/api-keys', - }, - 'openai-compatible': { - url: (cfg) => `${cfg.baseUrl?.replace(/\/+$/, '')}/models`, - headers: (k) => { - const h: Record = {} - if (k) h.Authorization = `Bearer ${k}` - return h - }, - envVar: 'OPENAI_COMPATIBLE_API_KEY', - docs: '(configured endpoint)', - }, -} - -async function pingProvider( - provider: string, - cfg: { baseUrl?: string; apiKey?: string }, -): Promise<{ ok: boolean; status?: number; message: string }> { - const diag = PROVIDER_DIAGNOSTICS[provider] - if (!diag) return { ok: false, message: `no diagnostic for provider '${provider}'` } - const apiKey = cfg.apiKey ?? process.env[diag.envVar] ?? '' - if (!apiKey && provider !== 'openai-compatible') { - return { ok: false, message: `${diag.envVar} not set (get a key: ${diag.docs})` } - } - const url = typeof diag.url === 'function' ? diag.url({ ...cfg, apiKey }) : diag.url - if (!url || url.startsWith('undefined')) { - return { ok: false, message: 'baseUrl is not set (openai-compatible requires it)' } - } - try { - const res = await fetch(url, { headers: diag.headers(apiKey), signal: AbortSignal.timeout(10000) }) - if (res.status === 401 || res.status === 403) { - return { ok: false, status: res.status, message: `invalid API key (HTTP ${res.status})` } - } - if (!res.ok) { - return { ok: false, status: res.status, message: `HTTP ${res.status}` } - } - return { ok: true, status: res.status, message: 'API reachable, key accepted' } - } catch (err) { - return { ok: false, message: (err as Error).message } - } -} - export function doctorCommand() { return new Command('doctor') .description('Run health diagnostics') @@ -97,7 +11,7 @@ export function doctorCommand() { try { const ctx = await getContext() - log.ok('Core v0.1.0') + log.ok(`Core v${VERSION}`) // Test SQLite try { @@ -154,41 +68,8 @@ export function doctorCommand() { } } - // Provider-specific diagnostics (non-Ollama cloud/self-hosted) - const provider = ctx.config.model.provider - if (provider !== 'ollama' && PROVIDER_DIAGNOSTICS[provider]) { - log.blank() - log.heading(`${provider} Diagnostics`) - const result = await pingProvider(provider, { - apiKey: ctx.config.model.apiKey, - baseUrl: ctx.config.model.baseUrl, - }) - if (result.ok) { - log.ok(`${provider.padEnd(15)} ${result.message}`) - } else { - log.fail(`${provider.padEnd(15)} ${result.message}`) - hasIssues = true - } - - // Also check secondary embedding provider if different - const embProvider = ctx.config.model.embeddingProvider - if (embProvider && embProvider !== provider && PROVIDER_DIAGNOSTICS[embProvider]) { - const embResult = await pingProvider(embProvider, { - apiKey: ctx.config.model.embeddingApiKey, - baseUrl: ctx.config.model.baseUrl, - }) - if (embResult.ok) { - log.ok(`${embProvider.padEnd(15)} ${embResult.message} (embedding)`) - } else { - log.fail(`${embProvider.padEnd(15)} ${embResult.message} (embedding)`) - hasIssues = true - } - } - } - - // Ollama-specific diagnostics (primary provider OR secondary embedder) - const usesOllama = provider === 'ollama' || ctx.config.model.embeddingProvider === 'ollama' - if (usesOllama) { + // Ollama-specific diagnostics + if (ctx.config.model.provider === 'ollama') { log.blank() log.heading('Ollama Diagnostics') @@ -221,15 +102,9 @@ export function doctorCommand() { } if (ollamaReachable) { - // Only check models that Ollama is actually responsible for. - const ollamaModels: string[] = [] - if (ctx.config.model.provider === 'ollama') { - ollamaModels.push(ctx.config.model.llm) - } - if (ctx.config.model.provider === 'ollama' || ctx.config.model.embeddingProvider === 'ollama') { - ollamaModels.push(ctx.config.model.embedding) - } - const requiredModels = Array.from(new Set(ollamaModels)) + const requiredModels = Array.from( + new Set([ctx.config.model.llm, ctx.config.model.embedding]) + ) for (const required of requiredModels) { // Ollama tags may include ":latest" suffix; match on base name or exact name diff --git a/packages/cli/src/commands/index-cmd.ts b/packages/cli/src/commands/index-cmd.ts index 02ccd6f..3681164 100644 --- a/packages/cli/src/commands/index-cmd.ts +++ b/packages/cli/src/commands/index-cmd.ts @@ -1,9 +1,37 @@ import { Command } from 'commander' -import { log, discoverFiles, FileWatcher } from 'opendocuments-core' +import { log, discoverFiles, FileWatcher, type FileChange } from 'opendocuments-core' import { getContext, shutdownContext } from '../utils/bootstrap.js' import { readFileSync } from 'node:fs' import { extname, basename, resolve } from 'node:path' +type FileWatcherCtor = new ( + dir: string, + extensions: Set, + onChange: (changes: FileChange[]) => void | Promise, + options?: number | { pollIntervalMs?: number; ignoreInitial?: boolean; stableMs?: number } +) => FileWatcher + +interface ForceIngestPipeline { + ingest( + input: { + title: string + content: string | Buffer + sourceType: string + sourcePath: string + fileType?: string + }, + options?: { force?: boolean } + ): Promise<{ documentId: string; chunks: number; status: 'indexed' | 'skipped' | 'error' }> +} + +const WATCH_EXTENSIONS = new Set([ + '.md', '.mdx', '.txt', + '.json', '.yaml', '.yml', '.toml', + '.zip', '.pdf', '.docx', '.pptx', + '.xlsx', '.xls', '.csv', + '.html', '.htm', '.ipynb', '.eml', +]) + export function indexCommand() { return new Command('index') .description('Index a file or directory') @@ -14,6 +42,8 @@ export function indexCommand() { const ctx = await getContext() const absPath = resolve(inputPath) const textExtensions = new Set(['.md', '.mdx', '.txt', '.json', '.yaml', '.yml', '.toml', '.csv', '.html', '.htm', '.ipynb']) + const readContent = (file: string): string | Buffer => + textExtensions.has(extname(file)) ? readFileSync(file, 'utf-8') : readFileSync(file) try { log.heading('Indexing') let files: string[] @@ -32,21 +62,14 @@ export function indexCommand() { } if (files.length === 0) { log.fail('No supported files found'); return } log.info(`Found ${files.length} file(s)`) + const pipeline = ctx.pipeline as unknown as ForceIngestPipeline for (const file of files) { - if (opts.reindex) { - // Delete existing document to force reindex (bypass content hash check) - const docs = ctx.store.listDocuments() - const existing = docs.find(d => d.source_path === file) - if (existing) { - await ctx.store.hardDeleteDocument(existing.id) - } - } const ext = extname(file) - const content = textExtensions.has(ext) ? readFileSync(file, 'utf-8') : readFileSync(file) - const result = await ctx.pipeline.ingest({ + const content = readContent(file) + const result = await pipeline.ingest({ title: basename(file), content, sourceType: 'local', sourcePath: file, fileType: extname(file), - }) + }, { force: Boolean(opts.reindex) }) if (result.status === 'indexed') log.ok(`${basename(file)} (${result.chunks} chunks)`) else if (result.status === 'skipped') log.info(`${basename(file)} (unchanged)`) else log.fail(`${basename(file)} (error)`) @@ -55,23 +78,33 @@ export function indexCommand() { // After the normal indexing loop, if --watch: if (opts.watch) { log.info('Watching for changes... (Ctrl+C to stop)') - const watcher = new FileWatcher( + const Watcher = FileWatcher as FileWatcherCtor + const watcher = new Watcher( absPath, - new Set(['.md', '.mdx', '.txt', '.pdf', '.docx', '.xlsx', '.html', '.ipynb', '.eml']), + WATCH_EXTENSIONS, async (changes) => { for (const change of changes) { if (change.type === 'deleted') { - log.info(`Deleted: ${change.path}`) + const existing = ctx.store.getDocumentBySourcePath(change.path) + if (existing) { + await ctx.store.hardDeleteDocument(existing.id) + log.ok(`deleted: ${basename(change.path)}`) + } else { + log.info(`deleted: ${basename(change.path)} (not indexed)`) + } } else { - const content = readFileSync(change.path, textExtensions.has(extname(change.path)) ? 'utf-8' : undefined as any) + const content = readContent(change.path) const result = await ctx.pipeline.ingest({ title: basename(change.path), content, sourceType: 'local', sourcePath: change.path, fileType: extname(change.path), }) - log.ok(`${change.type}: ${basename(change.path)} (${result.chunks} chunks)`) + if (result.status === 'indexed') log.ok(`${change.type}: ${basename(change.path)} (${result.chunks} chunks)`) + else if (result.status === 'skipped') log.info(`${change.type}: ${basename(change.path)} (unchanged)`) + else log.fail(`${change.type}: ${basename(change.path)} (error)`) } } - } + }, + { ignoreInitial: true } ) watcher.start() // Keep process alive diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index c20032a..1f639e0 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -4,54 +4,34 @@ import chalk from 'chalk' import { writeFileSync, existsSync, mkdirSync } from 'node:fs' import { join, resolve } from 'node:path' import { cpus, totalmem, platform, arch } from 'node:os' -import { - estimateModelSize, - formatBytes, - getAvailableDiskBytes, - getOllamaInstallCommand, - isOllamaRunning, - listOllamaModels, - pullOllamaModel as pullOllamaModelWithProgress, -} from '../utils/ollama.js' async function checkOllamaRunning(): Promise { - return isOllamaRunning() + try { + const res = await fetch('http://localhost:11434/api/tags', { signal: AbortSignal.timeout(3000) }) + return res.ok + } catch { + return false + } } -async function getOllamaModelNames(): Promise { - return (await listOllamaModels()).map((m) => m.name) +async function getOllamaModels(): Promise { + try { + const res = await fetch('http://localhost:11434/api/tags', { signal: AbortSignal.timeout(3000) }) + if (!res.ok) return [] + const data = await res.json() as { models?: Array<{ name: string }> } + return (data.models || []).map(m => m.name) + } catch { + return [] + } } async function pullOllamaModel(model: string): Promise { - let lastLine = '' - const ok = await pullOllamaModelWithProgress(model, 'http://localhost:11434', (line) => { - if (line !== lastLine) { - lastLine = line - process.stdout.write(`\r\x1b[K ${line}`) - } - }) - process.stdout.write('\n') - return ok -} - -async function tryInstallOllama(): Promise { - const install = getOllamaInstallCommand() - if (!install.supported) { - log.info(`Automatic install is not supported on this platform. Download from: ${install.url}`) - return false - } - const { confirm } = await import('@inquirer/prompts') - log.blank() - log.info('Ollama is not installed. The official install script is:') - log.dim(` ${install.command}`) - const go = await confirm({ message: 'Run the official Ollama install script now?', default: false }) - if (!go) return false const { execSync } = await import('node:child_process') try { - execSync(install.command!, { stdio: 'inherit' }) + console.log(` Pulling ${model}... (this may take a few minutes)`) + execSync(`ollama pull ${model}`, { stdio: 'inherit', timeout: 600000 }) return true - } catch (err) { - log.fail(`Install failed: ${(err as Error).message}`) + } catch { return false } } @@ -71,18 +51,6 @@ async function validateCloudApiKey(provider: string, apiKey: string): Promise = { + const providerDefaults: Record = { openai: { llm: 'gpt-4o', embedding: 'text-embedding-3-small' }, - anthropic: { llm: 'claude-sonnet-4-20250514', embedding: 'bge-m3', needsSecondaryEmbedding: true }, - google: { llm: 'gemini-2.5-flash', embedding: 'text-embedding-004' }, - grok: { llm: 'grok-4', embedding: 'grok-2-embed' }, - deepseek: { llm: 'deepseek-chat', embedding: 'bge-m3', needsSecondaryEmbedding: true }, - mistral: { llm: 'mistral-small-latest', embedding: 'mistral-embed' }, + anthropic: { llm: 'claude-sonnet-4-20250514', embedding: 'bge-m3' }, // Anthropic has no embedding + google: { llm: 'gemini-2.5-flash', embedding: 'gemini-embedding-001' }, + grok: { llm: 'grok-3', embedding: 'grok-2-embed' }, } llmModel = providerDefaults[provider].llm embeddingModel = providerDefaults[provider].embedding - if (providerDefaults[provider].needsSecondaryEmbedding) { - log.wait(`${provider} does not provide an embedding API — choose a secondary embedding provider.`) + if (provider === 'anthropic') { + log.wait('Anthropic does not provide an embedding API.') const embeddingChoice = await select({ message: 'Embedding provider:', choices: [ @@ -246,31 +187,25 @@ export function initCommand() { log.info(`Embedding will use ${embeddingChoice === 'ollama' ? 'Ollama BGE-M3' : 'OpenAI text-embedding-3-small'}`) } } else { - // Local model recommendation (April 2026) + // Local model recommendation log.blank() log.info('Recommended models for your system:') if (specs.ramGB >= 32) { - log.arrow(`${chalk.cyan('*')} Gemma 3 27B ${chalk.dim('(recommended -- 128K ctx, multilingual, multimodal)')}`) - log.dim(' Qwen 3.5 27B (flagship Korean)') - log.dim(' Llama 4 Scout (10M context, MoE)') - log.dim(' DeepSeek R1 14B (reasoning, distilled)') + log.arrow(`${chalk.cyan('*')} Qwen 2.5 14B ${chalk.dim('(recommended -- Vision, Korean support)')}`) + log.dim(' Llama 3.3 8B (lightweight)') + log.dim(' EXAONE 7.8B (Korean specialized)') } else if (specs.ramGB >= 16) { - log.arrow(`${chalk.cyan('*')} Gemma 3 12B ${chalk.dim('(recommended for 16GB RAM)')}`) - log.dim(' Qwen 3.5 9B (Korean excellent)') + log.arrow(`${chalk.cyan('*')} Qwen 2.5 7B ${chalk.dim('(recommended for 16GB RAM)')}`) log.dim(' Gemma 3 4B (lightweight)') } else { log.arrow(`${chalk.cyan('*')} Gemma 3 4B ${chalk.dim('(recommended for limited RAM)')}`) - log.dim(' Gemma 3n (selective activation, laptop/phone)') - log.info('Limited RAM detected. A cloud provider (DeepSeek is cheapest) may give better results.') + log.info('Limited RAM detected. Consider a cloud provider for better performance.') } llmModel = await input({ message: 'LLM model:', - default: - specs.ramGB >= 32 ? 'gemma3:27b' : - specs.ramGB >= 16 ? 'gemma3:12b' : - 'gemma3:4b', + default: specs.ramGB >= 32 ? 'qwen2.5:14b' : specs.ramGB >= 16 ? 'qwen2.5:7b' : 'gemma3:4b', }) embeddingModel = await input({ @@ -312,7 +247,6 @@ export function initCommand() { mode: mode as 'personal' | 'team', provider, apiKey, - baseUrl, embeddingProvider, embeddingApiKey, llmModel, @@ -373,64 +307,28 @@ export function initCommand() { log.dim(` Preset ${preset}`) log.blank() log.heading('Next Steps') - // Ollama flow triggers both when backend=='ollama' AND when cloud provider - // needs a secondary Ollama embedder. - const needsOllama = backend === 'ollama' || embeddingProvider === 'ollama' - if (needsOllama) { + if (backend === 'ollama') { log.blank() log.wait('Checking Ollama availability...') - let ollamaRunning = await checkOllamaRunning() - if (!ollamaRunning) { - // Try to auto-install - const installed = await tryInstallOllama() - if (installed) { - log.ok('Ollama installed') - log.wait('Waiting for Ollama daemon to come up...') - for (let i = 0; i < 10; i++) { - await new Promise((r) => setTimeout(r, 1500)) - if (await checkOllamaRunning()) { ollamaRunning = true; break } - } - } - } - + const ollamaRunning = await checkOllamaRunning() if (ollamaRunning) { log.ok('Ollama is running') - const models = await getOllamaModelNames() + const models = await getOllamaModels() const needsPull: string[] = [] - const modelsToCheck = backend === 'ollama' ? [llmModel, embeddingModel] : [embeddingModel] - for (const model of modelsToCheck) { - if (models.some(m => m === model || m === `${model}:latest` || m.startsWith(`${model}:`))) { + for (const model of [llmModel, embeddingModel]) { + if (models.some(m => m.startsWith(model.split(':')[0]))) { log.ok(`Model ${model} is available`) } else { needsPull.push(model) } } if (needsPull.length > 0) { - // Disk space pre-check - const available = getAvailableDiskBytes() - let totalEstimate = 0 - const sizeLines: string[] = [] - for (const m of needsPull) { - const est = estimateModelSize(m) - if (est) { totalEstimate += est; sizeLines.push(` ${m.padEnd(24)} ~${formatBytes(est)}`) } - } - if (sizeLines.length > 0) { - log.info('Estimated disk footprint:') - for (const l of sizeLines) console.log(l) - if (available !== null) { - log.info(`Available disk: ${formatBytes(available)}`) - if (available < totalEstimate + 1.5e9) { - log.fail(`Warning: disk may be too low (need ~${formatBytes(totalEstimate + 1.5e9)}).`) - } - } - } const autoPull = await confirm({ message: `Pull missing models (${needsPull.join(', ')})?`, default: true, }) if (autoPull) { for (const model of needsPull) { - log.wait(`Pulling ${model}...`) const success = await pullOllamaModel(model) if (success) { log.ok(`${model} pulled successfully`) @@ -443,18 +341,13 @@ export function initCommand() { } } } else { - log.fail('Ollama is still not reachable') - const install = getOllamaInstallCommand() - if (install.supported) { - log.arrow(`Install: ${install.command}`) - } else { - log.arrow(`Download: ${install.url}`) - } - log.arrow('Then start: ollama serve') - log.arrow(`Then pull: ollama pull ${llmModel} && ollama pull ${embeddingModel}`) + log.fail('Ollama is not running or not installed') + log.arrow('Install Ollama: https://ollama.com') + log.arrow('Then start it: ollama serve') + log.arrow(`Then pull models: ollama pull ${llmModel} && ollama pull ${embeddingModel}`) } } - if ((backend === 'cloud' || backend === 'openai-compatible') && !apiKey) { + if (backend === 'cloud' && !apiKey) { const envVarName = getEnvVarName(provider) log.arrow(`Set API key: export ${envVarName}=your-key-here`) } @@ -494,7 +387,6 @@ interface ConfigOptions { mode: 'personal' | 'team' provider: string apiKey: string - baseUrl?: string embeddingProvider?: string embeddingApiKey?: string llmModel: string @@ -521,11 +413,7 @@ function generateConfigFile(opts: ConfigOptions): string { lines.push(` embeddingProvider: '${opts.embeddingProvider}',`) } - if (opts.baseUrl) { - lines.push(` baseUrl: '${opts.baseUrl}',`) - } - - if (opts.apiKey || opts.provider === 'openai-compatible') { + if (opts.apiKey) { lines.push(` apiKey: process.env.${getEnvVarName(opts.provider)},`) } @@ -573,9 +461,6 @@ function getEnvVarName(provider: string): string { anthropic: 'ANTHROPIC_API_KEY', google: 'GOOGLE_API_KEY', grok: 'XAI_API_KEY', - deepseek: 'DEEPSEEK_API_KEY', - mistral: 'MISTRAL_API_KEY', - 'openai-compatible': 'OPENAI_COMPATIBLE_API_KEY', ollama: 'OLLAMA_URL', } return map[provider] || 'API_KEY' diff --git a/packages/cli/src/commands/model.ts b/packages/cli/src/commands/model.ts deleted file mode 100644 index 6a3f710..0000000 --- a/packages/cli/src/commands/model.ts +++ /dev/null @@ -1,509 +0,0 @@ -import { Command } from 'commander' -import { log, loadConfig } from 'opendocuments-core' -import chalk from 'chalk' -import { existsSync, readFileSync, writeFileSync } from 'node:fs' -import { join } from 'node:path' -import { - deleteOllamaModel, - estimateModelSize, - formatBytes, - getAvailableDiskBytes, - getOllamaInstallCommand, - isOllamaRunning, - listOllamaModels, - pullOllamaModel, -} from '../utils/ollama.js' - -/* ------------------------------------------------------------------ */ -/* Curated suggestion catalog (April 2026) */ -/* ------------------------------------------------------------------ */ - -interface Suggestion { - tag: string - description: string - size: string - provider: 'ollama' | 'cloud' -} - -const LOCAL_SUGGESTIONS: Suggestion[] = [ - // Google Gemma family - { tag: 'gemma3:27b', description: 'Google Gemma 3 27B — 128K ctx, multimodal, 140+ languages', size: '~17GB', provider: 'ollama' }, - { tag: 'gemma3:12b', description: 'Gemma 3 12B — balanced, 128K ctx', size: '~7.5GB', provider: 'ollama' }, - { tag: 'gemma3:4b', description: 'Gemma 3 4B — low-spec friendly', size: '~2.5GB', provider: 'ollama' }, - { tag: 'gemma3n', description: 'Gemma 3n — selective activation for laptops/phones', size: '~2.8GB', provider: 'ollama' }, - // Qwen - { tag: 'qwen2.5:14b', description: 'Qwen 2.5 14B — vision + Korean (legacy default)', size: '~9GB', provider: 'ollama' }, - { tag: 'qwen3.5:9b', description: 'Qwen 3.5 9B — general, Korean excellent', size: '~5.5GB', provider: 'ollama' }, - { tag: 'qwen3.5:27b', description: 'Qwen 3.5 27B — flagship local model', size: '~17GB', provider: 'ollama' }, - // Meta Llama - { tag: 'llama4:scout', description: 'Llama 4 Scout — 10M context, MoE', size: '~65GB', provider: 'ollama' }, - { tag: 'llama4:maverick', description: 'Llama 4 Maverick — top open-source quality', size: '~230GB', provider: 'ollama' }, - // DeepSeek (distilled for local) - { tag: 'deepseek-r1:14b', description: 'DeepSeek R1 distilled 14B — reasoning', size: '~9GB', provider: 'ollama' }, - // Embeddings - { tag: 'bge-m3', description: 'BGE-M3 — best open embedding, 1024-dim', size: '~1.2GB', provider: 'ollama' }, - { tag: 'nomic-embed-text', description: 'nomic-embed-text — lightweight 768-dim', size: '~300MB', provider: 'ollama' }, -] - -const CLOUD_SUGGESTIONS: Suggestion[] = [ - { tag: 'openai:gpt-4o', description: 'OpenAI GPT-4o — general purpose, vision', size: 'API', provider: 'cloud' }, - { tag: 'anthropic:claude-sonnet-4-20250514', description: 'Claude Sonnet 4 — long context, coding', size: 'API', provider: 'cloud' }, - { tag: 'google:gemini-2.5-flash', description: 'Gemini 2.5 Flash — multimodal', size: 'API', provider: 'cloud' }, - { tag: 'deepseek:deepseek-chat', description: 'DeepSeek V3.2 — $0.26/Mtok, 164K ctx', size: 'API', provider: 'cloud' }, - { tag: 'deepseek:deepseek-reasoner', description: 'DeepSeek R1 — reasoning specialist', size: 'API', provider: 'cloud' }, - { tag: 'mistral:mistral-small-latest', description: 'Mistral Small 4 — MoE w/ reasoning + vision + code', size: 'API', provider: 'cloud' }, - { tag: 'mistral:mistral-large-latest', description: 'Mistral Large 2.1 — flagship reasoning', size: 'API', provider: 'cloud' }, - { tag: 'grok:grok-4', description: 'xAI Grok 4 — real-time knowledge', size: 'API', provider: 'cloud' }, -] - -/* ------------------------------------------------------------------ */ -/* Command */ -/* ------------------------------------------------------------------ */ - -export function modelCommand() { - const cmd = new Command('model').description('Manage LLM / embedding models') - - // ---------- list ---------- - cmd.command('list') - .alias('ls') - .description('Show configured + locally installed models and suggestions') - .option('--suggestions', 'Also print curated model suggestions') - .action(async (opts) => { - const config = loadConfig(process.cwd()) - const baseUrl = config.model.baseUrl || 'http://localhost:11434' - - log.heading('Current configuration') - log.dim(` provider ${config.model.provider}`) - log.dim(` llm ${config.model.llm}`) - log.dim(` embedding ${config.model.embedding}`) - if (config.model.embeddingProvider) { - log.dim(` embeddingProvider ${config.model.embeddingProvider}`) - } - if (config.model.baseUrl) { - log.dim(` baseUrl ${config.model.baseUrl}`) - } - log.blank() - - if (config.model.provider === 'ollama' || config.model.embeddingProvider === 'ollama') { - log.heading(`Installed Ollama models @ ${baseUrl}`) - const reachable = await isOllamaRunning(baseUrl) - if (!reachable) { - log.fail(`Ollama not reachable at ${baseUrl}`) - log.arrow('Start Ollama with: ollama serve') - } else { - const models = await listOllamaModels(baseUrl) - if (models.length === 0) { - log.info('No local models installed.') - log.arrow(`Pull one: opendocuments model pull ${config.model.llm}`) - } else { - for (const m of models) { - const params = m.details?.parameter_size ?? '' - const quant = m.details?.quantization_level ?? '' - const tag = params || quant ? chalk.dim(` (${[params, quant].filter(Boolean).join(', ')})`) : '' - console.log(` ${chalk.cyan(m.name.padEnd(32))} ${formatBytes(m.size).padStart(8)}${tag}`) - } - } - } - log.blank() - } - - if (opts.suggestions) { - log.heading('Suggested local models (ollama pull)') - for (const s of LOCAL_SUGGESTIONS) { - console.log(` ${chalk.cyan(s.tag.padEnd(32))} ${s.size.padStart(8)} ${chalk.dim(s.description)}`) - } - log.blank() - log.heading('Suggested cloud providers') - for (const s of CLOUD_SUGGESTIONS) { - console.log(` ${chalk.cyan(s.tag.padEnd(48))} ${chalk.dim(s.description)}`) - } - log.blank() - log.arrow('See all: https://ollama.com/library') - } else { - log.arrow('Run with --suggestions to see popular models you can install') - } - }) - - // ---------- pull (one or many) ---------- - cmd.command('pull ') - .description('Pull one or more Ollama models (checks disk space, shows progress)') - .option('--base-url ', 'Ollama base URL', 'http://localhost:11434') - .option('--yes', 'Skip disk space confirmation') - .action(async (names: string[], opts: { baseUrl: string; yes?: boolean }) => { - const reachable = await isOllamaRunning(opts.baseUrl) - if (!reachable) { - log.fail(`Ollama not reachable at ${opts.baseUrl}`) - const install = getOllamaInstallCommand() - if (install.supported) { - log.arrow(`Install: opendocuments model install-ollama`) - log.arrow(` or: ${install.command}`) - } else { - log.arrow(`Download: ${install.url}`) - } - log.arrow('Then: ollama serve') - process.exitCode = 1 - return - } - - // Combined disk-space pre-check (sum across all requested models). - const estimates = names.map((n) => ({ name: n, size: estimateModelSize(n) })) - const totalEstimate = estimates.reduce((acc, e) => acc + (e.size ?? 0), 0) - const available = getAvailableDiskBytes() - const margin = 1.5e9 - if (totalEstimate > 0) { - log.info(`Estimated footprint:`) - for (const e of estimates) { - const line = e.size - ? ` ${e.name.padEnd(24)} ~${formatBytes(e.size)}` - : ` ${e.name.padEnd(24)} (size unknown)` - console.log(line) - } - if (names.length > 1) { - log.info(` ${'total'.padEnd(24)} ~${formatBytes(totalEstimate)}`) - } - if (available !== null) { - log.info(`Available disk: ${formatBytes(available)}`) - if (available < totalEstimate + margin) { - log.fail( - `Not enough free disk space (need ~${formatBytes(totalEstimate + margin)}, have ${formatBytes(available)}).`, - ) - if (!opts.yes) { - const { confirm } = await import('@inquirer/prompts') - const proceed = await confirm({ message: 'Pull anyway?', default: false }) - if (!proceed) return - } - } - } - } - - const failures: string[] = [] - for (const [i, name] of names.entries()) { - if (names.length > 1) { - log.blank() - log.heading(`[${i + 1}/${names.length}] ${name}`) - } - log.wait(`Pulling ${chalk.cyan(name)} ...`) - let lastPrinted = '' - const ok = await pullOllamaModel(name, opts.baseUrl, (line) => { - if (line !== lastPrinted) { - lastPrinted = line - process.stdout.write(`\r\x1b[K ${line}`) - } - }) - process.stdout.write('\n') - - if (ok) log.ok(`Pulled ${name}`) - else { - log.fail(`Failed to pull ${name}`) - failures.push(name) - } - } - - if (failures.length > 0) { - log.blank() - log.fail(`${failures.length}/${names.length} pulls failed: ${failures.join(', ')}`) - log.arrow(`Retry manually: ${failures.map((f) => `ollama pull ${f}`).join(' && ')}`) - process.exitCode = 1 - } - }) - - // ---------- install-ollama ---------- - cmd.command('install-ollama') - .description('Install Ollama via the official script (macOS/Linux)') - .option('--yes', 'Skip confirmation') - .action(async (opts: { yes?: boolean }) => { - const install = getOllamaInstallCommand() - if (!install.supported) { - log.fail('Automatic install is not supported on this platform (Windows).') - log.arrow(`Download manually: ${install.url}`) - process.exitCode = 1 - return - } - - if (await isOllamaRunning()) { - log.ok('Ollama is already running at http://localhost:11434') - return - } - - log.info('About to run the official Ollama install script:') - log.dim(` ${install.command}`) - log.blank() - if (!opts.yes) { - const { confirm } = await import('@inquirer/prompts') - const go = await confirm({ message: 'Proceed?', default: true }) - if (!go) return - } - - const { execSync } = await import('node:child_process') - try { - execSync(install.command!, { stdio: 'inherit' }) - } catch (err) { - log.fail(`Install failed: ${(err as Error).message}`) - process.exitCode = 1 - return - } - - log.wait('Waiting for Ollama daemon to come up...') - for (let i = 0; i < 10; i++) { - await new Promise((r) => setTimeout(r, 1500)) - if (await isOllamaRunning()) { - log.ok('Ollama is running') - log.arrow('Next: opendocuments model pull gemma3:12b') - return - } - } - log.fail('Ollama did not come up within 15s. Try: ollama serve') - process.exitCode = 1 - }) - - // ---------- set-key ---------- - cmd.command('set-key ') - .description('Save an API key to .env for the given provider') - .option('--key ', 'Pass the key inline (otherwise prompted)') - .option('--env-file ', 'Path to .env file', '.env') - .action(async (provider: string, opts: { key?: string; envFile: string }) => { - const envVar = envVarForProvider(provider) - if (!envVar) { - log.fail(`Unknown provider: ${provider}`) - log.arrow('Supported: openai, anthropic, google, grok, deepseek, mistral, openai-compatible') - process.exitCode = 1 - return - } - - let key = opts.key - if (!key) { - const { password } = await import('@inquirer/prompts') - key = await password({ message: `Enter ${envVar} (input hidden):`, mask: '*' }) - } - if (!key) { - log.fail('Empty key — aborting.') - process.exitCode = 1 - return - } - - const envPath = join(process.cwd(), opts.envFile) - const existing = existsSync(envPath) ? readFileSync(envPath, 'utf8') : '' - const pattern = new RegExp(`^${envVar}=.*$`, 'm') - - let next: string - if (pattern.test(existing)) { - next = existing.replace(pattern, `${envVar}=${key}`) - } else { - const needsNewline = existing && !existing.endsWith('\n') - next = existing + (needsNewline ? '\n' : '') + `${envVar}=${key}\n` - } - writeFileSync(envPath, next) - log.ok(`Wrote ${envVar} to ${chalk.cyan(opts.envFile)}`) - - // gitignore sanity check - const gitignorePath = join(process.cwd(), '.gitignore') - if (existsSync(gitignorePath)) { - const gi = readFileSync(gitignorePath, 'utf8') - if (!/^\.env($|\n)/m.test(gi) && !gi.split('\n').some((l) => l.trim() === '.env')) { - log.fail('.env is NOT in .gitignore — your key could leak if committed.') - log.arrow('Add to .gitignore: .env') - } - } else { - log.info('No .gitignore found — consider creating one with `.env` listed.') - } - - log.arrow('Validate with: opendocuments doctor') - }) - - // ---------- rm ---------- - cmd.command('rm ') - .alias('remove') - .description('Delete an Ollama model') - .option('--base-url ', 'Ollama base URL', 'http://localhost:11434') - .option('--yes', 'Skip confirmation') - .action(async (name: string, opts: { baseUrl: string; yes?: boolean }) => { - if (!opts.yes) { - const { confirm } = await import('@inquirer/prompts') - const yes = await confirm({ message: `Delete local model ${name}?`, default: false }) - if (!yes) return - } - const ok = await deleteOllamaModel(name, opts.baseUrl) - if (ok) log.ok(`Removed ${name}`) - else { - log.fail(`Failed to remove ${name}`) - process.exitCode = 1 - } - }) - - // ---------- test ---------- - cmd.command('test') - .description('Run a quick round-trip test against the configured model') - .option('-p, --prompt ', 'Prompt to send', 'Say "pong" and nothing else.') - .action(async (opts: { prompt: string }) => { - const { getContext, shutdownContext } = await import('../utils/bootstrap.js') - try { - const ctx = await getContext() - const llm = ctx.registry.getModels().find((m) => m.capabilities.llm && m.generate) - if (!llm || !llm.generate) { - log.fail('No LLM model registered') - process.exitCode = 1 - return - } - log.info(`Testing ${llm.name}`) - log.dim(`> ${opts.prompt}`) - const start = Date.now() - let output = '' - let chunks = 0 - for await (const chunk of llm.generate(opts.prompt, { maxTokens: 64, temperature: 0.1 })) { - output += chunk - chunks++ - } - const elapsed = Date.now() - start - log.blank() - console.log(chalk.cyan(`< ${output.trim()}`)) - log.blank() - log.ok(`Round-trip ok: ${chunks} chunks, ${elapsed}ms`) - - // Embedding round-trip too - const embedder = ctx.registry.getModels().find((m) => m.capabilities.embedding && m.embed) - if (embedder && embedder.embed) { - const embStart = Date.now() - const result = await embedder.embed(['test']) - const dim = result.dense[0]?.length ?? 0 - log.ok(`Embedding ok: ${dim}-dim, ${Date.now() - embStart}ms (${embedder.name})`) - } - } catch (err) { - log.fail(`Test failed: ${(err as Error).message}`) - process.exitCode = 1 - } finally { - await shutdownContext() - } - }) - - // ---------- switch ---------- - cmd.command('switch') - .description('Change the active model provider (edits opendocuments.config.ts)') - .action(async () => { - const configPath = join(process.cwd(), 'opendocuments.config.ts') - if (!existsSync(configPath)) { - log.fail('No config file found in current directory.') - log.arrow('Run: opendocuments init') - process.exitCode = 1 - return - } - - const { select, input } = await import('@inquirer/prompts') - const provider = await select({ - message: 'Switch to provider:', - choices: [ - { name: 'Ollama (local)', value: 'ollama' }, - { name: 'OpenAI', value: 'openai' }, - { name: 'Anthropic', value: 'anthropic' }, - { name: 'Google', value: 'google' }, - { name: 'Grok (xAI)', value: 'grok' }, - { name: 'DeepSeek', value: 'deepseek' }, - { name: 'Mistral', value: 'mistral' }, - { name: 'OpenAI-compatible (vLLM/LM Studio/Groq/Together/…)', value: 'openai-compatible' }, - ], - }) - - const defaults: Record = { - ollama: { llm: 'qwen2.5:14b', embedding: 'bge-m3' }, - openai: { llm: 'gpt-4o', embedding: 'text-embedding-3-small' }, - anthropic: { llm: 'claude-sonnet-4-20250514', embedding: 'bge-m3', embeddingProvider: 'ollama' }, - google: { llm: 'gemini-2.5-flash', embedding: 'text-embedding-004' }, - grok: { llm: 'grok-4', embedding: 'grok-2-embed' }, - deepseek: { llm: 'deepseek-chat', embedding: 'bge-m3', embeddingProvider: 'ollama' }, - mistral: { llm: 'mistral-small-latest', embedding: 'mistral-embed' }, - 'openai-compatible': { llm: '', embedding: 'bge-m3', embeddingProvider: 'ollama' }, - } - - const llm = await input({ message: 'LLM model:', default: defaults[provider].llm }) - const embedding = await input({ message: 'Embedding model:', default: defaults[provider].embedding }) - - let baseUrl = '' - if (provider === 'openai-compatible') { - baseUrl = await input({ message: 'baseUrl (OpenAI-compatible endpoint):', default: 'https://api.groq.com/openai/v1' }) - } - - const src = readFileSync(configPath, 'utf8') - const updated = rewriteModelBlock(src, { - provider, - llm, - embedding, - embeddingProvider: defaults[provider].embeddingProvider, - baseUrl, - }) - - writeFileSync(configPath, updated) - log.ok(`Updated ${configPath} → provider=${provider}, llm=${llm}`) - log.arrow('Restart: opendocuments start') - if (provider !== 'ollama') { - const envVar = envVarName(provider) - log.arrow(`Set API key: export ${envVar}=... (or put it in .env)`) - } - }) - - return cmd -} - -/* ------------------------------------------------------------------ */ -/* Config rewriting */ -/* ------------------------------------------------------------------ */ - -interface ModelBlockUpdate { - provider: string - llm: string - embedding: string - embeddingProvider?: string - baseUrl?: string -} - -/** - * Rewrite the `model: { ... }` block in opendocuments.config.ts. - * This is a best-effort string rewrite for the common cases produced by `init`. - * Preserves surrounding config — throws if the model block can't be located. - */ -export function rewriteModelBlock(src: string, update: ModelBlockUpdate): string { - const re = /model:\s*\{[\s\S]*?\n\s*\},/m - if (!re.test(src)) { - throw new Error('Could not locate `model: { ... }` block in config file — edit it manually.') - } - - const lines = [ - `model: {`, - ` provider: '${update.provider}',`, - update.llm ? ` llm: '${update.llm}',` : null, - ` embedding: '${update.embedding}',`, - update.embeddingProvider ? ` embeddingProvider: '${update.embeddingProvider}',` : null, - update.baseUrl ? ` baseUrl: '${update.baseUrl}',` : null, - update.provider !== 'ollama' ? ` apiKey: process.env.${envVarName(update.provider)},` : null, - update.embeddingProvider === 'openai' ? ` embeddingApiKey: process.env.OPENAI_API_KEY,` : null, - ` },`, - ].filter(Boolean).join('\n ') - - return src.replace(re, lines) -} - -function envVarName(provider: string): string { - const map: Record = { - openai: 'OPENAI_API_KEY', - anthropic: 'ANTHROPIC_API_KEY', - google: 'GOOGLE_API_KEY', - grok: 'XAI_API_KEY', - deepseek: 'DEEPSEEK_API_KEY', - mistral: 'MISTRAL_API_KEY', - 'openai-compatible': 'OPENAI_COMPATIBLE_API_KEY', - ollama: 'OLLAMA_URL', - } - return map[provider] || 'API_KEY' -} - -/** Like envVarName but returns null for unknown providers (used by set-key). */ -function envVarForProvider(provider: string): string | null { - const map: Record = { - openai: 'OPENAI_API_KEY', - anthropic: 'ANTHROPIC_API_KEY', - google: 'GOOGLE_API_KEY', - grok: 'XAI_API_KEY', - deepseek: 'DEEPSEEK_API_KEY', - mistral: 'MISTRAL_API_KEY', - 'openai-compatible': 'OPENAI_COMPATIBLE_API_KEY', - } - return map[provider] ?? null -} diff --git a/packages/cli/src/commands/plugin.ts b/packages/cli/src/commands/plugin.ts index 83d0903..2475193 100644 --- a/packages/cli/src/commands/plugin.ts +++ b/packages/cli/src/commands/plugin.ts @@ -3,8 +3,24 @@ import { log } from 'opendocuments-core' import chalk from 'chalk' import { writeFileSync, mkdirSync, existsSync } from 'node:fs' import { join } from 'node:path' +import { execFileSync } from 'node:child_process' import { getContext, shutdownContext } from '../utils/bootstrap.js' +function assertNpmPackageName(name: string): void { + if (!/^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/i.test(name)) { + throw new Error(`Invalid npm package name: ${name}`) + } +} + +function runNpm(args: string[], options: { capture?: boolean } = {}): string | void { + const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm' + const result = execFileSync(npmCmd, args, { + stdio: options.capture ? ['ignore', 'pipe', 'ignore'] : 'inherit', + encoding: options.capture ? 'utf-8' : undefined, + }) + return typeof result === 'string' ? result : undefined +} + export function pluginCommand() { const cmd = new Command('plugin') .description('Manage and develop plugins') @@ -95,7 +111,7 @@ export class MyParser implements ParserPlugin { type = 'parser' as const version = '0.1.0' coreVersion = '^0.3.0' - supportedTypes = ['.ext'] // TODO: set your file extensions + supportedTypes = ['.ext'] // Replace with the file extensions handled by this parser. async setup(_ctx: PluginContext): Promise {} async healthCheck(): Promise { return { healthy: true } } @@ -121,11 +137,11 @@ export class MyConnector implements ConnectorPlugin { async healthCheck(): Promise { return { healthy: true } } async *discover(): AsyncIterable { - // TODO: yield discovered documents + // Yield discovered documents from your source system. } async fetch(ref: DocumentRef): Promise { - // TODO: fetch document content + // Fetch document content from your source system. return { sourceId: ref.sourceId, title: ref.sourcePath, content: '' } } } @@ -158,7 +174,7 @@ export class MyMiddleware implements MiddlewarePlugin { { stage: 'after:parse' as PipelineStage, handler: async (data: unknown) => { - // TODO: process data + // Transform or enrich parsed data here. return data }, }, @@ -220,9 +236,8 @@ describe('${name}', () => { cmd.command('search ') .description('Search for OpenDocuments plugins on npm') .action(async (query) => { - const { execSync } = await import('node:child_process') try { - const result = execSync(`npm search opendocuments-plugin ${query} --json 2>/dev/null || echo "[]"`, { encoding: 'utf-8' }) + const result = runNpm(['search', 'opendocuments-plugin', query, '--json'], { capture: true }) || '[]' const packages = JSON.parse(result) if (packages.length === 0) { log.info('No plugins found'); return } log.heading(`Search Results (${packages.length})`) @@ -247,31 +262,30 @@ describe('${name}', () => { }) cmd.command('add ').description('Install a plugin').action(async (name) => { - const { execSync } = await import('node:child_process') log.wait(`Installing ${name}...`) try { - execSync(`npm install ${name}`, { stdio: 'inherit' }) + assertNpmPackageName(name) + runNpm(['install', name]) log.ok(`${name} installed. Restart server to activate.`) } catch { log.fail(`Failed to install ${name}`) } }) cmd.command('remove ').description('Remove a plugin').action(async (name) => { - const { execSync } = await import('node:child_process') log.wait(`Removing ${name}...`) try { - execSync(`npm uninstall ${name}`, { stdio: 'inherit' }) + assertNpmPackageName(name) + runNpm(['uninstall', name]) log.ok(`${name} removed. Restart server to apply.`) } catch { log.fail(`Failed to remove ${name}`) } }) cmd.command('update [name]').description('Update plugins').action(async (name) => { - const { execSync } = await import('node:child_process') if (name) { log.wait(`Updating ${name}...`) - try { execSync(`npm update ${name}`, { stdio: 'inherit' }); log.ok('Updated') } catch { log.fail('Failed') } + try { assertNpmPackageName(name); runNpm(['update', name]); log.ok('Updated') } catch { log.fail('Failed') } } else { log.wait('Updating all plugins...') - try { execSync('npm update', { stdio: 'inherit' }); log.ok('All plugins updated') } catch { log.fail('Failed') } + try { runNpm(['update']); log.ok('All plugins updated') } catch { log.fail('Failed') } } }) diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 80e45e7..287d281 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -105,13 +105,15 @@ export function startCommand() { } log.dim('Press Ctrl+C to stop') }) - process.on('SIGINT', async () => { + const shutdown = async () => { log.blank() log.wait('Shutting down...') try { unlinkSync(pidFile) } catch {} await ctx.shutdown() log.ok('Goodbye') process.exit(0) - }) + } + process.on('SIGINT', shutdown) + process.on('SIGTERM', shutdown) }) } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index ded4848..275461e 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -21,7 +21,6 @@ import { stopCommand } from './commands/stop.js' import { searchCommand } from './commands/search.js' import { completionCommand } from './commands/completion.js' import { upgradeCommand } from './commands/upgrade.js' -import { modelCommand } from './commands/model.js' import { backupCommand, restoreCommand } from './commands/backup.js' const program = new Command() @@ -49,6 +48,5 @@ program.addCommand(completionCommand()) program.addCommand(upgradeCommand()) program.addCommand(backupCommand()) program.addCommand(restoreCommand()) -program.addCommand(modelCommand()) program.parse() diff --git a/packages/cli/src/utils/ollama.ts b/packages/cli/src/utils/ollama.ts deleted file mode 100644 index 9b5d3a4..0000000 --- a/packages/cli/src/utils/ollama.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { statfsSync } from 'node:fs' -import { homedir, platform } from 'node:os' - -export interface OllamaModelInfo { - name: string - size: number - digest?: string - modified_at?: string - details?: { - parameter_size?: string - quantization_level?: string - family?: string - } -} - -export async function isOllamaRunning(baseUrl = 'http://localhost:11434'): Promise { - try { - const res = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(3000) }) - return res.ok - } catch { - return false - } -} - -export async function listOllamaModels(baseUrl = 'http://localhost:11434'): Promise { - try { - const res = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(5000) }) - if (!res.ok) return [] - const data = (await res.json()) as { models?: OllamaModelInfo[] } - return data.models ?? [] - } catch { - return [] - } -} - -/** - * Pull an Ollama model with progress streaming. - * Yields human-readable progress lines; resolves with true on success, false on failure. - */ -export async function pullOllamaModel( - model: string, - baseUrl = 'http://localhost:11434', - onProgress?: (line: string) => void, -): Promise { - try { - const res = await fetch(`${baseUrl}/api/pull`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ model, stream: true }), - }) - if (!res.ok || !res.body) return false - - const reader = res.body.getReader() - const decoder = new TextDecoder() - let buffer = '' - let lastStatus = '' - - while (true) { - const { done, value } = await reader.read() - if (done) break - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n') - buffer = lines.pop() || '' - for (const line of lines) { - if (!line.trim()) continue - try { - const evt = JSON.parse(line) as { - status?: string - total?: number - completed?: number - error?: string - } - if (evt.error) { - onProgress?.(`error: ${evt.error}`) - return false - } - if (evt.status && evt.status !== lastStatus) { - lastStatus = evt.status - if (evt.total && evt.completed !== undefined) { - const pct = ((evt.completed / evt.total) * 100).toFixed(1) - onProgress?.(`${evt.status} — ${pct}% (${formatBytes(evt.completed)}/${formatBytes(evt.total)})`) - } else { - onProgress?.(evt.status) - } - } else if (evt.status === lastStatus && evt.total && evt.completed !== undefined) { - const pct = ((evt.completed / evt.total) * 100).toFixed(1) - onProgress?.(`${evt.status} — ${pct}%`) - } - } catch { - // skip unparseable lines - } - } - } - return true - } catch (err) { - onProgress?.(`error: ${(err as Error).message}`) - return false - } -} - -export async function deleteOllamaModel( - model: string, - baseUrl = 'http://localhost:11434', -): Promise { - try { - const res = await fetch(`${baseUrl}/api/delete`, { - method: 'DELETE', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ model }), - }) - return res.ok - } catch { - return false - } -} - -/** - * Best-effort available disk space in bytes for a given directory. Returns null on - * unsupported platforms or permission errors. - */ -export function getAvailableDiskBytes(dir: string = homedir()): number | null { - try { - const stat = statfsSync(dir) - return Number(stat.bavail) * Number(stat.bsize) - } catch { - return null - } -} - -export function formatBytes(bytes: number): string { - if (!Number.isFinite(bytes) || bytes < 0) return '?' - const units = ['B', 'KB', 'MB', 'GB', 'TB'] - let v = bytes - let i = 0 - while (v >= 1024 && i < units.length - 1) { v /= 1024; i++ } - return `${v.toFixed(i === 0 ? 0 : 1)}${units[i]}` -} - -/** - * Rough estimate of disk footprint per Ollama model tag. - * Used for pre-pull warnings only; Ollama will be authoritative at pull time. - */ -const MODEL_SIZE_HINTS: Record = { - // llama family - 'llama3.3:8b': 5e9, - 'llama4:scout': 65e9, - 'llama4:maverick': 230e9, - // qwen - 'qwen2.5:7b': 4.7e9, - 'qwen2.5:14b': 9e9, - 'qwen2.5:32b': 20e9, - 'qwen3.5:9b': 5.5e9, - 'qwen3.5:27b': 17e9, - // gemma - 'gemma2:2b': 1.6e9, - 'gemma2:9b': 5.4e9, - 'gemma3:1b': 0.8e9, - 'gemma3:4b': 2.5e9, - 'gemma3:12b': 7.5e9, - 'gemma3:27b': 17e9, - 'gemma3n': 2.8e9, - // deepseek - 'deepseek-r1:7b': 4.7e9, - 'deepseek-r1:14b': 9e9, - 'deepseek-r1:32b': 20e9, - // phi - 'phi4:14b': 9e9, - // embedding - 'bge-m3': 1.2e9, - 'nomic-embed-text': 0.3e9, - 'mxbai-embed-large': 0.7e9, -} - -export function estimateModelSize(tag: string): number | null { - const lower = tag.toLowerCase() - if (MODEL_SIZE_HINTS[lower]) return MODEL_SIZE_HINTS[lower] - // Fuzzy match by prefix (e.g. "gemma3:27b-instruct-q4_0" → "gemma3:27b") - for (const [k, v] of Object.entries(MODEL_SIZE_HINTS)) { - if (lower.startsWith(k)) return v - } - return null -} - -/** - * Offer to install Ollama via the official install script. Returns true if the - * platform is supported and install script exists. Never runs without the caller - * confirming — this just resolves the install approach. - */ -export function getOllamaInstallCommand(): { supported: boolean; command?: string; url: string } { - const p = platform() - const url = 'https://ollama.com/download' - if (p === 'darwin' || p === 'linux') { - return { supported: true, command: 'curl -fsSL https://ollama.com/install.sh | sh', url } - } - return { supported: false, url } -} diff --git a/packages/cli/tests/commands/ask.test.ts b/packages/cli/tests/commands/ask.test.ts index 255fe5a..5eb798f 100644 --- a/packages/cli/tests/commands/ask.test.ts +++ b/packages/cli/tests/commands/ask.test.ts @@ -7,17 +7,20 @@ import { tmpdir } from 'node:os' describe('ask command logic', () => { let ctx: AppContext let tempDir: string + const stubModel = { + provider: 'stub', + llm: 'stub-llm', + embedding: 'stub-embedding', + apiKey: '', + baseUrl: '', + embeddingDimensions: 384, + } as any beforeEach(async () => { tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) - ctx = await bootstrap({ - dataDir: tempDir, - configOverrides: { - model: { provider: 'test-stub', llm: 'stub-llm', embedding: 'stub-embedding' }, - }, - }) + ctx = await bootstrap({ dataDir: tempDir, configOverrides: { model: stubModel } }) }) - afterEach(async () => { await ctx.shutdown(); rmSync(tempDir, { recursive: true, force: true }) }) + afterEach(async () => { if (ctx) await ctx.shutdown(); rmSync(tempDir, { recursive: true, force: true }) }) it('answers a greeting directly', async () => { const result = await ctx.ragEngine.query({ query: 'Hello', profile: 'balanced' }) diff --git a/packages/cli/tests/commands/doctor.test.ts b/packages/cli/tests/commands/doctor.test.ts index 6851989..323a76a 100644 --- a/packages/cli/tests/commands/doctor.test.ts +++ b/packages/cli/tests/commands/doctor.test.ts @@ -7,17 +7,20 @@ import { tmpdir } from 'node:os' describe('doctor command logic', () => { let ctx: AppContext let tempDir: string + const stubModel = { + provider: 'stub', + llm: 'stub-llm', + embedding: 'stub-embedding', + apiKey: '', + baseUrl: '', + embeddingDimensions: 384, + } as any beforeEach(async () => { tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) - ctx = await bootstrap({ - dataDir: tempDir, - configOverrides: { - model: { provider: 'test-stub', llm: 'stub-llm', embedding: 'stub-embedding' }, - }, - }) + ctx = await bootstrap({ dataDir: tempDir, configOverrides: { model: stubModel } }) }) - afterEach(async () => { await ctx.shutdown(); rmSync(tempDir, { recursive: true, force: true }) }) + afterEach(async () => { if (ctx) await ctx.shutdown(); rmSync(tempDir, { recursive: true, force: true }) }) it('reports healthy state', () => { expect(ctx.store.listDocuments()).toEqual([]) diff --git a/packages/cli/tests/commands/model.test.ts b/packages/cli/tests/commands/model.test.ts deleted file mode 100644 index 7068975..0000000 --- a/packages/cli/tests/commands/model.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { rewriteModelBlock } from '../../src/commands/model.js' - -const SAMPLE_CONFIG = `import { defineConfig } from 'opendocuments-core' - -export default defineConfig({ - workspace: 'demo', - mode: 'personal', - - model: { - provider: 'ollama', - llm: 'qwen2.5:14b', - embedding: 'bge-m3', - }, - - rag: { - profile: 'balanced', - }, -}) -` - -describe('rewriteModelBlock', () => { - it('switches to a cloud provider and adds apiKey', () => { - const out = rewriteModelBlock(SAMPLE_CONFIG, { - provider: 'deepseek', - llm: 'deepseek-chat', - embedding: 'bge-m3', - embeddingProvider: 'ollama', - }) - expect(out).toContain(`provider: 'deepseek',`) - expect(out).toContain(`llm: 'deepseek-chat',`) - expect(out).toContain(`embeddingProvider: 'ollama',`) - expect(out).toContain(`apiKey: process.env.DEEPSEEK_API_KEY,`) - // Outside model block is preserved - expect(out).toContain(`profile: 'balanced',`) - }) - - it('supports openai-compatible provider with baseUrl', () => { - const out = rewriteModelBlock(SAMPLE_CONFIG, { - provider: 'openai-compatible', - llm: 'llama-4-70b', - embedding: 'bge-m3', - embeddingProvider: 'ollama', - baseUrl: 'https://api.groq.com/openai/v1', - }) - expect(out).toContain(`baseUrl: 'https://api.groq.com/openai/v1',`) - expect(out).toContain(`apiKey: process.env.OPENAI_COMPATIBLE_API_KEY,`) - }) - - it('round-trips back to ollama without apiKey line', () => { - const withCloud = rewriteModelBlock(SAMPLE_CONFIG, { - provider: 'openai', - llm: 'gpt-4o', - embedding: 'text-embedding-3-small', - }) - const back = rewriteModelBlock(withCloud, { - provider: 'ollama', - llm: 'gemma3:12b', - embedding: 'bge-m3', - }) - expect(back).toContain(`provider: 'ollama',`) - expect(back).toContain(`llm: 'gemma3:12b',`) - expect(back).not.toContain('apiKey: process.env') - }) - - it('throws when model block is missing', () => { - expect(() => rewriteModelBlock('export default {}', { - provider: 'openai', - llm: 'x', - embedding: 'y', - })).toThrow(/model:/) - }) -}) diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 1861ea0..99fc9b3 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -3,14 +3,73 @@ export interface OpenDocumentsClientOptions { apiKey?: string } +export interface SearchResult { + chunkId?: string + content: string + score: number + documentId: string + chunkType?: string + headingHierarchy?: string[] + sourcePath?: string + sourceType?: string +} + +export interface ConfidenceResult { + score: number + level: 'high' | 'medium' | 'low' | 'none' + reason: string +} + +export interface AskOptions { + profile?: 'fast' | 'balanced' | 'precise' | string + conversationId?: string +} + export interface QueryResult { - queryId: string; answer: string; sources: Array<{ documentId: string; content: string; score: number }>; confidence: number; route: string; profile: string + queryId: string + answer: string + sources: SearchResult[] + confidence: ConfidenceResult + route: string + profile: string } export interface DocumentListResponse { documents: Array<{ id: string; title: string; source_type: string; status: string; chunk_count: number }> } +export interface UploadDocumentResponse { + documentId: string + chunks: number + status: string +} + +export interface Conversation { + id: string + title: string | null + workspace_id?: string + shared?: number | boolean + share_token?: string | null + created_at?: string + updated_at?: string +} + +export interface ConversationMessage { + id: string + conversation_id?: string + role: 'user' | 'assistant' | 'system' + content: string + sources?: SearchResult[] | string | null + metadata?: Record | string | null + created_at?: string +} + +export interface ConversationListResponse { + conversations: Conversation[] + limit: number + offset: number +} + export interface StatsResponse { documents: number; workspaces: number; plugins: number } @@ -33,15 +92,16 @@ export class OpenDocumentsClient { return res.json() } - async ask(query: string, profile?: string): Promise { - return this.request('/chat', { method: 'POST', body: JSON.stringify({ query, profile }) }) + async ask(query: string, optionsOrProfile?: AskOptions | string): Promise { + const options = typeof optionsOrProfile === 'string' ? { profile: optionsOrProfile } : (optionsOrProfile ?? {}) + return this.request('/chat', { method: 'POST', body: JSON.stringify({ query, ...options }) }) } async listDocuments(): Promise { return this.request('/documents') } - async uploadDocument(file: File): Promise<{ id: string; status: string }> { + async uploadDocument(file: File): Promise { const formData = new FormData() formData.append('file', file) const res = await fetch(`${this.baseUrl}/api/v1/documents/upload`, { @@ -62,4 +122,38 @@ export class OpenDocumentsClient { async getStats(): Promise { return this.request('/stats') } + + async listConversations(opts?: { limit?: number; offset?: number }): Promise { + const params = new URLSearchParams() + if (opts?.limit) params.set('limit', String(opts.limit)) + if (opts?.offset) params.set('offset', String(opts.offset)) + const query = params.toString() + return this.request(`/conversations${query ? `?${query}` : ''}`) + } + + async createConversation(title?: string): Promise { + return this.request('/conversations', { + method: 'POST', + body: JSON.stringify(title ? { title } : {}), + }) + } + + async listConversationMessages(id: string): Promise<{ messages: ConversationMessage[] }> { + return this.request(`/conversations/${encodeURIComponent(id)}/messages`) + } + + async updateConversation(id: string, input: { title?: string }): Promise<{ updated: true }> { + return this.request(`/conversations/${encodeURIComponent(id)}`, { + method: 'PATCH', + body: JSON.stringify(input), + }) + } + + async deleteConversation(id: string): Promise<{ deleted: true }> { + return this.request(`/conversations/${encodeURIComponent(id)}`, { method: 'DELETE' }) + } + + async shareConversation(id: string): Promise<{ shareUrl: string }> { + return this.request(`/conversations/${encodeURIComponent(id)}/share`, { method: 'POST' }) + } } diff --git a/packages/client/tests/client.test.ts b/packages/client/tests/client.test.ts index 6978dd5..a10bdd9 100644 --- a/packages/client/tests/client.test.ts +++ b/packages/client/tests/client.test.ts @@ -22,10 +22,43 @@ describe('OpenDocumentsClient', () => { it('asks a question', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ - ok: true, json: async () => ({ queryId: 'q1', answer: 'test', sources: [], confidence: {}, route: 'rag', profile: 'balanced' }), + ok: true, json: async () => ({ + queryId: 'q1', + answer: 'test', + sources: [], + confidence: { score: 0.8, level: 'high', reason: 'grounded' }, + route: 'rag', + profile: 'balanced', + }), })) - const result = await client.ask('Hello') + const result = await client.ask('Hello', { profile: 'balanced' }) expect(result.answer).toBe('test') + expect(result.confidence.level).toBe('high') + vi.unstubAllGlobals() + }) + + it('uploads documents with the server response shape', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ documentId: 'doc-1', chunks: 3, status: 'indexed' }), + })) + + const result = await client.uploadDocument(new File(['hello'], 'hello.md')) + + expect(result.documentId).toBe('doc-1') + expect(result.chunks).toBe(3) + vi.unstubAllGlobals() + }) + + it('lists conversations', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ conversations: [{ id: 'c1', title: 'Thread' }], limit: 50, offset: 0 }), + })) + + const result = await client.listConversations() + + expect(result.conversations[0].id).toBe('c1') vi.unstubAllGlobals() }) }) diff --git a/packages/core/src/auth/api-key.ts b/packages/core/src/auth/api-key.ts index fa0cbb2..aa321eb 100644 --- a/packages/core/src/auth/api-key.ts +++ b/packages/core/src/auth/api-key.ts @@ -62,14 +62,14 @@ export class APIKeyManager { const scopes = input.scopes || (input.role === 'admin' ? ['*'] : input.role === 'member' ? ['ask', 'search', 'document:read', 'document:write', 'connector:read'] : ['ask', 'search', 'document:read']) this.db.run( - `INSERT INTO api_keys (id, name, key_hash, key_prefix, workspace_id, user_id, role, scopes, rate_limit, expires_at, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - [id, input.name, keyHash, keyPrefix, input.workspaceId, input.userId, input.role, JSON.stringify(scopes), input.rateLimit || null, input.expiresAt || null, now] + `INSERT INTO api_keys (id, name, key_hash, key_prefix, workspace_id, user_id, role, scopes, rate_limit, allowed_ips, expires_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [id, input.name, keyHash, keyPrefix, input.workspaceId, input.userId, input.role, JSON.stringify(scopes), input.rateLimit || null, JSON.stringify(input.allowedIps || []), input.expiresAt || null, now] ) return { rawKey, - record: { id, name: input.name, keyHash, keyPrefix, workspaceId: input.workspaceId, userId: input.userId, role: input.role, scopes, rateLimit: input.rateLimit, expiresAt: input.expiresAt, createdAt: now }, + record: { id, name: input.name, keyHash, keyPrefix, workspaceId: input.workspaceId, userId: input.userId, role: input.role, scopes, rateLimit: input.rateLimit, allowedIps: input.allowedIps || [], expiresAt: input.expiresAt, createdAt: now }, } } @@ -105,6 +105,7 @@ export class APIKeyManager { role: row.role, scopes: JSON.parse(row.scopes), rateLimit: row.rate_limit, + allowedIps: row.allowed_ips ? JSON.parse(row.allowed_ips) : [], expiresAt: row.expires_at, lastUsedAt: row.last_used_at, createdAt: row.created_at, @@ -133,6 +134,7 @@ export class APIKeyManager { role: row.role, scopes: JSON.parse(row.scopes), rateLimit: row.rate_limit, + allowedIps: row.allowed_ips ? JSON.parse(row.allowed_ips) : [], expiresAt: row.expires_at, lastUsedAt: row.last_used_at, createdAt: row.created_at, diff --git a/packages/core/src/config/defaults.ts b/packages/core/src/config/defaults.ts index 6a67f58..e527e69 100644 --- a/packages/core/src/config/defaults.ts +++ b/packages/core/src/config/defaults.ts @@ -15,12 +15,12 @@ export const DEFAULT_CONFIG: OpenDocumentsConfig = { sourceRestrictions: { localOnly: [], cloudAllowed: [] }, workspaceOverrides: {}, }, - transport: { enforceHTTPS: true, allowedEndpoints: [] }, + transport: { enforceHTTPS: true, allowedEndpoints: [], allowedOrigins: [], widgetAllowedDomains: [] }, storage: { encryptAtRest: false, redactLogsContent: true }, audit: { enabled: false, events: [], destination: 'local' }, }, ui: { locale: 'auto', theme: 'auto' }, - telemetry: { enabled: false }, + telemetry: { enabled: false, endpoint: undefined }, storage: { db: 'sqlite', vectorDb: 'lancedb', dataDir: '~/.opendocuments' }, webhooks: [], } diff --git a/packages/core/src/config/env-loader.ts b/packages/core/src/config/env-loader.ts index 386cf77..d3da2dd 100644 --- a/packages/core/src/config/env-loader.ts +++ b/packages/core/src/config/env-loader.ts @@ -41,6 +41,7 @@ const DETECTED_VARS: ReadonlyArray = [ 'OPENDOCUMENTS_UI_LOCALE', 'OPENDOCUMENTS_UI_THEME', 'OPENDOCUMENTS_TELEMETRY_ENABLED', + 'OPENDOCUMENTS_TELEMETRY_ENDPOINT', ] /** @@ -181,8 +182,11 @@ export function buildConfigFromEnv(): EnvConfig | null { } // ── Telemetry ──────────────────────────────────────────────────────────── - if (env.OPENDOCUMENTS_TELEMETRY_ENABLED !== undefined) { - config.telemetry = { enabled: env.OPENDOCUMENTS_TELEMETRY_ENABLED === 'true' } + if (env.OPENDOCUMENTS_TELEMETRY_ENABLED !== undefined || env.OPENDOCUMENTS_TELEMETRY_ENDPOINT !== undefined) { + config.telemetry = { + ...(env.OPENDOCUMENTS_TELEMETRY_ENABLED !== undefined && { enabled: env.OPENDOCUMENTS_TELEMETRY_ENABLED === 'true' }), + ...(env.OPENDOCUMENTS_TELEMETRY_ENDPOINT !== undefined && { endpoint: env.OPENDOCUMENTS_TELEMETRY_ENDPOINT }), + } } return config diff --git a/packages/core/src/config/loader.ts b/packages/core/src/config/loader.ts index 9578f17..e8637e7 100644 --- a/packages/core/src/config/loader.ts +++ b/packages/core/src/config/loader.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync } from 'node:fs' import { resolve } from 'node:path' -import { createJiti } from 'jiti' +import createJiti from 'jiti' import { configSchema, type OpenDocumentsConfig } from './schema.js' import { DEFAULT_CONFIG } from './defaults.js' import { buildConfigFromEnv } from './env-loader.js' diff --git a/packages/core/src/config/schema.ts b/packages/core/src/config/schema.ts index d080b3f..4f86f50 100644 --- a/packages/core/src/config/schema.ts +++ b/packages/core/src/config/schema.ts @@ -21,8 +21,12 @@ export const securitySchema = z.object({ }).default({}), transport: z.object({ enforceHTTPS: z.boolean().default(true), + // Comma-separated reverse-proxy IPs or CIDR ranges. Forwarded headers are + // ignored unless the direct TCP peer matches this allowlist. proxy: z.string().optional(), allowedEndpoints: z.array(z.string()).default([]), + allowedOrigins: z.array(z.string()).default([]), + widgetAllowedDomains: z.array(z.string()).default([]), }).default({}), storage: z.object({ encryptAtRest: z.boolean().default(false), @@ -78,6 +82,7 @@ export const configSchema = z.object({ }).default({}), telemetry: z.object({ enabled: z.boolean().default(false), + endpoint: z.string().url().optional(), }).default({}), storage: z.object({ db: z.enum(['sqlite', 'postgres']).default('sqlite'), diff --git a/packages/core/src/connector/manager.ts b/packages/core/src/connector/manager.ts index 5be1ee4..b7d47f8 100644 --- a/packages/core/src/connector/manager.ts +++ b/packages/core/src/connector/manager.ts @@ -23,7 +23,8 @@ export class ConnectorManager { private store: DocumentStore, private eventBus: EventBus, private db: DB, - private workspaceId: string = 'default' + private workspaceId: string = 'default', + private syncConcurrency: number = 4 ) {} /** @@ -32,18 +33,39 @@ export class ConnectorManager { registerConnector(plugin: ConnectorPlugin, config: { name?: string syncIntervalSeconds?: number + autoSync?: boolean + config?: Record } = {}): string { - const connectorId = randomUUID() const name = config.name || plugin.name - - // Save connector record to DB - this.db.run( - `INSERT INTO connectors (id, workspace_id, name, type, config, sync_interval_seconds, status, created_at) - VALUES (?, ?, ?, ?, ?, ?, 'active', ?)`, - [connectorId, this.workspaceId, name, plugin.name, '{}', config.syncIntervalSeconds || 300, new Date().toISOString()] + const serializedConfig = JSON.stringify(config.config || {}) + const existing = this.db.get<{ id: string }>( + `SELECT id FROM connectors + WHERE workspace_id = ? AND name = ? AND type = ? AND deleted_at IS NULL + ORDER BY created_at DESC LIMIT 1`, + [this.workspaceId, name, plugin.name] ) + const connectorId = existing?.id || randomUUID() + + if (existing) { + this.db.run( + `UPDATE connectors + SET config = ?, sync_interval_seconds = ?, status = 'active', error_message = NULL + WHERE id = ?`, + [serializedConfig, config.syncIntervalSeconds || 300, connectorId] + ) + } else { + this.db.run( + `INSERT INTO connectors (id, workspace_id, name, type, config, sync_interval_seconds, status, created_at) + VALUES (?, ?, ?, ?, ?, ?, 'active', ?)`, + [connectorId, this.workspaceId, name, plugin.name, serializedConfig, config.syncIntervalSeconds || 300, new Date().toISOString()] + ) + } + this.connectors.set(plugin.name, { plugin, connectorId }) + if (config.autoSync) { + this.startPeriodicSync(plugin.name, config.syncIntervalSeconds || 300) + } return connectorId } @@ -66,7 +88,9 @@ export class ConnectorManager { this.eventBus.emit('connector:sync:started', { connectorId }) try { - for await (const discovered of plugin.discover()) { + const concurrency = Math.max(1, this.syncConcurrency) + const inFlight = new Set>() + const syncOne = async (discovered: DiscoveredDocument): Promise => { result.documentsDiscovered++ this.eventBus.emit('document:discovered', { documentId: discovered.sourceId, source: plugin.name }) @@ -75,7 +99,7 @@ export class ConnectorManager { const existing = this.store.getDocumentBySourcePath(discovered.sourcePath) if (existing && discovered.contentHash && !this.store.hasContentChanged(existing.id, discovered.contentHash)) { result.documentsSkipped++ - continue + return } // Fetch full content @@ -101,6 +125,17 @@ export class ConnectorManager { result.errors.push(`${discovered.title}: ${(err as Error).message}`) } } + + for await (const discovered of plugin.discover()) { + const task = syncOne(discovered).finally(() => { + inFlight.delete(task) + }) + inFlight.add(task) + if (inFlight.size >= concurrency) { + await Promise.race(inFlight) + } + } + await Promise.all(inFlight) } catch (err) { result.errors.push(`Discovery failed: ${(err as Error).message}`) } @@ -164,14 +199,22 @@ export class ConnectorManager { /** * List registered connectors with their DB status. */ - listConnectors(): { name: string; connectorId: string; status: string; lastSyncedAt: string | null }[] { + listConnectors(): { name: string; connectorId: string; type: string; status: string; lastSyncedAt: string | null; syncIntervalSeconds: number | null; repo?: string }[] { return Array.from(this.connectors.entries()).map(([name, entry]) => { - const row = this.db.get('SELECT status, last_synced_at FROM connectors WHERE id = ?', [entry.connectorId]) + const row = this.db.get('SELECT name, type, config, status, last_synced_at, sync_interval_seconds FROM connectors WHERE id = ?', [entry.connectorId]) + let repo: string | undefined + try { + const parsed = JSON.parse(row?.config || '{}') as { repo?: unknown } + if (typeof parsed.repo === 'string') repo = parsed.repo + } catch {} return { - name, + name: row?.name || name, connectorId: entry.connectorId, + type: row?.type || name, status: row?.status || 'unknown', lastSyncedAt: row?.last_synced_at || null, + syncIntervalSeconds: row?.sync_interval_seconds || null, + repo, } }) } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 15c682b..e947042 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -6,7 +6,7 @@ export const VERSION = '0.3.0' export { log } from './utils/logger.js' export { estimateTokens } from './utils/tokenizer.js' export { sha256 } from './utils/hash.js' -export { discoverFiles } from './utils/file-discovery.js' +export { discoverFiles, SUPPORTED_EXTENSIONS } from './utils/file-discovery.js' export { fetchWithTimeout } from './utils/fetch.js' export { parseModelName, @@ -17,7 +17,7 @@ export { ensureOllamaModel, type OllamaModel, } from './utils/ollama.js' -export { FileWatcher, type FileChange } from './utils/file-watcher.js' +export { FileWatcher, type FileChange, type FileWatcherOptions } from './utils/file-watcher.js' export { detectHardware, recommendModels, diff --git a/packages/core/src/ingest/chunk-strategies.ts b/packages/core/src/ingest/chunk-strategies.ts index 83ab944..02ead6c 100644 --- a/packages/core/src/ingest/chunk-strategies.ts +++ b/packages/core/src/ingest/chunk-strategies.ts @@ -1,7 +1,8 @@ import type { EmbeddingResult } from '../plugin/interfaces.js' import { chunkText, semanticChunkText, type ChunkOptions, type TextChunk } from './chunker.js' +import { estimateTokens } from '../utils/tokenizer.js' -export type ChunkStrategy = 'markdown' | 'code' | 'table' | 'data' | 'prose' | 'api' +export type ChunkStrategy = 'markdown' | 'code' | 'table' | 'data' | 'prose' | 'api' | 'slide' const MARKDOWN_EXT = new Set(['.md', '.mdx', '.markdown']) const CODE_EXT = new Set([ @@ -19,6 +20,7 @@ const TABLE_EXT = new Set(['.csv', '.tsv', '.xlsx', '.xls']) */ export function selectChunkStrategy(fileType: string, chunkType: string): ChunkStrategy { if (chunkType === 'api-endpoint') return 'api' + if (chunkType === 'slide') return 'slide' const ext = (fileType || '').toLowerCase() if (chunkType === 'code-ast' || CODE_EXT.has(ext)) return 'code' if (chunkType === 'table' || TABLE_EXT.has(ext)) return 'table' @@ -44,11 +46,12 @@ const BUDGETS: Record = { data: { maxTokens: 400, overlap: 0 }, prose: { maxTokens: 512, overlap: 64 }, api: { maxTokens: 800, overlap: 0 }, + slide: { maxTokens: 500, overlap: 0 }, } /** * Dispatch chunking based on document strategy. - * - code / table / api: parsers already produced atomic units — pass through unchanged. + * - code / table / api / slide: preserve parser units when possible, but split oversized units. * - markdown / prose: embedding-aware semantic chunking (falls back to paragraph split when no embedder). * - data: paragraph chunking only (JSON/YAML blank-line separation preserves top-level keys). */ @@ -61,14 +64,12 @@ export async function dispatchChunk( switch (strategy) { case 'code': + return splitOversizedCode(content, opts) case 'table': + return splitOversizedTable(content, opts) case 'api': - return [{ - content, - position: 0, - tokenCount: Math.ceil(content.length / 4), - headingHierarchy: [], - }] + case 'slide': + return splitOversizedLines(content, opts) case 'markdown': case 'prose': return ctx.embed @@ -78,3 +79,199 @@ export async function dispatchChunk( return chunkText(content, opts) } } + +function toTextChunks(chunks: string[]): TextChunk[] { + return chunks + .map(content => content.trim()) + .filter(content => content.length > 0) + .map((content, position) => ({ + content, + position, + tokenCount: estimateTokens(content), + headingHierarchy: [], + })) +} + +function splitOversizedLines(content: string, opts: ChunkOptions): TextChunk[] { + if (estimateTokens(content) <= opts.maxTokens) return toTextChunks([content]) + + const chunks: string[] = [] + let current: string[] = [] + let currentTokens = 0 + + const flush = () => { + if (current.length === 0) return + chunks.push(current.join('\n')) + current = [] + currentTokens = 0 + } + + for (const line of content.split('\n')) { + const lineTokens = estimateTokens(line) + if (lineTokens > opts.maxTokens) { + flush() + chunks.push(...splitTextByTokenBudget(line, opts.maxTokens)) + continue + } + if (current.length > 0 && currentTokens + lineTokens > opts.maxTokens) { + flush() + } + current.push(line) + currentTokens += lineTokens + } + flush() + + return toTextChunks(chunks) +} + +function splitTextByTokenBudget(text: string, maxTokens: number): string[] { + const totalTokens = estimateTokens(text) + if (totalTokens <= maxTokens) return [text] + + const parts = text.includes(' ') + ? text.split(/(\s+)/).filter(part => part.length > 0) + : splitByEstimatedCharacterBudget(text, maxTokens, totalTokens) + const chunks: string[] = [] + let current = '' + + const flush = () => { + if (!current) return + chunks.push(current) + current = '' + } + + for (const part of parts) { + const candidate = current + part + if (current && estimateTokens(candidate) > maxTokens) { + flush() + } + + if (estimateTokens(part) <= maxTokens) { + current += part + continue + } + + let remainder = part + while (estimateTokens(remainder) > maxTokens) { + let lo = 1 + let hi = remainder.length + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2) + if (estimateTokens(remainder.slice(0, mid)) <= maxTokens) lo = mid + else hi = mid - 1 + } + chunks.push(remainder.slice(0, lo)) + remainder = remainder.slice(lo) + } + current = remainder + } + + flush() + return chunks +} + +function splitByEstimatedCharacterBudget(text: string, maxTokens: number, totalTokens: number): string[] { + const charsPerToken = text.length / totalTokens + const targetChars = Math.max(1, Math.floor(maxTokens * charsPerToken * 0.9)) + const chunks: string[] = [] + let index = 0 + + while (index < text.length) { + let end = Math.min(text.length, index + targetChars) + let candidate = text.slice(index, end) + + while (candidate.length > 1 && estimateTokens(candidate) > maxTokens) { + end = index + Math.max(1, Math.floor((end - index) * 0.9)) + candidate = text.slice(index, end) + } + + chunks.push(candidate) + index = end + } + + return chunks +} + +function splitOversizedCode(content: string, opts: ChunkOptions): TextChunk[] { + if (estimateTokens(content) <= opts.maxTokens) return toTextChunks([content]) + + const lines = content.split('\n') + const blocks: string[] = [] + let current: string[] = [] + const definitionPattern = /^\s*(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|enum|def|func|fn|struct|trait|impl)\b|^\s*(?:export\s+)?(?:const|let|var)\s+\w+\s*=/ + + for (const line of lines) { + if (definitionPattern.test(line) && current.length > 0) { + blocks.push(current.join('\n')) + current = [] + } + current.push(line) + } + if (current.length > 0) blocks.push(current.join('\n')) + + const chunks: string[] = [] + let buffer: string[] = [] + let bufferTokens = 0 + const flush = () => { + if (buffer.length === 0) return + chunks.push(buffer.join('\n\n')) + buffer = [] + bufferTokens = 0 + } + + for (const block of blocks) { + const blockTokens = estimateTokens(block) + if (blockTokens > opts.maxTokens) { + flush() + chunks.push(...splitOversizedLines(block, opts).map(chunk => chunk.content)) + continue + } + if (buffer.length > 0 && bufferTokens + blockTokens > opts.maxTokens) { + flush() + } + buffer.push(block) + bufferTokens += blockTokens + } + flush() + + return toTextChunks(chunks) +} + +function splitOversizedTable(content: string, opts: ChunkOptions): TextChunk[] { + if (estimateTokens(content) <= opts.maxTokens) return toTextChunks([content]) + + const lines = content.split('\n').filter(line => line.trim().length > 0) + const separatorIndex = lines.findIndex(line => /^-+$/.test(line.trim())) + const headerEnd = separatorIndex >= 0 ? separatorIndex + 1 : Math.min(2, lines.length) + const header = lines.slice(0, headerEnd) + const rows = lines.slice(headerEnd) + if (rows.length === 0) return splitOversizedLines(content, opts) + + const chunks: string[] = [] + let currentRows: string[] = [] + + const build = (batch: string[]) => [...header, ...batch].join('\n') + const flush = () => { + if (currentRows.length === 0) return + chunks.push(build(currentRows)) + currentRows = [] + } + + for (const row of rows) { + const candidate = build([...currentRows, row]) + if (currentRows.length > 0 && estimateTokens(candidate) > opts.maxTokens) { + flush() + } + if (currentRows.length === 0 && estimateTokens(build([row])) > opts.maxTokens) { + const availableTokens = Math.max(1, opts.maxTokens - estimateTokens(header.join('\n'))) + for (const part of splitTextByTokenBudget(row, availableTokens)) { + chunks.push(build([part])) + } + continue + } + currentRows.push(row) + } + flush() + + return toTextChunks(chunks) +} diff --git a/packages/core/src/ingest/chunker.ts b/packages/core/src/ingest/chunker.ts index 0ee0bce..1b82c55 100644 --- a/packages/core/src/ingest/chunker.ts +++ b/packages/core/src/ingest/chunker.ts @@ -368,6 +368,41 @@ function extractHeadingMap(text: string, sentences: string[]): Map s.trim()) - .filter(s => s.length > 0) + const sentences = splitIntoSentences(text) if (sentences.length === 0) return [] diff --git a/packages/core/src/ingest/document-store.ts b/packages/core/src/ingest/document-store.ts index 27a4d11..8b576de 100644 --- a/packages/core/src/ingest/document-store.ts +++ b/packages/core/src/ingest/document-store.ts @@ -99,6 +99,26 @@ export class DocumentStore { return { id, status: 'pending' } } + updateDocumentForReindex(documentId: string, input: CreateDocumentInput): void { + this.db.run( + `UPDATE documents + SET title = ?, source_type = ?, source_path = ?, file_type = ?, file_size_bytes = ?, + connector_id = ?, status = 'pending', error_message = NULL, updated_at = ? + WHERE id = ? AND workspace_id = ? AND deleted_at IS NULL`, + [ + input.title, + input.sourceType, + input.sourcePath, + input.fileType || null, + input.fileSizeBytes || null, + input.connectorId || null, + new Date().toISOString(), + documentId, + this.workspaceId, + ] + ) + } + getDocument(id: string): DocumentRow | undefined { return this.db.get( 'SELECT * FROM documents WHERE id = ? AND workspace_id = ? AND deleted_at IS NULL', @@ -121,6 +141,8 @@ export class DocumentStore { } async storeChunks(documentId: string, chunks: StoredChunk[]): Promise { + const existing = this.getDocument(documentId) + const previousChunkCount = existing?.chunk_count ?? 0 const vectorDocs = chunks.map((chunk, i) => ({ id: `${documentId}_chunk_${i}`, content: chunk.content, @@ -141,12 +163,20 @@ export class DocumentStore { // Step 1: Upsert vectors await this.vectorDb.upsert(COLLECTION, vectorDocs) + if (previousChunkCount > chunks.length) { + const staleChunkIds: string[] = [] + for (let i = chunks.length; i < previousChunkCount; i++) { + staleChunkIds.push(`${documentId}_chunk_${i}`) + } + await this.vectorDb.delete(COLLECTION, staleChunkIds).catch(() => {}) + } // Step 2: Insert into FTS5 index -- if this fails, clean up vectors. // When ftsAugment is present, index `content + ftsAugment` to boost lexical // recall while keeping the vector embedding and generator-facing content // unchanged. try { + this.deleteFtsRowsForDocument(documentId) for (const chunk of chunks) { const chunkId = `${documentId}_chunk_${chunk.position}` const ftsContent = chunk.ftsAugment @@ -185,31 +215,19 @@ export class DocumentStore { filter: { workspace_id: this.workspaceId }, minScore, }) - return results.map(r => { - const docId = r.metadata.document_id as string - const doc = this.getDocument(docId) - if (!doc) { - console.warn(`[searchChunks] Orphaned chunk: ${r.id}, document ${docId} not found`) - } - return { - chunkId: r.id, - content: r.content, - score: r.score, - documentId: docId, - chunkType: r.metadata.chunk_type as string, - headingHierarchy: JSON.parse((r.metadata.heading_hierarchy as string) || '[]'), - sourcePath: doc?.source_path || '', - sourceType: doc?.source_type || '', - contextualPrefix: (r.metadata.contextual_prefix as string) || undefined, - parentSection: (r.metadata.parent_section as string) || undefined, - } - }) + return results + .map(result => this.toSearchResult(result, result.score)) + .filter((result): result is SearchResult => result !== null) } private toSearchResult(record: VectorRecord, score: number): SearchResult | null { const docId = record.metadata.document_id as string const doc = this.getDocument(docId) - if (!doc) return null + if (!doc) { + console.warn(`[DocumentStore] Orphaned chunk: ${record.id}, document ${docId} not found`) + return null + } + if (doc.status !== 'indexed') return null return { chunkId: record.id, @@ -304,8 +322,7 @@ export class DocumentStore { [now, now, documentId, this.workspaceId] ) // Clean FTS index - const escapedId = documentId.replace(/%/g, '\\%').replace(/_/g, '\\_') - this.db.run("DELETE FROM chunks_fts WHERE chunk_id LIKE ? ESCAPE '\\'", [escapedId + '_%']) + this.deleteFtsRowsForDocument(documentId) // Also remove vectors (they can't be soft-deleted in LanceDB) await this.vectorDb.deleteByFilter(COLLECTION, { document_id: documentId }) } @@ -313,11 +330,17 @@ export class DocumentStore { async hardDeleteDocument(documentId: string): Promise { // Step 1: Delete vectors. If this throws, SQLite delete is never reached -- both stores remain consistent. await this.vectorDb.deleteByFilter(COLLECTION, { document_id: documentId }) + this.deleteFtsRowsForDocument(documentId) // Step 2: Delete SQLite row permanently. this.db.run('DELETE FROM documents WHERE id = ? AND workspace_id = ?', [documentId, this.workspaceId]) } + private deleteFtsRowsForDocument(documentId: string): void { + const escapedId = documentId.replace(/%/g, '\\%').replace(/_/g, '\\_') + this.db.run("DELETE FROM chunks_fts WHERE chunk_id LIKE ? ESCAPE '\\'", [escapedId + '_chunk_%']) + } + /** * Restore a soft-deleted document. Resets status to 'pending'. * The document will need to be re-indexed (opendocuments index) to regenerate embeddings. diff --git a/packages/core/src/ingest/pipeline.ts b/packages/core/src/ingest/pipeline.ts index 1d8fd0c..2adb692 100644 --- a/packages/core/src/ingest/pipeline.ts +++ b/packages/core/src/ingest/pipeline.ts @@ -91,31 +91,38 @@ export class IngestPipeline { async ingest( input: IngestInput, - options: { contextualRetrieval?: boolean; chunkAugmentation?: boolean } = {} + options: { contextualRetrieval?: boolean; chunkAugmentation?: boolean; force?: boolean } = {} ): Promise { const { store, registry, eventBus, middleware } = this.opts const contentHash = sha256(input.content) + const fileType = input.fileType ?? extname(input.sourcePath) + let documentId: string // Check for existing document by sourcePath const existing = store.getDocumentBySourcePath(input.sourcePath) if (existing) { - if (!store.hasContentChanged(existing.id, contentHash)) { + if (!options.force && !store.hasContentChanged(existing.id, contentHash)) { return { documentId: existing.id, chunks: existing.chunk_count ?? 0, status: 'skipped' } } - // Content changed -- delete old document first - await store.hardDeleteDocument(existing.id) + documentId = existing.id + store.updateDocumentForReindex(documentId, { + title: input.title, + sourceType: input.sourceType, + sourcePath: input.sourcePath, + fileType, + connectorId: input.connectorId, + }) + } else { + const created = store.createDocument({ + title: input.title, + sourceType: input.sourceType, + sourcePath: input.sourcePath, + fileType, + connectorId: input.connectorId, + }) + documentId = created.id } - // Create new document record - const fileType = input.fileType ?? extname(input.sourcePath) - const { id: documentId } = store.createDocument({ - title: input.title, - sourceType: input.sourceType, - sourcePath: input.sourcePath, - fileType, - connectorId: input.connectorId, - }) - try { // Build RawDocument for parser const rawDoc: RawDocument = { diff --git a/packages/core/src/parsers/archive.ts b/packages/core/src/parsers/archive.ts index fbd5224..1f232fc 100644 --- a/packages/core/src/parsers/archive.ts +++ b/packages/core/src/parsers/archive.ts @@ -1,7 +1,27 @@ import type { ParserPlugin, RawDocument, ParsedChunk, PluginContext, HealthStatus } from '../plugin/interfaces.js' +import { constants as zlibConstants, inflateRawSync } from 'node:zlib' -// ZIP parsing requires a proper library (adm-zip, unzipper, etc.) -// For now, yield a placeholder indicating archive support is limited +const MAX_ARCHIVE_ENTRIES = 1_000 +const MAX_ENTRY_UNCOMPRESSED_BYTES = 16 * 1024 * 1024 +const MAX_TOTAL_UNCOMPRESSED_BYTES = 64 * 1024 * 1024 +const MAX_COMPRESSION_RATIO = 200 + +function assertSafeEntry( + entryName: string, + compressedSize: number, + uncompressedSize: number, + totalUncompressedBytes: number +): void { + if (uncompressedSize > MAX_ENTRY_UNCOMPRESSED_BYTES) { + throw new Error(`ZIP entry exceeds the 16MB extraction limit: ${entryName}`) + } + if (totalUncompressedBytes + uncompressedSize > MAX_TOTAL_UNCOMPRESSED_BYTES) { + throw new Error('ZIP archive exceeds the 64MB total extraction limit') + } + if (compressedSize > 0 && uncompressedSize / compressedSize > MAX_COMPRESSION_RATIO) { + throw new Error(`ZIP entry has an unsafe compression ratio: ${entryName}`) + } +} export class ArchiveParser implements ParserPlugin { name = '@opendocuments/parser-archive' @@ -15,13 +35,98 @@ export class ArchiveParser implements ParserPlugin { async healthCheck(): Promise { return { healthy: true } } async *parse(raw: RawDocument): AsyncIterable { - // ZIP parsing requires a proper library (adm-zip, unzipper, etc.) - // For now, yield a placeholder indicating archive support is limited - yield { - content: `[ZIP Archive] ${raw.title}\nFull ZIP extraction requires the 'adm-zip' package. Install it and re-index to extract contents.\nTo add ZIP support: npm install adm-zip`, - chunkType: 'semantic', - headingHierarchy: ['Archive: ' + raw.title], - metadata: { type: 'archive-placeholder', needsExtraction: true }, + const buffer = Buffer.isBuffer(raw.content) + ? raw.content + : Buffer.from(raw.content) + + let offset = 0 + let yielded = false + let entryCount = 0 + let totalUncompressedBytes = 0 + + while (offset + 30 <= buffer.length) { + const signature = buffer.readUInt32LE(offset) + if (signature !== 0x04034b50) break + + const flags = buffer.readUInt16LE(offset + 6) + const method = buffer.readUInt16LE(offset + 8) + const compressedSize = buffer.readUInt32LE(offset + 18) + const uncompressedSize = buffer.readUInt32LE(offset + 22) + const fileNameLength = buffer.readUInt16LE(offset + 26) + const extraLength = buffer.readUInt16LE(offset + 28) + const nameStart = offset + 30 + const dataStart = nameStart + fileNameLength + extraLength + + entryCount++ + if (entryCount > MAX_ARCHIVE_ENTRIES) { + throw new Error(`ZIP archive exceeds the ${MAX_ARCHIVE_ENTRIES}-entry limit`) + } + + if (flags & 0x08) { + break + } + if (dataStart + compressedSize > buffer.length) { + break + } + + const entryName = buffer.slice(nameStart, nameStart + fileNameLength).toString('utf-8') + const data = buffer.slice(dataStart, dataStart + compressedSize) + offset = dataStart + compressedSize + + if (!entryName || entryName.endsWith('/')) continue + + assertSafeEntry(entryName, compressedSize, uncompressedSize, totalUncompressedBytes) + + let content: Buffer + if (method === 0) { + content = data + } else if (method === 8) { + const remainingBytes = MAX_TOTAL_UNCOMPRESSED_BYTES - totalUncompressedBytes + content = inflateRawSync(data, { + finishFlush: zlibConstants.Z_SYNC_FLUSH, + maxOutputLength: Math.min(MAX_ENTRY_UNCOMPRESSED_BYTES, remainingBytes), + }) + } else { + yield { + content: `[ZIP Entry Skipped] ${entryName}\nUnsupported compression method: ${method}`, + chunkType: 'semantic', + headingHierarchy: ['Archive: ' + raw.title, entryName], + metadata: { type: 'archive-entry', entryName, skipped: true, compressionMethod: method }, + } + yielded = true + continue + } + + if (content.length > MAX_ENTRY_UNCOMPRESSED_BYTES) { + throw new Error(`ZIP entry exceeds the 16MB extraction limit: ${entryName}`) + } + totalUncompressedBytes += content.length + if (totalUncompressedBytes > MAX_TOTAL_UNCOMPRESSED_BYTES) { + throw new Error('ZIP archive exceeds the 64MB total extraction limit') + } + + yield { + content: content.toString('utf-8'), + chunkType: 'semantic', + headingHierarchy: ['Archive: ' + raw.title, entryName], + metadata: { + type: 'archive-entry', + entryName, + compressedSize, + uncompressedSize, + needsExtraction: false, + }, + } + yielded = true + } + + if (!yielded) { + yield { + content: `[ZIP Archive] ${raw.title}\nNo extractable ZIP entries were found.`, + chunkType: 'semantic', + headingHierarchy: ['Archive: ' + raw.title], + metadata: { type: 'archive-empty', needsExtraction: false }, + } } } } diff --git a/packages/core/src/rag/generator.ts b/packages/core/src/rag/generator.ts index 52866c5..ed16cac 100644 --- a/packages/core/src/rag/generator.ts +++ b/packages/core/src/rag/generator.ts @@ -115,6 +115,6 @@ export async function* generateAnswer( yield* model.generate(prompt, { temperature: 0.3, - maxTokens: 4096, + maxTokens: 1024, }) } diff --git a/packages/core/src/rag/router.ts b/packages/core/src/rag/router.ts index 19d45cc..3ecbc71 100644 --- a/packages/core/src/rag/router.ts +++ b/packages/core/src/rag/router.ts @@ -1,6 +1,6 @@ export type QueryRoute = 'rag' | 'direct' | 'web_only' | 'rag_web' -// TODO: Make patterns configurable via middleware (before:query hook) or config +// Keep these deterministic so direct utility queries do not consume model calls. const GREETING_PATTERNS = [ /^(hi|hello|hey|howdy|yo|sup|greetings)\b/i, /^(안녕|안녕하세요|반갑습니다|하이|헬로)/, diff --git a/packages/core/src/storage/migrations/007_add_api_key_allowed_ips.sql b/packages/core/src/storage/migrations/007_add_api_key_allowed_ips.sql new file mode 100644 index 0000000..811d654 --- /dev/null +++ b/packages/core/src/storage/migrations/007_add_api_key_allowed_ips.sql @@ -0,0 +1 @@ +ALTER TABLE api_keys ADD COLUMN allowed_ips TEXT DEFAULT '[]'; diff --git a/packages/core/src/telemetry/collector.ts b/packages/core/src/telemetry/collector.ts index cef6634..311625c 100644 --- a/packages/core/src/telemetry/collector.ts +++ b/packages/core/src/telemetry/collector.ts @@ -1,4 +1,5 @@ import { platform, arch } from 'node:os' +import { fetchWithTimeout } from '../utils/fetch.js' export interface TelemetryEvent { event: string @@ -9,9 +10,11 @@ export class TelemetryCollector { private enabled: boolean private queue: TelemetryEvent[] = [] private sessionId: string + private endpoint?: string - constructor(config?: { enabled?: boolean }) { - this.enabled = config?.enabled ?? false + constructor(config?: { enabled?: boolean; endpoint?: string }) { + this.endpoint = config?.endpoint + this.enabled = Boolean(config?.enabled && this.endpoint) this.sessionId = Math.random().toString(36).substring(2) } @@ -30,16 +33,19 @@ export class TelemetryCollector { } async flush(): Promise { - if (!this.enabled || this.queue.length === 0) return + if (!this.enabled || !this.endpoint || this.queue.length === 0) return - // In a real implementation, this would send to a telemetry endpoint - // For now, just clear the queue (opt-in but no endpoint configured) - // eslint-disable-next-line @typescript-eslint/no-unused-vars const events = [...this.queue] + const res = await fetchWithTimeout(this.endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ events }), + }, 5000) + + if (!res.ok) { + throw new Error(`Telemetry endpoint returned HTTP ${res.status}`) + } this.queue = [] - - // TODO: Send to telemetry endpoint when configured - // try { await fetchWithTimeout('https://telemetry.opendocuments.dev/events', { method: 'POST', body: JSON.stringify(events) }) } catch {} } isEnabled(): boolean { diff --git a/packages/core/src/utils/file-discovery.ts b/packages/core/src/utils/file-discovery.ts index 9381890..2968540 100644 --- a/packages/core/src/utils/file-discovery.ts +++ b/packages/core/src/utils/file-discovery.ts @@ -1,7 +1,7 @@ import { readdirSync, statSync } from 'node:fs' import { join, extname } from 'node:path' -const SUPPORTED_EXTENSIONS = new Set([ +export const SUPPORTED_EXTENSIONS = new Set([ '.md', '.mdx', '.txt', // built-in '.json', '.yaml', '.yml', '.toml', // structured data parsers '.zip', // archive parser diff --git a/packages/core/src/utils/file-watcher.ts b/packages/core/src/utils/file-watcher.ts index 4c12d53..28209af 100644 --- a/packages/core/src/utils/file-watcher.ts +++ b/packages/core/src/utils/file-watcher.ts @@ -6,51 +6,91 @@ export interface FileChange { type: 'added' | 'modified' | 'deleted' } +export interface FileWatcherOptions { + pollIntervalMs?: number + ignoreInitial?: boolean + stableMs?: number +} + +interface FileState { + mtimeMs: number + size: number +} + export class FileWatcher { - private fileHashes = new Map() // path -> mtime + private fileHashes = new Map() private interval: ReturnType | null = null + private isScanning = false + private readonly pollIntervalMs: number + private readonly ignoreInitial: boolean + private readonly stableMs: number constructor( private dir: string, private extensions: Set, - private onChange: (changes: FileChange[]) => void, - private pollIntervalMs = 3000 - ) {} + private onChange: (changes: FileChange[]) => void | Promise, + options: number | FileWatcherOptions = 3000 + ) { + const opts = typeof options === 'number' ? { pollIntervalMs: options } : options + this.pollIntervalMs = opts.pollIntervalMs ?? 3000 + this.ignoreInitial = opts.ignoreInitial ?? false + this.stableMs = opts.stableMs ?? 500 + } start(): void { - // Initial scan - this.scan() - this.interval = setInterval(() => this.scan(), this.pollIntervalMs) + void this.scan({ emitInitial: !this.ignoreInitial }) + this.interval = setInterval(() => { void this.scan() }, this.pollIntervalMs) } stop(): void { if (this.interval) { clearInterval(this.interval); this.interval = null } } - private scan(): void { - const currentFiles = new Map() - this.walkDir(this.dir, currentFiles) + private async scan(opts: { emitInitial?: boolean } = {}): Promise { + if (this.isScanning) return + this.isScanning = true + try { + const currentFiles = new Map() + this.walkDir(this.dir, currentFiles) - const changes: FileChange[] = [] + const changes: FileChange[] = [] + const nextHashes = new Map() - // Check for added/modified - for (const [path, mtime] of currentFiles) { - const prev = this.fileHashes.get(path) - if (!prev) changes.push({ path, type: 'added' }) - else if (prev !== mtime) changes.push({ path, type: 'modified' }) - } + // Check for added/modified + for (const [path, state] of currentFiles) { + const prev = this.fileHashes.get(path) + const isStable = Date.now() - state.mtimeMs >= this.stableMs + if (!prev) { + if (opts.emitInitial === false) { + nextHashes.set(path, state) + continue + } + if (!isStable) continue + changes.push({ path, type: 'added' }) + } else if (prev.mtimeMs !== state.mtimeMs || prev.size !== state.size) { + if (!isStable) { + nextHashes.set(path, prev) + continue + } + changes.push({ path, type: 'modified' }) + } + nextHashes.set(path, state) + } - // Check for deleted - for (const [path] of this.fileHashes) { - if (!currentFiles.has(path)) changes.push({ path, type: 'deleted' }) - } + // Check for deleted + for (const [path] of this.fileHashes) { + if (!currentFiles.has(path)) changes.push({ path, type: 'deleted' }) + } - this.fileHashes = currentFiles + this.fileHashes = nextHashes - if (changes.length > 0) this.onChange(changes) + if (changes.length > 0) await this.onChange(changes) + } finally { + this.isScanning = false + } } - private walkDir(dir: string, files: Map): void { + private walkDir(dir: string, files: Map): void { try { const entries = readdirSync(dir, { withFileTypes: true }) for (const entry of entries) { @@ -58,7 +98,10 @@ export class FileWatcher { const fullPath = join(dir, entry.name) if (entry.isDirectory()) this.walkDir(fullPath, files) else if (this.extensions.has(extname(entry.name))) { - try { files.set(fullPath, statSync(fullPath).mtimeMs) } catch (err) { + try { + const stat = statSync(fullPath) + files.set(fullPath, { mtimeMs: stat.mtimeMs, size: stat.size }) + } catch (err) { // Log but don't crash -- file may be temporarily unavailable console.error(`FileWatcher error: ${(err as Error).message}`) } diff --git a/packages/core/tests/config/env-loader.test.ts b/packages/core/tests/config/env-loader.test.ts index e69e547..6d5ac85 100644 --- a/packages/core/tests/config/env-loader.test.ts +++ b/packages/core/tests/config/env-loader.test.ts @@ -294,6 +294,12 @@ describe('buildConfigFromEnv', () => { expect(result?.telemetry?.enabled).toBe(true) }) + it('maps OPENDOCUMENTS_TELEMETRY_ENDPOINT', () => { + setEnv({ OPENDOCUMENTS_TELEMETRY_ENDPOINT: 'https://telemetry.example/events' }) + const result = buildConfigFromEnv() + expect(result?.telemetry?.endpoint).toBe('https://telemetry.example/events') + }) + it('maps OPENDOCUMENTS_TELEMETRY_ENABLED = false', () => { setEnv({ OPENDOCUMENTS_TELEMETRY_ENABLED: 'false' }) const result = buildConfigFromEnv() @@ -312,6 +318,7 @@ describe('buildConfigFromEnv', () => { OPENDOCUMENTS_RAG_PROFILE: 'precise', OPENDOCUMENTS_STORAGE_DB: 'postgres', OPENDOCUMENTS_TELEMETRY_ENABLED: 'true', + OPENDOCUMENTS_TELEMETRY_ENDPOINT: 'https://telemetry.example/events', }) const result = buildConfigFromEnv() expect(result).not.toBeNull() @@ -323,6 +330,7 @@ describe('buildConfigFromEnv', () => { expect(result?.rag?.profile).toBe('precise') expect(result?.storage?.db).toBe('postgres') expect(result?.telemetry?.enabled).toBe(true) + expect(result?.telemetry?.endpoint).toBe('https://telemetry.example/events') }) }) }) diff --git a/packages/core/tests/connector/manager.test.ts b/packages/core/tests/connector/manager.test.ts index 6747ec4..f0807d2 100644 --- a/packages/core/tests/connector/manager.test.ts +++ b/packages/core/tests/connector/manager.test.ts @@ -94,6 +94,60 @@ describe('ConnectorManager', () => { expect(list[0].name).toBe('@opendocuments/connector-mock') }) + it('stores connector config and updates existing named connector', () => { + const connector = createMockConnector([]) + const firstId = manager.registerConnector(connector, { + name: 'github', + config: { type: 'github', repo: 'owner/first', branch: 'main', token: 'secret-token' }, + }) + const secondId = manager.registerConnector(connector, { + name: 'github', + config: { type: 'github', repo: 'owner/second', branch: 'develop', token: 'secret-token' }, + }) + + expect(secondId).toBe(firstId) + + const rows = db.all('SELECT * FROM connectors WHERE name = ?', ['github']) + expect(rows).toHaveLength(1) + expect(JSON.parse(rows[0].config)).toMatchObject({ + type: 'github', + repo: 'owner/second', + branch: 'develop', + token: 'secret-token', + }) + + const list = manager.listConnectors() + expect(list[0]).toMatchObject({ + name: 'github', + type: '@opendocuments/connector-mock', + repo: 'owner/second', + }) + }) + + it('starts periodic sync when registered with autoSync enabled', async () => { + vi.useFakeTimers() + try { + const connector = createMockConnector([]) + const syncSpy = vi.spyOn(manager, 'syncConnector').mockResolvedValue({ + connectorName: '@opendocuments/connector-mock', + documentsDiscovered: 0, + documentsIndexed: 0, + documentsSkipped: 0, + errors: [], + }) + + manager.registerConnector(connector, { + syncIntervalSeconds: 60, + autoSync: true, + }) + + await vi.advanceTimersByTimeAsync(60000) + expect(syncSpy).toHaveBeenCalledWith('@opendocuments/connector-mock') + } finally { + vi.useRealTimers() + } + }) + it('syncs a connector and indexes discovered documents', { timeout: 15000 }, async () => { const connector = createMockConnector([ { id: '1', title: 'readme.md', path: '/repo/README.md', content: '# Hello\n\nWorld' }, diff --git a/packages/core/tests/ingest/chunk-strategies.test.ts b/packages/core/tests/ingest/chunk-strategies.test.ts index 5898c93..40c3aeb 100644 --- a/packages/core/tests/ingest/chunk-strategies.test.ts +++ b/packages/core/tests/ingest/chunk-strategies.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest' -import { selectChunkStrategy, type ChunkStrategy } from '../../src/ingest/chunk-strategies.js' +import { dispatchChunk, selectChunkStrategy, type ChunkStrategy } from '../../src/ingest/chunk-strategies.js' describe('selectChunkStrategy', () => { const cases: Array<{ fileType: string; chunkType: string; expected: ChunkStrategy }> = [ @@ -16,6 +16,7 @@ describe('selectChunkStrategy', () => { { fileType: '.yml', chunkType: 'semantic', expected: 'data' }, { fileType: '.toml', chunkType: 'semantic', expected: 'data' }, { fileType: '.txt', chunkType: 'semantic', expected: 'prose' }, + { fileType: '.pptx', chunkType: 'slide', expected: 'slide' }, { fileType: '', chunkType: 'semantic', expected: 'prose' }, { fileType: '', chunkType: 'api-endpoint', expected: 'api' }, { fileType: '.md', chunkType: 'api-endpoint', expected: 'api' }, // chunkType wins for api @@ -27,3 +28,82 @@ describe('selectChunkStrategy', () => { }) } }) + +describe('dispatchChunk', () => { + it('splits oversized code chunks at symbol boundaries instead of embedding whole files', async () => { + const code = Array.from({ length: 18 }, (_, i) => [ + `export function handler${i}() {`, + ` const value = '${'x'.repeat(120)}'`, + ' return value', + '}', + ].join('\n')).join('\n\n') + + const chunks = await dispatchChunk(code, { + fileType: '.ts', + chunkType: 'code-ast', + embed: null, + }) + + expect(chunks.length).toBeGreaterThan(1) + for (const chunk of chunks) { + expect(chunk.tokenCount).toBeLessThanOrEqual(400) + } + }) + + it('splits a single oversized code line within the token budget', async () => { + const code = `const payload = "${'very long token group '.repeat(900)}"` + + const chunks = await dispatchChunk(code, { + fileType: '.ts', + chunkType: 'code-ast', + embed: null, + }) + + expect(chunks.length).toBeGreaterThan(1) + for (const chunk of chunks) { + expect(chunk.tokenCount).toBeLessThanOrEqual(400) + } + }) + + it('splits oversized table chunks by rows while repeating the header', async () => { + const rows = [ + 'Sheet: Sales', + 'Name | Notes', + '---', + ...Array.from({ length: 60 }, (_, i) => `Customer ${i} | ${'large cell '.repeat(20)}`), + ] + + const chunks = await dispatchChunk(rows.join('\n'), { + fileType: '.xlsx', + chunkType: 'table', + embed: null, + }) + + expect(chunks.length).toBeGreaterThan(1) + expect(chunks.every(chunk => chunk.content.includes('Name | Notes'))).toBe(true) + for (const chunk of chunks) { + expect(chunk.tokenCount).toBeLessThanOrEqual(800) + } + }) + + it('splits a single oversized table row while repeating the header', async () => { + const table = [ + 'Sheet: Sales', + 'Name | Notes', + '---', + `Customer 1 | ${'very wide cell '.repeat(800)}`, + ].join('\n') + + const chunks = await dispatchChunk(table, { + fileType: '.xlsx', + chunkType: 'table', + embed: null, + }) + + expect(chunks.length).toBeGreaterThan(1) + expect(chunks.every(chunk => chunk.content.includes('Name | Notes'))).toBe(true) + for (const chunk of chunks) { + expect(chunk.tokenCount).toBeLessThanOrEqual(800) + } + }) +}) diff --git a/packages/core/tests/ingest/chunker.test.ts b/packages/core/tests/ingest/chunker.test.ts index 262ee2a..5f3480c 100644 --- a/packages/core/tests/ingest/chunker.test.ts +++ b/packages/core/tests/ingest/chunker.test.ts @@ -167,6 +167,44 @@ describe('semanticChunkText', () => { } }) + it('recognizes Korean sentence endings as semantic boundaries', async () => { + const text = '첫 번째 주제입니다.같은 주제의 보충 설명입니다.완전히 다른 결제 정책입니다.환불 규정도 여기에 포함됩니다.' + const embed = mockEmbedder([ + [1, 0], + [0.95, 0.05], + [0, 1], + [0.05, 0.95], + ]) + + const chunks = await semanticChunkText(text, defaultOpts, embed, 0.5) + + expect(chunks).toHaveLength(2) + expect(chunks[0].content).toContain('첫 번째 주제입니다.') + expect(chunks[0].content).toContain('같은 주제의 보충 설명입니다.') + expect(chunks[1].content).toContain('완전히 다른 결제 정책입니다.') + expect(chunks[1].content).toContain('환불 규정도 여기에 포함됩니다.') + }) + + it('does not split inside common ASCII dotted terms', async () => { + const embeddedTexts: string[] = [] + const embed = async (texts: string[]): Promise => { + embeddedTexts.push(...texts) + return { dense: texts.map(() => [1, 0]) } + } + + await semanticChunkText( + 'Node.js v20.1.0 is required for README.md parsing. Next sentence mentions e.g. examples.', + defaultOpts, + embed, + 0.5, + ) + + expect(embeddedTexts).toEqual([ + 'Node.js v20.1.0 is required for README.md parsing.', + 'Next sentence mentions e.g. examples.', + ]) + }) + it('keeps semantically similar sentences together and splits on dissimilar ones', async () => { const text = 'Dogs are great pets. Cats are also wonderful animals. The stock market crashed today. Investors are worried about recession.' // First two sentences: similar direction, last two: different direction diff --git a/packages/core/tests/ingest/document-store.test.ts b/packages/core/tests/ingest/document-store.test.ts index c32d8ae..ee8817f 100644 --- a/packages/core/tests/ingest/document-store.test.ts +++ b/packages/core/tests/ingest/document-store.test.ts @@ -114,6 +114,20 @@ describe('DocumentStore', () => { expect(deleted).toHaveLength(0) }) + it('hard deletes FTS rows for the document', async () => { + const doc = store.createDocument({ + title: 'test.md', sourceType: 'local', sourcePath: '/docs/test.md', fileType: '.md', + }) + await store.storeChunks(doc.id, [ + { content: 'Unique searchable phrase', embedding: [1, 0, 0], chunkType: 'semantic', position: 0, tokenCount: 3, headingHierarchy: [] }, + ]) + + await store.hardDeleteDocument(doc.id) + + const rows = db.all('SELECT chunk_id FROM chunks_fts WHERE chunk_id = ?', [`${doc.id}_chunk_0`]) + expect(rows).toHaveLength(0) + }) + it('checks content hash for change detection', () => { const doc = store.createDocument({ title: 'test.md', sourceType: 'local', sourcePath: '/docs/test.md', fileType: '.md', diff --git a/packages/core/tests/ingest/pipeline.test.ts b/packages/core/tests/ingest/pipeline.test.ts index db91d2d..fb7de63 100644 --- a/packages/core/tests/ingest/pipeline.test.ts +++ b/packages/core/tests/ingest/pipeline.test.ts @@ -21,6 +21,7 @@ describe('IngestPipeline', () => { let vectorDb: VectorDB let tempDir: string let pipeline: IngestPipeline + let store: DocumentStore beforeEach(async () => { db = createSQLiteDB(':memory:') @@ -36,7 +37,7 @@ describe('IngestPipeline', () => { await registry.register(createMockEmbedder(), ctx) await registry.register(new MarkdownParser(), ctx) - const store = new DocumentStore(db, vectorDb, 'ws-1') + store = new DocumentStore(db, vectorDb, 'ws-1') await store.initialize(3) pipeline = new IngestPipeline({ store, registry, eventBus, middleware, embeddingDimensions: 3 }) @@ -105,6 +106,82 @@ describe('IngestPipeline', () => { expect(second.status).toBe('skipped') }) + it('reindexes changed documents without replacing the document row', async () => { + const first = await pipeline.ingest({ + title: 'test.md', + content: '# Test\n\nOriginal content.', + sourceType: 'local', + sourcePath: '/docs/test.md', + fileType: '.md', + }) + expect(first.status).toBe('indexed') + + db.run("INSERT INTO tags (id, workspace_id, name) VALUES ('tag-1', 'ws-1', 'important')") + db.run('INSERT INTO document_tags (document_id, tag_id) VALUES (?, ?)', [first.documentId, 'tag-1']) + + const second = await pipeline.ingest({ + title: 'renamed.md', + content: '# Test\n\nUpdated content.', + sourceType: 'local', + sourcePath: '/docs/test.md', + fileType: '.md', + }) + + expect(second.status).toBe('indexed') + expect(second.documentId).toBe(first.documentId) + const docs = db.all('SELECT id, title FROM documents WHERE source_path = ? AND deleted_at IS NULL', ['/docs/test.md']) + expect(docs).toHaveLength(1) + expect(docs[0].title).toBe('renamed.md') + const tagRows = db.all('SELECT * FROM document_tags WHERE document_id = ?', [first.documentId]) + expect(tagRows).toHaveLength(1) + }) + + it('does not return stale chunks after a failed reindex', async () => { + const first = await pipeline.ingest({ + title: 'test.md', + content: '# Test\n\nOriginal searchable content.', + sourceType: 'local', + sourcePath: '/docs/test.md', + fileType: '.md', + }) + expect(first.status).toBe('indexed') + + const failed = await pipeline.ingest({ + title: 'test.unsupported', + content: 'Changed content that cannot be parsed.', + sourceType: 'local', + sourcePath: '/docs/test.md', + fileType: '.unsupported', + }) + expect(failed.status).toBe('error') + expect(store.getDocument(first.documentId)?.status).toBe('error') + + await expect(store.searchFTS('Original', 10)).resolves.toEqual([]) + await expect(store.searchChunks([0.1, 0.2, 0.3], 10)).resolves.toEqual([]) + }) + + it('force reindexes unchanged documents without replacing the document row', async () => { + const content = '# Test\n\nSame content.' + const first = await pipeline.ingest({ + title: 'test.md', + content, + sourceType: 'local', + sourcePath: '/docs/test.md', + fileType: '.md', + }) + + const second = await pipeline.ingest({ + title: 'test.md', + content, + sourceType: 'local', + sourcePath: '/docs/test.md', + fileType: '.md', + }, { force: true }) + + expect(second.status).toBe('indexed') + expect(second.documentId).toBe(first.documentId) + }) + it('uses semanticChunkText when an embedding model is registered', async () => { // Spy on the embedder's embed calls to confirm sentence-level inputs were embedded. // The MarkdownParser produces a single 'semantic' ParsedChunk from the body; diff --git a/packages/core/tests/parsers/archive.test.ts b/packages/core/tests/parsers/archive.test.ts new file mode 100644 index 0000000..e8caaa1 --- /dev/null +++ b/packages/core/tests/parsers/archive.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect } from 'vitest' +import { deflateRawSync } from 'node:zlib' +import { ArchiveParser } from '../../src/parsers/archive.js' + +function localZipEntry(name: string, content: string): Buffer { + const nameBuffer = Buffer.from(name) + const raw = Buffer.from(content) + const compressed = deflateRawSync(raw) + const header = Buffer.alloc(30) + header.writeUInt32LE(0x04034b50, 0) + header.writeUInt16LE(20, 4) + header.writeUInt16LE(0, 6) + header.writeUInt16LE(8, 8) + header.writeUInt16LE(0, 10) + header.writeUInt16LE(0, 12) + header.writeUInt32LE(0, 14) + header.writeUInt32LE(compressed.length, 18) + header.writeUInt32LE(raw.length, 22) + header.writeUInt16LE(nameBuffer.length, 26) + header.writeUInt16LE(0, 28) + return Buffer.concat([header, nameBuffer, compressed]) +} + +describe('ArchiveParser', () => { + it('extracts deflated text entries from ZIP archives', async () => { + const parser = new ArchiveParser() + const zip = Buffer.concat([ + localZipEntry('docs/readme.md', '# Readme\n\nHello from zip.'), + localZipEntry('docs/notes.txt', 'Plain notes.'), + ]) + + const chunks: any[] = [] + for await (const chunk of parser.parse({ + sourceId: 'archive', + title: 'docs.zip', + content: zip, + })) { + chunks.push(chunk) + } + + expect(chunks).toHaveLength(2) + expect(chunks[0].content).toContain('# Readme') + expect(chunks[0].headingHierarchy).toEqual(['Archive: docs.zip', 'docs/readme.md']) + expect(chunks[0].metadata.needsExtraction).toBe(false) + expect(chunks[1].content).toContain('Plain notes.') + }) + + it('rejects entries with zip-bomb compression ratios', async () => { + const parser = new ArchiveParser() + const zip = localZipEntry('oversized.txt', 'A'.repeat(2 * 1024 * 1024)) + + await expect(async () => { + for await (const _chunk of parser.parse({ + sourceId: 'archive', + title: 'unsafe.zip', + content: zip, + })) { + // Consume the parser so extraction is attempted. + } + }).rejects.toThrow('unsafe compression ratio') + }) +}) diff --git a/packages/core/tests/rag/generator.test.ts b/packages/core/tests/rag/generator.test.ts index 20d95be..a12dfd6 100644 --- a/packages/core/tests/rag/generator.test.ts +++ b/packages/core/tests/rag/generator.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from 'vitest' -import { buildPrompt, type GenerateInput } from '../../src/rag/generator.js' +import { buildPrompt, generateAnswer, type GenerateInput } from '../../src/rag/generator.js' +import type { ModelPlugin, GenerateOpts } from '../../src/plugin/interfaces.js' describe('buildPrompt', () => { const baseInput: GenerateInput = { @@ -46,4 +47,26 @@ describe('buildPrompt', () => { const prompt = buildPrompt(baseInput) expect(prompt).toContain('configuration') }) + + it('keeps answer generation token budget bounded for local models', async () => { + let seenOpts: GenerateOpts | undefined + const model: ModelPlugin = { + name: '@opendocuments/model-test', + type: 'model', + version: '0.0.0', + coreVersion: '^0.3.0', + capabilities: { llm: true }, + setup: async () => {}, + async *generate(_prompt: string, opts?: GenerateOpts) { + seenOpts = opts + yield 'ok' + }, + } + + for await (const _chunk of generateAnswer(model, baseInput)) { + // consume stream + } + + expect(seenOpts?.maxTokens).toBeLessThanOrEqual(1024) + }) }) diff --git a/packages/core/tests/telemetry/collector.test.ts b/packages/core/tests/telemetry/collector.test.ts new file mode 100644 index 0000000..94eb109 --- /dev/null +++ b/packages/core/tests/telemetry/collector.test.ts @@ -0,0 +1,35 @@ +import { TelemetryCollector } from '../../src/telemetry/collector.js' + +describe('TelemetryCollector', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('stays disabled when no endpoint is configured', () => { + const collector = new TelemetryCollector({ enabled: true }) + expect(collector.isEnabled()).toBe(false) + }) + + it('flushes queued events to the configured endpoint', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true, status: 200 }) + vi.stubGlobal('fetch', fetchMock) + + const collector = new TelemetryCollector({ + enabled: true, + endpoint: 'https://telemetry.example/events', + }) + collector.track('document:indexed', { chunks: 3 }) + await collector.flush() + + expect(fetchMock).toHaveBeenCalledWith( + 'https://telemetry.example/events', + expect.objectContaining({ + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }), + ) + const body = JSON.parse(fetchMock.mock.calls[0][1].body as string) + expect(body.events[0].event).toBe('document:indexed') + expect(body.events[0].properties.chunks).toBe(3) + }) +}) diff --git a/packages/core/tests/utils/file-watcher.test.ts b/packages/core/tests/utils/file-watcher.test.ts new file mode 100644 index 0000000..f7f4fd0 --- /dev/null +++ b/packages/core/tests/utils/file-watcher.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect, vi, afterEach } from 'vitest' +import { FileWatcher } from '../../src/utils/file-watcher.js' +import { mkdtempSync, rmSync, writeFileSync, unlinkSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' + +describe('FileWatcher', () => { + const tempDirs: string[] = [] + + afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } + }) + + function makeTempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'opendocuments-watch-test-')) + tempDirs.push(dir) + return dir + } + + it('can seed existing files without emitting initial added events', async () => { + const dir = makeTempDir() + writeFileSync(join(dir, 'a.md'), '# A') + const onChange = vi.fn() + + const watcher = new FileWatcher(dir, new Set(['.md']), onChange, { ignoreInitial: true }) + watcher.start() + watcher.stop() + + expect(onChange).not.toHaveBeenCalled() + }) + + it('emits deleted events after the initial state is seeded', async () => { + const dir = makeTempDir() + const file = join(dir, 'a.md') + writeFileSync(file, '# A') + const onChange = vi.fn() + + const watcher = new FileWatcher(dir, new Set(['.md']), onChange, { ignoreInitial: true }) + watcher.start() + unlinkSync(file) + await (watcher as any).scan() + watcher.stop() + + expect(onChange).toHaveBeenCalledWith([{ path: file, type: 'deleted' }]) + }) +}) diff --git a/packages/server/src/bootstrap.ts b/packages/server/src/bootstrap.ts index c315f00..558298e 100644 --- a/packages/server/src/bootstrap.ts +++ b/packages/server/src/bootstrap.ts @@ -53,17 +53,13 @@ const PROVIDER_MAP: Record = { anthropic: 'opendocuments-model-anthropic', google: 'opendocuments-model-google', grok: 'opendocuments-model-grok', - deepseek: 'opendocuments-model-deepseek', - mistral: 'opendocuments-model-mistral', - 'openai-compatible': 'opendocuments-model-openai-compatible', } const EMBEDDING_DIMENSIONS: Record = { ollama: 1024, openai: 1536, - google: 768, + google: 3072, grok: 1536, - mistral: 1024, default: 384, } @@ -347,6 +343,13 @@ export async function bootstrap(opts: BootstrapOptions = {}): Promise() const configuredConnectors: Array<{ plugin: ConnectorPlugin - config: { name?: string; syncIntervalSeconds?: number } + config: { name?: string; syncIntervalSeconds?: number; autoSync?: boolean; config?: Record } }> = [] const ensureWorkspaceExists = (workspaceId: string) => { @@ -672,41 +675,57 @@ export async function bootstrap(opts: BootstrapOptions = {}): Promise package mapping const CONNECTOR_PLUGINS_MAP: Record = { - github: 'opendocuments-connector-github', - notion: 'opendocuments-connector-notion', - 'web-crawler': 'opendocuments-connector-web-crawler', + github: '@opendocuments/connector-github', + notion: '@opendocuments/connector-notion', + 'web-crawler': '@opendocuments/connector-web-crawler', 'gdrive': '@opendocuments/connector-gdrive', 'google-drive': '@opendocuments/connector-gdrive', 's3': '@opendocuments/connector-s3', 'gcs': '@opendocuments/connector-s3', 'confluence': '@opendocuments/connector-confluence', - 'swagger': 'opendocuments-connector-swagger', - 'openapi': 'opendocuments-connector-swagger', + 'swagger': '@opendocuments/connector-swagger', + 'openapi': '@opendocuments/connector-swagger', + } + + const createConnector = async ( + connectorConfig: Record, + dataDirForConnector: string + ): Promise => { + const type = typeof connectorConfig.type === 'string' ? connectorConfig.type : '' + const packageName = CONNECTOR_PLUGINS_MAP[type] || type + if (!packageName) return null + + const mod = await import(packageName) + const ConnectorClass = mod.default + if (typeof ConnectorClass !== 'function') return null + + const connector = new ConnectorClass() as ConnectorPlugin + const connectorCtx: PluginContext = { + config: connectorConfig, + dataDir: dataDirForConnector, + log: pluginCtx.log, + } + await connector.setup(connectorCtx) + return connector } // Config-driven connector registration for (const connectorConfig of config.connectors) { - const packageName = CONNECTOR_PLUGINS_MAP[connectorConfig.type] - if (!packageName) continue - try { - const mod = await import(packageName) - const ConnectorClass = mod.default - if (typeof ConnectorClass !== 'function') continue - - const connector = new ConnectorClass() - - // Create a context with connector-specific config + const connectorRecord = connectorConfig as unknown as Record + const connector = await createConnector(connectorRecord, dataDir) + if (!connector) continue const connectorCtx: PluginContext = { - config: connectorConfig as unknown as Record, + config: connectorRecord, dataDir, log: pluginCtx.log, } - await registry.register(connector, connectorCtx) const registration = { name: connectorConfig.type, syncIntervalSeconds: (connectorConfig as any).syncInterval || 300, + autoSync: true, + config: connectorRecord, } configuredConnectors.push({ plugin: connector, config: registration }) for (const manager of connectorManagers.values()) { @@ -717,6 +736,42 @@ export async function bootstrap(opts: BootstrapOptions = {}): Promise( + `SELECT workspace_id, name, type, config, sync_interval_seconds + FROM connectors + WHERE deleted_at IS NULL` + ) + for (const row of persistedConnectors) { + try { + const parsedConfig = JSON.parse(row.config || '{}') as Record + const connectorType = typeof parsedConfig.type === 'string' ? parsedConfig.type : row.name + const connector = await createConnector({ ...parsedConfig, type: connectorType }, dataDir) + if (!connector) continue + if (!registry.get(connector.name)) { + const connectorCtx: PluginContext = { + config: { ...parsedConfig, type: connectorType }, + dataDir, + log: pluginCtx.log, + } + await registry.register(connector, connectorCtx) + } + getConnectorManagerForWorkspace(row.workspace_id).registerConnector(connector, { + name: row.name, + syncIntervalSeconds: row.sync_interval_seconds || 300, + autoSync: true, + config: { ...parsedConfig, type: connectorType }, + }) + } catch (err) { + log.fail(`Failed to restore connector ${row.name}: ${(err as Error).message}`) + } + } + // Shutdown function const shutdown = async (): Promise => { clearInterval(purgeTimer) diff --git a/packages/server/src/http/app.ts b/packages/server/src/http/app.ts index 50d5300..2629eb5 100644 --- a/packages/server/src/http/app.ts +++ b/packages/server/src/http/app.ts @@ -13,9 +13,11 @@ import { tagRoutes } from './routes/tags.js' import { collectionRoutes } from './routes/collections.js' import { pluginRoutes } from './routes/plugins.js' import { authRoutes } from './routes/auth-routes.js' +import { workbenchRoutes } from './routes/workbench.js' import { getSharedConversationHandler } from './routes/conversations.js' import { authMiddleware } from './middleware/auth.js' import { rateLimit } from './middleware/rate-limit.js' +import { isSecureRequest } from './request-security.js' import type { AppContext } from '../bootstrap.js' import { generateWidgetScript } from '../widget/widget.js' @@ -26,18 +28,21 @@ export interface AppOptions { export function createApp(ctx: AppContext, opts?: AppOptions) { const app = new Hono() - // CORS: Allows localhost origins by default for personal mode development. - // In production team mode, configure allowed origins in opendocuments.config.ts: - // security: { access: { allowedOrigins: ['https://your-domain.com'] } } - // For now, reverse proxy (nginx) CORS headers are recommended for production. + // Public liveness probe. It intentionally exposes no configuration or + // dependency state and must remain available in authenticated team mode. + app.get('/healthz', (c) => c.json({ status: 'ok' })) + + // CORS: localhost is allowed for local web development; production origins + // are configured in security.transport.allowedOrigins. app.use('*', cors({ origin: (origin) => { - // Allow localhost on any port if (!origin) return '*' // same-origin requests if (origin.startsWith('http://localhost:') || origin.startsWith('http://127.0.0.1:')) { return origin } - // TODO: Read additional origins from config + if (ctx.config.security.transport.allowedOrigins.includes(origin)) { + return origin + } return null // reject other origins }, })) @@ -48,11 +53,30 @@ export function createApp(ctx: AppContext, opts?: AppOptions) { app.get('/shared/:token', sharedConversationHandler) app.get('/api/v1/shared/:token', sharedConversationHandler) + app.use('/api/*', async (c, next) => { + const transport = ctx.config.security.transport + if (ctx.config.mode !== 'team' || !transport.enforceHTTPS) { + await next() + return + } + + if (isSecureRequest(c, transport.proxy)) { + await next() + return + } + + return c.json({ error: 'HTTPS required' }, 426) + }) + // Auth middleware: personal mode passes through, team mode requires X-API-Key app.use('/api/*', authMiddleware(ctx)) // Rate limiting: 60 requests per minute per API key / IP - app.use('/api/*', rateLimit({ max: 60, windowMs: 60000 })) + app.use('/api/*', rateLimit({ + max: 60, + windowMs: 60000, + trustedProxy: ctx.config.security.transport.proxy, + })) // Body size limit (50MB) const MAX_BODY_SIZE = 50 * 1024 * 1024 @@ -65,13 +89,12 @@ export function createApp(ctx: AppContext, opts?: AppOptions) { }) app.get('/widget.js', (c) => { - // Check Referer against config.security.access.widgetAllowedDomains const referer = c.req.header('referer') - const allowedDomains = (ctx.config as any)?.security?.access?.widgetAllowedDomains as string[] | undefined + const allowedDomains = ctx.config.security.transport.widgetAllowedDomains if (allowedDomains && allowedDomains.length > 0 && referer) { try { const refOrigin = new URL(referer).origin - if (!allowedDomains.some(d => refOrigin === d || refOrigin === `https://${d}` || refOrigin === `http://${d}`)) { + if (!allowedDomains.some((domain) => originMatchesDomain(refOrigin, domain))) { return c.text('Forbidden', 403) } } catch {} @@ -87,6 +110,7 @@ export function createApp(ctx: AppContext, opts?: AppOptions) { app.route('/', tagRoutes(ctx)) app.route('/', collectionRoutes(ctx)) app.route('/', pluginRoutes(ctx)) + app.route('/', workbenchRoutes(ctx)) app.onError((err, c) => { console.error('Unhandled error:', err.message) @@ -149,8 +173,9 @@ export function createApp(ctx: AppContext, opts?: AppOptions) { }) } - // TODO(Phase 2): Add WebSocket endpoint at /api/v1/ws/chat - // Currently covered by SSE streaming at POST /api/v1/chat/stream - return app } + +function originMatchesDomain(origin: string, domain: string): boolean { + return origin === domain || origin === `https://${domain}` || origin === `http://${domain}` +} diff --git a/packages/server/src/http/middleware/auth.ts b/packages/server/src/http/middleware/auth.ts index c838ca0..0bd9431 100644 --- a/packages/server/src/http/middleware/auth.ts +++ b/packages/server/src/http/middleware/auth.ts @@ -1,6 +1,7 @@ import type { Context, Next } from 'hono' import type { AppContext } from '../../bootstrap.js' import type { APIKeyScope, ValidatedKey } from 'opendocuments-core' +import { getClientIp } from '../request-security.js' export interface PersonalModeAuth { mode: 'personal' @@ -53,13 +54,18 @@ export function authMiddleware(appCtx: AppContext) { return c.json({ error: 'Invalid or expired API key' }, 401) } - // TODO: Check validated.record.allowedIps against client IP when ip_restrictions column is added - // if (validated.record.allowedIps && validated.record.allowedIps.length > 0) { - // const clientIp = c.req.header('x-forwarded-for')?.split(',')[0]?.trim() || '' - // if (!validated.record.allowedIps.includes(clientIp)) { - // return c.json({ error: 'IP not allowed for this API key' }, 403) - // } - // } + if (validated.record.allowedIps && validated.record.allowedIps.length > 0) { + const clientIp = getClientIp(c, appCtx.config.security.transport.proxy) + if (!clientIp || !validated.record.allowedIps.includes(clientIp)) { + appCtx.auditLogger?.log({ + eventType: 'auth:failed', + userId: validated.record.userId, + workspaceId: validated.record.workspaceId, + details: { reason: 'ip not allowed', clientIp }, + }) + return c.json({ error: 'IP not allowed for this API key' }, 403) + } + } appCtx.auditLogger?.log({ eventType: 'auth:login', diff --git a/packages/server/src/http/middleware/rate-limit.ts b/packages/server/src/http/middleware/rate-limit.ts index be9e176..a38a0a9 100644 --- a/packages/server/src/http/middleware/rate-limit.ts +++ b/packages/server/src/http/middleware/rate-limit.ts @@ -1,19 +1,17 @@ import type { Context, Next } from 'hono' import { createHash } from 'node:crypto' +import { getClientIp } from '../request-security.js' interface RateLimitEntry { count: number resetAt: number } -export function rateLimit(opts: { max: number; windowMs: number }) { +export function rateLimit(opts: { max: number; windowMs: number; trustedProxy?: string }) { const store = new Map() return async (c: Context, next: Next) => { - // WARNING: x-forwarded-for can be spoofed. In production, use a reverse proxy (nginx/cloudflare) for rate limiting. - // This is a best-effort rate limiter for development and personal mode. - // Use API key or first IP from x-forwarded-for as identifier - const rawKey = c.req.header('x-api-key') || c.req.header('x-forwarded-for')?.split(',')[0]?.trim() || 'anonymous' + const rawKey = c.req.header('x-api-key') || getClientIp(c, opts.trustedProxy) || 'anonymous' const key = createHash('sha256').update(rawKey).digest('hex').substring(0, 16) // short hash for memory efficiency const now = Date.now() diff --git a/packages/server/src/http/request-security.ts b/packages/server/src/http/request-security.ts new file mode 100644 index 0000000..8d53b8d --- /dev/null +++ b/packages/server/src/http/request-security.ts @@ -0,0 +1,129 @@ +import { BlockList, isIP } from 'node:net' +import { getConnInfo } from '@hono/node-server/conninfo' +import type { Context } from 'hono' + +const TRUSTED_PROXY_CACHE = new Map() + +function normalizeAddress(address: string): string { + const trimmed = address.trim() + if (trimmed.startsWith('::ffff:')) return trimmed.slice(7) + if (trimmed.startsWith('[') && trimmed.endsWith(']')) return trimmed.slice(1, -1) + return trimmed +} + +function getTrustedProxyBlockList(config: string): BlockList { + const cached = TRUSTED_PROXY_CACHE.get(config) + if (cached) return cached + + const blockList = new BlockList() + for (const rawEntry of config.split(',')) { + const entry = normalizeAddress(rawEntry) + if (!entry) continue + + if (entry === 'loopback') { + blockList.addSubnet('127.0.0.0', 8, 'ipv4') + blockList.addAddress('::1', 'ipv6') + continue + } + + const [address, prefixValue] = entry.split('/', 2) + const addressType = isIP(address) + if (addressType === 0) continue + const family = addressType === 4 ? 'ipv4' : 'ipv6' + + if (prefixValue !== undefined) { + const prefix = Number(prefixValue) + const maxPrefix = addressType === 4 ? 32 : 128 + if (Number.isInteger(prefix) && prefix >= 0 && prefix <= maxPrefix) { + blockList.addSubnet(address, prefix, family) + } + continue + } + + blockList.addAddress(address, family) + } + + TRUSTED_PROXY_CACHE.set(config, blockList) + return blockList +} + +/** Return the TCP peer address without consulting spoofable HTTP headers. */ +export function getRemoteAddress(c: Context): string { + try { + return normalizeAddress(getConnInfo(c).remote.address || '') + } catch { + // Hono's in-process request helper has no socket. Production Node requests do. + return '' + } +} + +/** Check whether an address belongs to the configured reverse-proxy allowlist. */ +export function isTrustedProxy(address: string, trustedProxy?: string): boolean { + const normalized = normalizeAddress(address) + if (!normalized || !trustedProxy) return false + const addressType = isIP(normalized) + if (addressType === 0) return false + return getTrustedProxyBlockList(trustedProxy).check( + normalized, + addressType === 4 ? 'ipv4' : 'ipv6' + ) +} + +/** Resolve the client IP, honoring forwarding headers only from a trusted proxy. */ +export function getClientIp(c: Context, trustedProxy?: string): string { + const remoteAddress = getRemoteAddress(c) + if (!isTrustedProxy(remoteAddress, trustedProxy)) return remoteAddress + + const forwardedFor = c.req.header('x-forwarded-for') + if (forwardedFor) { + const chain = forwardedFor + .split(',') + .map(normalizeAddress) + .filter(Boolean) + + for (let i = chain.length - 1; i >= 0; i--) { + if (!isTrustedProxy(chain[i], trustedProxy)) return chain[i] + } + } + + return normalizeAddress(c.req.header('x-real-ip') || '') || remoteAddress +} + +function isLoopbackAddress(address: string): boolean { + const normalized = normalizeAddress(address) + return normalized === '::1' || normalized.startsWith('127.') +} + +/** Determine whether a request arrived securely, with trusted proxy support. */ +export function isSecureRequest(c: Context, trustedProxy?: string): boolean { + const remoteAddress = getRemoteAddress(c) + if (isLoopbackAddress(remoteAddress)) return true + + try { + if (new URL(c.req.url).protocol === 'https:') return true + } catch { + // Continue to trusted-proxy evaluation. + } + + if (isTrustedProxy(remoteAddress, trustedProxy)) { + const forwardedProto = c.req.header('x-forwarded-proto') + ?.split(',') + .map(value => value.trim().toLowerCase()) + .filter(Boolean) + .pop() + return forwardedProto === 'https' + } + + // Hono's in-process request helper has no peer address. Permit only its + // localhost URL so unit tests and embedded usage keep their local semantics. + if (!remoteAddress) { + try { + const hostname = new URL(c.req.url).hostname + return hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '[::1]' + } catch { + return false + } + } + + return false +} diff --git a/packages/server/src/http/routes/admin.ts b/packages/server/src/http/routes/admin.ts index 49d6ac7..72edd0b 100644 --- a/packages/server/src/http/routes/admin.ts +++ b/packages/server/src/http/routes/admin.ts @@ -2,6 +2,62 @@ import { Hono } from 'hono' import type { AppContext } from '../../bootstrap.js' import { requireRole, requireScope } from '../middleware/auth.js' import { resolveRequestWorkspaceId } from '../workspace.js' +import type { ConnectorPlugin, PluginContext } from 'opendocuments-core' + +const GITHUB_REPO_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/ +const GITHUB_BRANCH_PATTERN = /^[A-Za-z0-9._/-]+$/ + +interface GitHubConnectorRequest { + repo?: string + token?: string + branch?: string + paths?: string[] + syncInterval?: number +} + +interface GitHubConnectorConfig { + type: 'github' + repo: string + token?: string + branch: string + paths?: string[] + syncInterval?: number +} + +function sanitizeConnector(connector: { + name: string + connectorId: string + type: string + status: string + lastSyncedAt: string | null + syncIntervalSeconds?: number | null + repo?: string +}) { + return connector +} + +async function createGitHubConnector(config: GitHubConnectorConfig, dataDir: string): Promise { + const packageName = '@opendocuments/connector-github' + const mod = await import(/* @vite-ignore */ packageName) + const ConnectorClass = mod.default + if (typeof ConnectorClass !== 'function') { + throw new Error('GitHub connector package does not export a connector class') + } + + const connector = new ConnectorClass() as ConnectorPlugin + const connectorCtx: PluginContext = { + config: config as unknown as Record, + dataDir, + log: { + ok: (msg: string) => console.log(`[ok] ${msg}`), + fail: (msg: string) => console.error(`[fail] ${msg}`), + info: (msg: string) => console.log(`[info] ${msg}`), + wait: (msg: string) => console.log(`[wait] ${msg}`), + }, + } + await connector.setup(connectorCtx) + return connector +} export function adminRoutes(ctx: AppContext) { const app = new Hono() @@ -240,5 +296,76 @@ export function adminRoutes(ctx: AppContext) { return c.json({ connectors }) }) + app.post('/api/v1/admin/connectors/github', requireRole('admin'), requireScope('admin'), async (c) => { + const workspaceId = resolveRequestWorkspaceId(c, ctx) + const body = (await c.req.json().catch(() => ({}))) as GitHubConnectorRequest + const repo = body.repo?.trim() + const branch = body.branch?.trim() || 'main' + const token = body.token?.trim() || undefined + const paths = Array.isArray(body.paths) + ? body.paths.map(p => p.trim()).filter(Boolean) + : undefined + const syncInterval = body.syncInterval && Number.isFinite(body.syncInterval) + ? Math.max(60, Math.floor(body.syncInterval)) + : 300 + + if (!repo || !GITHUB_REPO_PATTERN.test(repo)) { + return c.json({ error: 'GitHub repo must be in owner/repo format.' }, 400) + } + if (!GITHUB_BRANCH_PATTERN.test(branch)) { + return c.json({ error: 'GitHub branch contains invalid characters.' }, 400) + } + + const config: GitHubConnectorConfig = { + type: 'github', + repo, + branch, + ...(token ? { token } : {}), + ...(paths && paths.length > 0 ? { paths } : {}), + syncInterval, + } + + try { + const connector = await createGitHubConnector(config, ctx.config.storage.dataDir) + const health = await connector.healthCheck?.() + if (health && !health.healthy) { + return c.json({ error: `GitHub connection failed: ${health.message || 'unhealthy'}` }, 400) + } + + const manager = ctx.forWorkspace(workspaceId).connectorManager + const connectorId = manager.registerConnector(connector, { + name: 'github', + syncIntervalSeconds: syncInterval, + autoSync: true, + config: config as unknown as Record, + }) + const registered = manager.listConnectors().find(conn => conn.connectorId === connectorId) + + return c.json({ + connector: registered ? sanitizeConnector(registered) : { + name: 'github', + connectorId, + type: connector.name, + status: 'active', + lastSyncedAt: null, + repo, + }, + health: health || { healthy: true }, + }, 201) + } catch (err) { + return c.json({ error: `Failed to configure GitHub connector: ${(err as Error).message}` }, 500) + } + }) + + app.post('/api/v1/admin/connectors/github/sync', requireRole('admin'), requireScope('admin'), async (c) => { + const workspaceId = resolveRequestWorkspaceId(c, ctx) + try { + const result = await ctx.forWorkspace(workspaceId).connectorManager.syncConnector('@opendocuments/connector-github') + return c.json({ result }) + } catch (err) { + return c.json({ error: (err as Error).message }, 404) + } + }) + return app } diff --git a/packages/server/src/http/routes/collections.ts b/packages/server/src/http/routes/collections.ts index 12d02c8..937b764 100644 --- a/packages/server/src/http/routes/collections.ts +++ b/packages/server/src/http/routes/collections.ts @@ -1,6 +1,7 @@ import { Hono } from 'hono' import type { AppContext } from '../../bootstrap.js' import { getWorkspaceServices } from '../workspace.js' +import { requireScope } from '../middleware/auth.js' export function collectionRoutes(ctx: AppContext) { const app = new Hono() @@ -10,23 +11,50 @@ export function collectionRoutes(ctx: AppContext) { return c.json({ collections: collectionManager.list() }) }) - app.post('/api/v1/collections', async (c) => { + app.get('/api/v1/collections/:id/documents', (c) => { + const { collectionManager, store } = getWorkspaceServices(c, ctx) + const collectionId = c.req.param('id') + const collection = collectionManager.list().find((item) => item.id === collectionId) + if (!collection) return c.json({ error: 'Collection not found' }, 404) + + const documentIds = new Set(collectionManager.getDocuments(collectionId)) + const documents = store.listDocuments().filter((document) => documentIds.has(document.id)) + return c.json({ collection, documents }) + }) + + app.post('/api/v1/collections', requireScope('document:write'), async (c) => { const { collectionManager } = getWorkspaceServices(c, ctx) const body = await c.req.json<{ name: string; description?: string }>() - return c.json(collectionManager.create(body.name, body.description), 201) + const name = body.name?.trim() + if (!name) return c.json({ error: 'Collection name required' }, 400) + return c.json(collectionManager.create(name, body.description?.trim() || undefined), 201) }) - app.delete('/api/v1/collections/:id', (c) => { + app.delete('/api/v1/collections/:id', requireScope('document:write'), (c) => { const { collectionManager } = getWorkspaceServices(c, ctx) - collectionManager.delete(c.req.param('id')) + const collectionId = c.req.param('id') + if (!collectionId) return c.json({ error: 'Collection id required' }, 400) + collectionManager.delete(collectionId) return c.json({ deleted: true }) }) - app.post('/api/v1/collections/:id/documents/:docId', (c) => { + app.post('/api/v1/collections/:id/documents/:docId', requireScope('document:write'), (c) => { const { collectionManager } = getWorkspaceServices(c, ctx) - collectionManager.addDocument(c.req.param('id'), c.req.param('docId')) + const collectionId = c.req.param('id') + const documentId = c.req.param('docId') + if (!collectionId || !documentId) return c.json({ error: 'Collection and document ids required' }, 400) + collectionManager.addDocument(collectionId, documentId) return c.json({ added: true }) }) + app.delete('/api/v1/collections/:id/documents/:docId', requireScope('document:write'), (c) => { + const { collectionManager } = getWorkspaceServices(c, ctx) + const collectionId = c.req.param('id') + const documentId = c.req.param('docId') + if (!collectionId || !documentId) return c.json({ error: 'Collection and document ids required' }, 400) + collectionManager.removeDocument(collectionId, documentId) + return c.json({ removed: true }) + }) + return app } diff --git a/packages/server/src/http/routes/conversations.ts b/packages/server/src/http/routes/conversations.ts index fd0a1b8..9b5d0d4 100644 --- a/packages/server/src/http/routes/conversations.ts +++ b/packages/server/src/http/routes/conversations.ts @@ -2,6 +2,7 @@ import { randomBytes } from 'node:crypto' import { Hono, type Context } from 'hono' import type { AppContext } from '../../bootstrap.js' import { getWorkspaceServices, resolveRequestWorkspaceId } from '../workspace.js' +import { requireScope } from '../middleware/auth.js' export function getSharedConversationHandler(ctx: AppContext) { return async (c: Context) => { @@ -86,7 +87,7 @@ export function conversationRoutes(ctx: AppContext) { return c.json({ updated: true }) }) - app.post('/api/v1/conversations/:id/share', async (c) => { + app.post('/api/v1/conversations/:id/share', requireScope('ask'), async (c) => { const id = c.req.param('id') const workspaceId = resolveRequestWorkspaceId(c, ctx) const convo = ctx.db.get( diff --git a/packages/server/src/http/routes/documents.ts b/packages/server/src/http/routes/documents.ts index 0c5ab7f..943a96a 100644 --- a/packages/server/src/http/routes/documents.ts +++ b/packages/server/src/http/routes/documents.ts @@ -1,6 +1,7 @@ import { Hono } from 'hono' import type { AppContext } from '../../bootstrap.js' import { getWorkspaceServices } from '../workspace.js' +import { requireScope } from '../middleware/auth.js' export function documentRoutes(ctx: AppContext) { const app = new Hono() @@ -18,9 +19,10 @@ export function documentRoutes(ctx: AppContext) { }) // Restore a deleted document - app.post('/api/v1/documents/:id/restore', (c) => { + app.post('/api/v1/documents/:id/restore', requireScope('document:write'), (c) => { const { store } = getWorkspaceServices(c, ctx) const id = c.req.param('id') + if (!id) return c.json({ error: 'Document id required' }, 400) store.restoreDocument(id) return c.json({ restored: true }) }) @@ -32,15 +34,17 @@ export function documentRoutes(ctx: AppContext) { return c.json(doc) }) - app.delete('/api/v1/documents/:id', async (c) => { + app.delete('/api/v1/documents/:id', requireScope('document:write'), async (c) => { const { store } = getWorkspaceServices(c, ctx) - const doc = store.getDocument(c.req.param('id')) + const id = c.req.param('id') + if (!id) return c.json({ error: 'Document id required' }, 400) + const doc = store.getDocument(id) if (!doc) return c.json({ error: 'Document not found' }, 404) - await store.softDeleteDocument(c.req.param('id')) + await store.softDeleteDocument(id) return c.json({ deleted: true }) }) - app.post('/api/v1/documents/upload', async (c) => { + app.post('/api/v1/documents/upload', requireScope('document:write'), async (c) => { const body = await c.req.parseBody() const file = body['file'] if (!file || !(file instanceof File)) { diff --git a/packages/server/src/http/routes/health.ts b/packages/server/src/http/routes/health.ts index 4219092..3789638 100644 --- a/packages/server/src/http/routes/health.ts +++ b/packages/server/src/http/routes/health.ts @@ -61,9 +61,12 @@ export function healthRoutes(ctx: AppContext) { if (typeof model.healthCheck === 'function') { try { const result = await model.healthCheck() - checks[`model:${model.name}`] = result.healthy - ? { status: 'ok', message: result.message } - : { status: 'warning', message: result.message } + if (result.healthy) { + checks[`model:${model.name}`] = { status: 'ok', message: result.message } + } else { + checks[`model:${model.name}`] = { status: 'error', message: result.message } + allOk = false + } } catch (err) { checks[`model:${model.name}`] = { status: 'error', message: (err as Error).message } allOk = false diff --git a/packages/server/src/http/routes/workbench.ts b/packages/server/src/http/routes/workbench.ts new file mode 100644 index 0000000..9bf1e54 --- /dev/null +++ b/packages/server/src/http/routes/workbench.ts @@ -0,0 +1,160 @@ +import { Hono } from 'hono' +import type { AppContext } from '../../bootstrap.js' +import { SERVER_VERSION } from '../../version.js' +import { resolveRequestWorkspaceId } from '../workspace.js' + +interface DocumentSummaryRow extends Record { + docCount: number + chunkCount: number +} + +interface DistributionRow extends Record { + key: string + count: number +} + +interface QualitySummaryRow extends Record { + totalQueries: number + avgConfidence: number | null + avgResponseTime: number | null +} + +interface FeedbackRow extends Record { + positive: number | null + negative: number | null +} + +interface RecentQueryRow extends Record { + query: string + profile: string + confidence_score: number | null + response_time_ms: number | null + route: string | null + created_at: string +} + +function toDistribution(rows: DistributionRow[]): Record { + const distribution: Record = {} + for (const row of rows) { + distribution[row.key || 'unknown'] = row.count + } + return distribution +} + +function suggestedQuestions(documentCount: number): string[] { + if (documentCount === 0) { + return [ + 'What should I upload first?', + 'How do I connect a documentation source?', + 'What can OpenDocuments answer once documents are indexed?', + ] + } + + return [ + 'Summarize the most important docs in this workspace.', + 'Which source explains the current deployment process?', + 'Find policy or architecture notes related to authentication.', + ] +} + +export function workbenchRoutes(ctx: AppContext) { + const app = new Hono() + + app.get('/api/v1/workbench', (c) => { + const workspaceId = resolveRequestWorkspaceId(c, ctx) + + const documentSummary = ctx.db.get( + 'SELECT COUNT(*) as docCount, COALESCE(SUM(chunk_count), 0) as chunkCount FROM documents WHERE deleted_at IS NULL AND workspace_id = ?', + [workspaceId] + ) + + const sourceDistribution = toDistribution(ctx.db.all( + 'SELECT source_type as key, COUNT(*) as count FROM documents WHERE deleted_at IS NULL AND workspace_id = ? GROUP BY source_type', + [workspaceId] + )) + + const statusDistribution = toDistribution(ctx.db.all( + 'SELECT status as key, COUNT(*) as count FROM documents WHERE deleted_at IS NULL AND workspace_id = ? GROUP BY status', + [workspaceId] + )) + + const quality = ctx.db.get( + 'SELECT COUNT(*) as totalQueries, AVG(confidence_score) as avgConfidence, AVG(response_time_ms) as avgResponseTime FROM query_logs WHERE workspace_id = ?', + [workspaceId] + ) + + const feedback = ctx.db.get( + `SELECT + SUM(CASE WHEN feedback = 'positive' THEN 1 ELSE 0 END) as positive, + SUM(CASE WHEN feedback = 'negative' THEN 1 ELSE 0 END) as negative + FROM query_logs WHERE feedback IS NOT NULL AND workspace_id = ?`, + [workspaceId] + ) + + const recentQueries = ctx.db.all( + `SELECT query, profile, confidence_score, response_time_ms, route, created_at + FROM query_logs + WHERE workspace_id = ? + ORDER BY created_at DESC + LIMIT 6`, + [workspaceId] + ).map((row) => ({ + query: row.query, + profile: row.profile, + confidenceScore: row.confidence_score, + responseTimeMs: row.response_time_ms, + route: row.route, + createdAt: row.created_at, + })) + + const connectors = ctx.forWorkspace(workspaceId).connectorManager.listConnectors() + const workspace = ctx.workspaceManager.getById(workspaceId) + const activeConnectors = connectors.filter((connector) => connector.status === 'active').length + const modelCount = ctx.registry.getModels().length + const stubModelCount = ctx.registry.getModels().filter((model) => model.name.includes('stub')).length + const docCount = documentSummary?.docCount || 0 + + return c.json({ + health: { + status: 'ok', + version: SERVER_VERSION, + modelStatus: stubModelCount > 0 ? 'degraded' : 'ready', + models: modelCount, + }, + corpus: { + documents: docCount, + chunks: documentSummary?.chunkCount || 0, + sourceDistribution, + statusDistribution, + }, + quality: { + totalQueries: quality?.totalQueries || 0, + avgConfidence: Math.round((quality?.avgConfidence || 0) * 100) / 100, + avgResponseTimeMs: Math.round(quality?.avgResponseTime || 0), + feedback: { + positive: feedback?.positive || 0, + negative: feedback?.negative || 0, + }, + }, + connectors: { + total: connectors.length, + active: activeConnectors, + recent: connectors.slice(0, 4).map((connector) => ({ + name: connector.name, + type: connector.type, + status: connector.status, + lastSyncedAt: connector.lastSyncedAt, + repo: connector.repo, + })), + }, + workspace: { + name: workspace?.name || 'default', + mode: workspace?.mode || ctx.config.mode, + }, + recentQueries, + suggestedQuestions: suggestedQuestions(docCount), + }) + }) + + return app +} diff --git a/packages/server/tests/bootstrap.test.ts b/packages/server/tests/bootstrap.test.ts index c3efadc..97af4e4 100644 --- a/packages/server/tests/bootstrap.test.ts +++ b/packages/server/tests/bootstrap.test.ts @@ -40,4 +40,47 @@ describe('bootstrap', () => { const ws = ctx.workspaceManager.getByName('default') expect(ws).toBeDefined() }) + + it('rejects unsupported relational database configuration', async () => { + tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + await expect(bootstrap({ + dataDir: tempDir, + configOverrides: { model: stubModel, storage: { db: 'postgres' } }, + })).rejects.toThrow('Postgres storage is not implemented') + }) + + it('rejects unsupported vector database configuration', async () => { + tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + await expect(bootstrap({ + dataDir: tempDir, + configOverrides: { model: stubModel, storage: { vectorDb: 'qdrant' } }, + })).rejects.toThrow('Qdrant storage is not implemented') + }) + + it('restores persisted GitHub connectors on bootstrap', async () => { + tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + ctx = await bootstrap({ dataDir: tempDir, configOverrides: { model: stubModel } }) + const ws = ctx.workspaceManager.getByName('default') + expect(ws).toBeDefined() + ctx.db.run( + `INSERT INTO connectors (id, workspace_id, name, type, config, sync_interval_seconds, status, created_at) + VALUES (?, ?, ?, ?, ?, ?, 'active', ?)`, + [ + 'github-connector-1', + ws!.id, + 'github', + '@opendocuments/connector-github', + JSON.stringify({ type: 'github', repo: 'owner/repo', branch: 'main' }), + 300, + new Date().toISOString(), + ] + ) + await ctx.shutdown() + ctx = null + + ctx = await bootstrap({ dataDir: tempDir, configOverrides: { model: stubModel } }) + expect(ctx.connectorManager.listConnectors()).toEqual([ + expect.objectContaining({ name: 'github', repo: 'owner/repo' }), + ]) + }) }) diff --git a/packages/server/tests/http/admin.test.ts b/packages/server/tests/http/admin.test.ts index 434f955..f55fd71 100644 --- a/packages/server/tests/http/admin.test.ts +++ b/packages/server/tests/http/admin.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { bootstrap, type AppContext } from '../../src/bootstrap.js' import { createApp } from '../../src/http/app.js' import { mkdtempSync, rmSync } from 'node:fs' @@ -26,6 +26,7 @@ describe('Admin Routes', () => { }) afterEach(async () => { + vi.unstubAllGlobals() await ctx.shutdown() rmSync(tempDir, { recursive: true, force: true }) }) @@ -69,6 +70,112 @@ describe('Admin Routes', () => { const body = await res.json() expect(body.connectors).toBeDefined() }) + + it('POST /api/v1/admin/connectors/github registers a GitHub connector', async () => { + vi.stubGlobal('fetch', vi.fn(async (input: RequestInfo | URL) => { + const url = String(input) + if (url.includes('/repos/owner/repo')) { + return new Response(JSON.stringify({ full_name: 'owner/repo' }), { status: 200 }) + } + return new Response('{}', { status: 404 }) + })) + + const res = await app.request('/api/v1/admin/connectors/github', { + method: 'POST', + body: JSON.stringify({ repo: 'owner/repo', branch: 'main' }), + headers: { 'Content-Type': 'application/json' }, + }) + + expect(res.status).toBe(201) + const body = await res.json() + expect(body.connector).toMatchObject({ + name: 'github', + repo: 'owner/repo', + status: 'active', + }) + + const list = await app.request('/api/v1/admin/connectors') + const listBody = await list.json() + expect(listBody.connectors).toEqual([ + expect.objectContaining({ name: 'github', repo: 'owner/repo' }), + ]) + }) + + it('POST /api/v1/admin/connectors/github starts automatic sync', async () => { + vi.useFakeTimers() + try { + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input) + if (url.endsWith('/repos/owner/repo')) { + return new Response(JSON.stringify({ full_name: 'owner/repo' }), { status: 200 }) + } + if (url.includes('/git/trees/main?recursive=1')) { + return new Response(JSON.stringify({ tree: [] }), { status: 200 }) + } + return new Response('{}', { status: 404 }) + }) + vi.stubGlobal('fetch', fetchMock) + + const res = await app.request('/api/v1/admin/connectors/github', { + method: 'POST', + body: JSON.stringify({ repo: 'owner/repo', branch: 'main', syncInterval: 60 }), + headers: { 'Content-Type': 'application/json' }, + }) + + expect(res.status).toBe(201) + await vi.advanceTimersByTimeAsync(60000) + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining('/repos/owner/repo/git/trees/main?recursive=1'), + expect.any(Object), + ) + } finally { + vi.useRealTimers() + } + }) + + it('POST /api/v1/admin/connectors/github/sync indexes repository files', async () => { + vi.stubGlobal('fetch', vi.fn(async (input: RequestInfo | URL) => { + const url = String(input) + if (url.endsWith('/repos/owner/repo')) { + return new Response(JSON.stringify({ full_name: 'owner/repo' }), { status: 200 }) + } + if (url.includes('/git/trees/main?recursive=1')) { + return new Response(JSON.stringify({ + tree: [{ path: 'README.md', type: 'blob', sha: 'sha-readme' }], + }), { status: 200 }) + } + if (url.includes('/contents/README.md?ref=main')) { + return new Response(JSON.stringify({ + name: 'README.md', + encoding: 'base64', + content: Buffer.from('# Demo Repo\n\nGitHub connector content').toString('base64'), + }), { status: 200 }) + } + return new Response('{}', { status: 404 }) + })) + + await app.request('/api/v1/admin/connectors/github', { + method: 'POST', + body: JSON.stringify({ repo: 'owner/repo', branch: 'main' }), + headers: { 'Content-Type': 'application/json' }, + }) + + const res = await app.request('/api/v1/admin/connectors/github/sync', { method: 'POST' }) + expect(res.status).toBe(200) + const body = await res.json() + expect(body.result.documentsIndexed).toBe(1) + expect(body.result.errors).toEqual([]) + + const docs = ctx.store.listDocuments() + expect(docs).toEqual([ + expect.objectContaining({ + title: 'README.md', + source_type: '@opendocuments/connector-github', + source_path: 'github://owner/repo/README.md', + status: 'indexed', + }), + ]) + }) }) describe('Admin Routes (team mode)', () => { diff --git a/packages/server/tests/http/auth.test.ts b/packages/server/tests/http/auth.test.ts index 178b780..9503f51 100644 --- a/packages/server/tests/http/auth.test.ts +++ b/packages/server/tests/http/auth.test.ts @@ -5,6 +5,20 @@ import { mkdtempSync, rmSync } from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' +function nodeEnv(remoteAddress: string) { + return { + server: { + incoming: { + socket: { + remoteAddress, + remotePort: 12345, + remoteFamily: remoteAddress.includes(':') ? 'IPv6' : 'IPv4', + }, + }, + }, + } +} + describe('Auth Middleware', () => { let ctx: AppContext let tempDir: string @@ -58,6 +72,56 @@ describe('Auth Middleware', () => { expect(res.status).toBe(200) }) + it('enforces API key IP restrictions', async () => { + const { rawKey } = ctx.apiKeyManager.create({ + name: 'ip-test', + workspaceId: ctx.workspaceManager.list()[0].id, + userId: 'user-1', + role: 'admin', + allowedIps: ['203.0.113.10'], + }) + ctx.config.security.transport.proxy = '10.0.0.1' + const app = createApp(ctx) + + const blocked = await app.request('/api/v1/health', { + headers: { + 'X-API-Key': rawKey, + 'X-Forwarded-For': '198.51.100.7', + 'X-Forwarded-Proto': 'https', + }, + }, nodeEnv('10.0.0.1') as any) + const allowed = await app.request('/api/v1/health', { + headers: { + 'X-API-Key': rawKey, + 'X-Forwarded-For': '203.0.113.10, 10.0.0.1', + 'X-Forwarded-Proto': 'https', + }, + }, nodeEnv('10.0.0.1') as any) + + expect(blocked.status).toBe(403) + expect(allowed.status).toBe(200) + }) + + it('ignores spoofed forwarding headers from untrusted clients', async () => { + const { rawKey } = ctx.apiKeyManager.create({ + name: 'ip-spoof-test', + workspaceId: ctx.workspaceManager.list()[0].id, + userId: 'user-1', + role: 'admin', + allowedIps: ['203.0.113.10'], + }) + const app = createApp(ctx) + + const res = await app.request('https://docs.company.com/api/v1/health', { + headers: { + 'X-API-Key': rawKey, + 'X-Forwarded-For': '203.0.113.10', + }, + }, nodeEnv('198.51.100.7') as any) + + expect(res.status).toBe(403) + }) + it('rejects expired API key', async () => { const { rawKey } = ctx.apiKeyManager.create({ name: 'expired', workspaceId: ctx.workspaceManager.list()[0].id, diff --git a/packages/server/tests/http/collections.test.ts b/packages/server/tests/http/collections.test.ts new file mode 100644 index 0000000..42cd489 --- /dev/null +++ b/packages/server/tests/http/collections.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { bootstrap, type AppContext } from '../../src/bootstrap.js' +import { createApp } from '../../src/http/app.js' +import { mkdtempSync, rmSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' + +const stubModel = { + provider: 'stub', + llm: 'stub-llm', + embedding: 'stub-embedding', + apiKey: '', + baseUrl: '', + embeddingDimensions: 384, +} as any + +describe('Collection Routes', () => { + let ctx: AppContext + let app: ReturnType + let tempDir: string + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + ctx = await bootstrap({ dataDir: tempDir, configOverrides: { model: stubModel } }) + app = createApp(ctx) + }) + + afterEach(async () => { + await ctx.shutdown() + rmSync(tempDir, { recursive: true, force: true }) + }) + + it('creates a collection and returns its assigned documents', async () => { + const formData = new FormData() + formData.append('file', new File(['# Handbook\n\nUse cited answers.'], 'handbook.md', { type: 'text/markdown' })) + const upload = await app.request('/api/v1/documents/upload', { method: 'POST', body: formData }) + expect(upload.status).toBe(201) + const uploaded = await upload.json() + + const create = await app.request('/api/v1/collections', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: 'Policies', description: 'Operational policies' }), + }) + expect(create.status).toBe(201) + const collection = await create.json() + + const add = await app.request(`/api/v1/collections/${collection.id}/documents/${uploaded.documentId}`, { method: 'POST' }) + expect(add.status).toBe(200) + + const detail = await app.request(`/api/v1/collections/${collection.id}/documents`) + expect(detail.status).toBe(200) + const body = await detail.json() + expect(body.collection).toMatchObject({ id: collection.id, name: 'Policies' }) + expect(body.documents).toEqual([ + expect.objectContaining({ + id: uploaded.documentId, + title: 'handbook.md', + status: 'indexed', + }), + ]) + }) + + it('rejects empty collection names', async () => { + const res = await app.request('/api/v1/collections', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: ' ' }), + }) + expect(res.status).toBe(400) + }) + + it('requires document:write scope for collection mutations in team mode', async () => { + const teamDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + const teamCtx = await bootstrap({ + dataDir: teamDir, + configOverrides: { mode: 'team', model: stubModel }, + }) + try { + const workspaceId = teamCtx.workspaceManager.list()[0].id + const viewer = teamCtx.apiKeyManager.create({ + name: 'viewer', workspaceId, userId: 'viewer-1', role: 'viewer', + }) + const member = teamCtx.apiKeyManager.create({ + name: 'member', workspaceId, userId: 'member-1', role: 'member', + }) + const teamApp = createApp(teamCtx) + const body = JSON.stringify({ name: 'Restricted collection' }) + + const blocked = await teamApp.request('/api/v1/collections', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-API-Key': viewer.rawKey }, + body, + }) + const allowed = await teamApp.request('/api/v1/collections', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-API-Key': member.rawKey }, + body, + }) + + expect(blocked.status).toBe(403) + expect(allowed.status).toBe(201) + } finally { + await teamCtx.shutdown() + rmSync(teamDir, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/tests/http/conversations.test.ts b/packages/server/tests/http/conversations.test.ts new file mode 100644 index 0000000..7191b1f --- /dev/null +++ b/packages/server/tests/http/conversations.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { bootstrap, type AppContext } from '../../src/bootstrap.js' +import { createApp } from '../../src/http/app.js' +import { mkdtempSync, rmSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' + +describe('Conversation Routes', () => { + let ctx: AppContext + let app: ReturnType + let tempDir: string + const stubModel = { + provider: 'stub', + llm: 'stub-llm', + embedding: 'stub-embedding', + apiKey: '', + baseUrl: '', + embeddingDimensions: 384, + } as any + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + ctx = await bootstrap({ dataDir: tempDir, configOverrides: { mode: 'team', model: stubModel } }) + app = createApp(ctx) + }) + + afterEach(async () => { + await ctx.shutdown() + rmSync(tempDir, { recursive: true, force: true }) + }) + + it('requires ask scope to share conversations in team mode', async () => { + const workspaceId = ctx.workspaceManager.list()[0].id + const { rawKey } = ctx.apiKeyManager.create({ + name: 'reader', + workspaceId, + userId: 'user-1', + role: 'viewer', + scopes: ['document:read'], + }) + const conversation = ctx.forWorkspace(workspaceId).conversationManager.create('Private thread') + + const res = await app.request(`/api/v1/conversations/${conversation.id}/share`, { + method: 'POST', + headers: { 'X-API-Key': rawKey }, + }) + + expect(res.status).toBe(403) + }) +}) diff --git a/packages/server/tests/http/documents.test.ts b/packages/server/tests/http/documents.test.ts index 7d6bbb9..3e0aa49 100644 --- a/packages/server/tests/http/documents.test.ts +++ b/packages/server/tests/http/documents.test.ts @@ -49,4 +49,46 @@ describe('Document Routes', () => { const res = await app.request('/api/v1/documents/nonexistent') expect(res.status).toBe(404) }) + + it('requires document:write scope to upload documents in team mode', async () => { + const teamDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + const teamCtx = await bootstrap({ + dataDir: teamDir, + configOverrides: { + mode: 'team', + model: { + provider: 'stub', + llm: 'stub-llm', + embedding: 'stub-embedding', + apiKey: '', + baseUrl: '', + embeddingDimensions: 384, + } as any, + }, + }) + try { + const workspaceId = teamCtx.workspaceManager.list()[0].id + const { rawKey } = teamCtx.apiKeyManager.create({ + name: 'reader', + workspaceId, + userId: 'user-1', + role: 'viewer', + scopes: ['document:read'], + }) + const teamApp = createApp(teamCtx) + const formData = new FormData() + formData.append('file', new File(['# Secret'], 'secret.md', { type: 'text/markdown' })) + + const res = await teamApp.request('/api/v1/documents/upload', { + method: 'POST', + headers: { 'X-API-Key': rawKey }, + body: formData, + }) + + expect(res.status).toBe(403) + } finally { + await teamCtx.shutdown() + rmSync(teamDir, { recursive: true, force: true }) + } + }) }) diff --git a/packages/server/tests/http/health.test.ts b/packages/server/tests/http/health.test.ts index 2b4f304..91547b6 100644 --- a/packages/server/tests/http/health.test.ts +++ b/packages/server/tests/http/health.test.ts @@ -5,6 +5,20 @@ import { mkdtempSync, rmSync } from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' +function nodeEnv(remoteAddress: string) { + return { + server: { + incoming: { + socket: { + remoteAddress, + remotePort: 12345, + remoteFamily: remoteAddress.includes(':') ? 'IPv6' : 'IPv4', + }, + }, + }, + } +} + describe('Health Routes', () => { let ctx: AppContext let app: ReturnType @@ -41,10 +55,176 @@ describe('Health Routes', () => { expect(body.plugins).toBeGreaterThan(0) }) - it('GET /api/v1/readyz returns ready when dependencies are healthy', async () => { + it('GET /api/v1/readyz returns not_ready when model plugins are degraded', async () => { const res = await app.request('/api/v1/readyz') + expect(res.status).toBe(503) + const body = await res.json() + expect(body.status).toBe('not_ready') + expect(body.checks['model:@opendocuments/stub-embedder'].status).toBe('error') + }) + + it('GET /api/v1/healthz remains a liveness check when readiness fails', async () => { + const res = await app.request('/api/v1/healthz') expect(res.status).toBe(200) const body = await res.json() - expect(body.status).toBe('ready') + expect(body.status).toBe('ok') + }) + + it('GET /healthz is public for container liveness probes in team mode', async () => { + const teamDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + const teamCtx = await bootstrap({ + dataDir: teamDir, + configOverrides: { mode: 'team', model: stubModel }, + }) + try { + const teamApp = createApp(teamCtx) + const res = await teamApp.request('/healthz') + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ status: 'ok' }) + } finally { + await teamCtx.shutdown() + rmSync(teamDir, { recursive: true, force: true }) + } + }) + + it('uses configured CORS allowed origins', async () => { + const secureDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + const secureCtx = await bootstrap({ + dataDir: secureDir, + configOverrides: { + model: stubModel, + security: { + transport: { + enforceHTTPS: false, + allowedOrigins: ['https://docs.company.com'], + }, + }, + } as any, + }) + try { + const secureApp = createApp(secureCtx) + const res = await secureApp.request('/api/v1/health', { + method: 'OPTIONS', + headers: { + Origin: 'https://docs.company.com', + 'Access-Control-Request-Method': 'GET', + }, + }) + expect(res.headers.get('access-control-allow-origin')).toBe('https://docs.company.com') + } finally { + await secureCtx.shutdown() + rmSync(secureDir, { recursive: true, force: true }) + } + }) + + it('enforces HTTPS for non-local team-mode API requests when configured', async () => { + const secureDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + const secureCtx = await bootstrap({ + dataDir: secureDir, + configOverrides: { + mode: 'team', + model: stubModel, + security: { transport: { enforceHTTPS: true } }, + } as any, + }) + try { + const secureApp = createApp(secureCtx) + const res = await secureApp.request('/api/v1/health', { + headers: { + Host: 'docs.company.com', + 'X-Forwarded-Proto': 'http', + }, + }, nodeEnv('198.51.100.7') as any) + expect(res.status).toBe(426) + } finally { + await secureCtx.shutdown() + rmSync(secureDir, { recursive: true, force: true }) + } + }) + + it('ignores spoofed HTTPS forwarding headers from untrusted clients', async () => { + const secureDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + const secureCtx = await bootstrap({ + dataDir: secureDir, + configOverrides: { + mode: 'team', + model: stubModel, + security: { transport: { enforceHTTPS: true } }, + } as any, + }) + try { + const secureApp = createApp(secureCtx) + const res = await secureApp.request('/api/v1/health', { + headers: { + Host: 'docs.company.com', + 'X-Forwarded-Proto': 'https', + }, + }, nodeEnv('198.51.100.7') as any) + expect(res.status).toBe(426) + } finally { + await secureCtx.shutdown() + rmSync(secureDir, { recursive: true, force: true }) + } + }) + + it('honors HTTPS forwarding headers from a configured trusted proxy', async () => { + const secureDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + const secureCtx = await bootstrap({ + dataDir: secureDir, + configOverrides: { + mode: 'team', + model: stubModel, + security: { transport: { enforceHTTPS: true, proxy: '10.0.0.1' } }, + } as any, + }) + try { + const { rawKey } = secureCtx.apiKeyManager.create({ + name: 'proxy-health', + workspaceId: secureCtx.workspaceManager.list()[0].id, + userId: 'user-1', + role: 'admin', + }) + const secureApp = createApp(secureCtx) + const res = await secureApp.request('/api/v1/health', { + headers: { + Host: 'docs.company.com', + 'X-Forwarded-Proto': 'https', + 'X-API-Key': rawKey, + }, + }, nodeEnv('10.0.0.1') as any) + expect(res.status).toBe(200) + } finally { + await secureCtx.shutdown() + rmSync(secureDir, { recursive: true, force: true }) + } + }) + + it('restricts widget script by configured referer domains', async () => { + const widgetDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + const widgetCtx = await bootstrap({ + dataDir: widgetDir, + configOverrides: { + model: stubModel, + security: { + transport: { + widgetAllowedDomains: ['https://docs.company.com'], + }, + }, + } as any, + }) + try { + const widgetApp = createApp(widgetCtx) + const blocked = await widgetApp.request('/widget.js', { + headers: { Referer: 'https://evil.example/page' }, + }) + const allowed = await widgetApp.request('/widget.js', { + headers: { Referer: 'https://docs.company.com/page' }, + }) + expect(blocked.status).toBe(403) + expect(allowed.status).toBe(200) + } finally { + await widgetCtx.shutdown() + rmSync(widgetDir, { recursive: true, force: true }) + } }) }) diff --git a/packages/server/tests/http/workbench.test.ts b/packages/server/tests/http/workbench.test.ts new file mode 100644 index 0000000..9eb4f81 --- /dev/null +++ b/packages/server/tests/http/workbench.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { bootstrap, type AppContext } from '../../src/bootstrap.js' +import { createApp } from '../../src/http/app.js' +import { mkdtempSync, rmSync } from 'node:fs' +import { join } from 'node:path' +import { tmpdir } from 'node:os' + +const stubModel = { + provider: 'stub', + llm: 'stub-llm', + embedding: 'stub-embedding', + apiKey: '', + baseUrl: '', + embeddingDimensions: 384, +} as any + +describe('Workbench Routes', () => { + let ctx: AppContext + let app: ReturnType + let tempDir: string + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + ctx = await bootstrap({ dataDir: tempDir, configOverrides: { model: stubModel } }) + app = createApp(ctx) + }) + + afterEach(async () => { + await ctx.shutdown() + rmSync(tempDir, { recursive: true, force: true }) + }) + + it('GET /api/v1/workbench returns a compact workspace summary', async () => { + const workspaceId = ctx.workspaceManager.list()[0].id + ctx.db.run( + `INSERT INTO query_logs (id, workspace_id, query, intent, profile, confidence_score, response_time_ms, route, feedback, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ['query-1', workspaceId, 'How does onboarding work?', 'general', 'balanced', 0.82, 240, 'rag', 'positive', new Date().toISOString()] + ) + + const res = await app.request('/api/v1/workbench') + expect(res.status).toBe(200) + + const body = await res.json() + expect(body.health).toMatchObject({ status: 'ok' }) + expect(body.corpus).toMatchObject({ + documents: 0, + chunks: 0, + sourceDistribution: {}, + statusDistribution: {}, + }) + expect(body.quality).toMatchObject({ + totalQueries: 1, + avgConfidence: 0.82, + avgResponseTimeMs: 240, + feedback: { positive: 1, negative: 0 }, + }) + expect(body.connectors).toMatchObject({ + total: 0, + active: 0, + recent: [], + }) + expect(body.workspace).toMatchObject({ + name: 'default', + mode: 'personal', + }) + expect(body.recentQueries).toEqual([ + expect.objectContaining({ + query: 'How does onboarding work?', + confidenceScore: 0.82, + profile: 'balanced', + }), + ]) + expect(body.suggestedQuestions.length).toBeGreaterThan(0) + }) +}) + +describe('Workbench Routes (team mode)', () => { + let ctx: AppContext + let app: ReturnType + let tempDir: string + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'opendocuments-test-')) + ctx = await bootstrap({ dataDir: tempDir, configOverrides: { mode: 'team', model: stubModel } }) + app = createApp(ctx) + }) + + afterEach(async () => { + await ctx.shutdown() + rmSync(tempDir, { recursive: true, force: true }) + }) + + it('requires an API key in team mode', async () => { + const res = await app.request('/api/v1/workbench') + expect(res.status).toBe(401) + }) +}) diff --git a/packages/server/tests/unsupported-connectors.test.ts b/packages/server/tests/unsupported-connectors.test.ts new file mode 100644 index 0000000..a7c87a1 --- /dev/null +++ b/packages/server/tests/unsupported-connectors.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest' + +const unsupportedConnectorPackages = [ + '@opendocuments/connector-discord', + '@opendocuments/connector-jira', + '@opendocuments/connector-linear', + '@opendocuments/connector-slack', +] + +describe('unsupported connector packages', () => { + it('are not importable from the workspace', async () => { + for (const packageName of unsupportedConnectorPackages) { + await expect(import(packageName)).rejects.toThrow() + } + }) +}) diff --git a/packages/web/package.json b/packages/web/package.json index c9c4e7b..3ee124e 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -10,6 +10,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "lucide-react": "^1.22.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-markdown": "^9.0.0", diff --git a/packages/web/src/App.tsx b/packages/web/src/App.tsx index d6d52c9..e00e6e2 100644 --- a/packages/web/src/App.tsx +++ b/packages/web/src/App.tsx @@ -11,13 +11,16 @@ import { WorkspacesPage } from './components/workspaces/WorkspacesPage' import { PluginsPage } from './components/plugins/PluginsPage' import { UnifiedDashboard } from './components/dashboard/UnifiedDashboard' import { LoginPage } from './components/auth/LoginPage' +import { CollectionsPage } from './components/collections/CollectionsPage' import { getHealth } from './lib/api' import { clearStoredApiKey, setStoredApiKey } from './lib/auth' +import { translate as tr } from './lib/i18n' const PAGES: Record React.ReactElement> = { dashboard: UnifiedDashboard, chat: ChatPage, documents: DocumentsPage, + collections: CollectionsPage, settings: SettingsPage, health: HealthPage, connectors: ConnectorsPage, @@ -26,14 +29,18 @@ const PAGES: Record React.ReactElement> = { } export function App() { - const { currentPage, effectiveTheme } = useAppStore() - const [authState, setAuthState] = useState<'checking' | 'authenticated' | 'unauthenticated'>('checking') + const { currentPage, effectiveTheme, locale } = useAppStore() + const [authState, setAuthState] = useState<'checking' | 'authenticated' | 'unauthenticated' | 'unavailable'>('checking') const [authError, setAuthError] = useState(null) useEffect(() => { document.documentElement.classList.toggle('dark', effectiveTheme === 'dark') }, [effectiveTheme]) + useEffect(() => { + document.documentElement.lang = locale + }, [locale]) + useEffect(() => { let cancelled = false @@ -52,7 +59,8 @@ export function App() { return } - setAuthState('authenticated') + setAuthState('unavailable') + setAuthError(error instanceof Error ? error.message : tr(locale, 'app.serverUnavailable')) } } @@ -74,7 +82,7 @@ export function App() { } catch (error) { clearStoredApiKey() setAuthState('unauthenticated') - setAuthError('Authentication failed. Check the API key or sign in again.') + setAuthError(tr(locale, 'login.authFailed')) } } @@ -83,7 +91,7 @@ export function App() { if (authState === 'checking') { return (
- Checking authentication... + {tr(locale, 'app.checkingAuth')}
) } @@ -92,6 +100,17 @@ export function App() { return } + if (authState === 'unavailable') { + return ( +
+
+

{tr(locale, 'app.notReachable')}

+

{authError}

+
+
+ ) + } + return ( <> diff --git a/packages/web/src/components/auth/LoginPage.tsx b/packages/web/src/components/auth/LoginPage.tsx index 385af5c..47c0d19 100644 --- a/packages/web/src/components/auth/LoginPage.tsx +++ b/packages/web/src/components/auth/LoginPage.tsx @@ -1,4 +1,7 @@ import { useState } from 'react' +import { GitBranch, KeyRound, Lock, ShieldCheck } from 'lucide-react' +import { useAppStore } from '../../stores/appStore' +import { translate as tr } from '../../lib/i18n' interface Props { onLogin: (apiKey: string) => void @@ -6,47 +9,96 @@ interface Props { } export function LoginPage({ onLogin, errorMessage }: Props) { + const { locale } = useAppStore() + const t = (key: string, values?: Record) => tr(locale, key, values) const [key, setKey] = useState('') + const submit = () => { + const trimmed = key.trim() + if (trimmed) onLogin(trimmed) + } + return ( -
-
-

OpenDocuments

+
+
+
+
+ +
+

+ {t('login.headline')} +

+

+ {t('login.subtitle')} +

+
+ {[ + [t('login.workspaceScoped'), t('login.workspaceScopedDesc')], + [t('login.apiProtected'), t('login.apiProtectedDesc')], + [t('login.selfHosted'), t('login.selfHostedDesc')], + ].map(([title, detail]) => ( +
+ +

{title}

+

{detail}

+
+ ))} +
+
+ +
+
+
+ +
+
+

{t('login.apiKeySignIn')}

+

{t('login.apiKeyHelp')}

+
+
-
{errorMessage && ( -
+
{errorMessage}
)} -
- + +
+ + -
- - + + + +

+ {t('login.help')} +

+
) diff --git a/packages/web/src/components/chat/ChatInput.tsx b/packages/web/src/components/chat/ChatInput.tsx index 4d1fbf5..f7b0c7d 100644 --- a/packages/web/src/components/chat/ChatInput.tsx +++ b/packages/web/src/components/chat/ChatInput.tsx @@ -1,16 +1,22 @@ import { useState, useRef, useEffect } from 'react' +import { Paperclip, Send } from 'lucide-react' import { useAppStore } from '../../stores/appStore' -import type { RAGProfile } from '../../lib/types' +import { translate as tr } from '../../lib/i18n' interface Props { onSend: (query: string) => void + onAttach?: (file: File) => Promise disabled?: boolean + uploading?: boolean + className?: string } -export function ChatInput({ onSend, disabled }: Props) { +export function ChatInput({ onSend, onAttach, disabled, uploading, className = '' }: Props) { + const { locale } = useAppStore() + const t = (key: string, values?: Record) => tr(locale, key, values) const [input, setInput] = useState('') const textareaRef = useRef(null) - const { profile, setProfile } = useAppStore() + const fileInputRef = useRef(null) useEffect(() => { if (!disabled) textareaRef.current?.focus() @@ -30,55 +36,62 @@ export function ChatInput({ onSend, disabled }: Props) { } } + const handleAttach = async (file: File | undefined) => { + if (!file || !onAttach || disabled || uploading) return + try { + await onAttach(file) + } finally { + if (fileInputRef.current) fileInputRef.current.value = '' + } + } + return ( -
-
+
+