From 6afdd7cf2d9492a0f0b4bd64185491b8d6492cbf Mon Sep 17 00:00:00 2001 From: dusan Date: Tue, 9 Jun 2026 19:39:54 +0200 Subject: [PATCH 1/4] Add advanced filter Signed-off-by: dusan --- internal/embedder/api/transport/chat.go | 10 + internal/embedder/api/transport/records.go | 4 + internal/embedder/domain/record.go | 2 + internal/embedder/postgres/records.go | 4 + .../postgres/sql/009_records_name_trgm.sql | 8 + ui/README.md | 20 ++ ui/package-lock.json | 49 ++- ui/package.json | 4 +- ui/src/components/AddSourceModal.tsx | 154 ++++++--- .../components/EditSourceSelectionModal.tsx | 74 +++-- ui/src/lib/embedder/service.ts | 22 +- ui/src/pages/ChatPage.tsx | 294 ++++++++++++++++-- ui/src/pages/HomePage.tsx | 8 +- 13 files changed, 516 insertions(+), 137 deletions(-) create mode 100644 internal/embedder/postgres/sql/009_records_name_trgm.sql diff --git a/internal/embedder/api/transport/chat.go b/internal/embedder/api/transport/chat.go index 6af6cd56..50a0e761 100644 --- a/internal/embedder/api/transport/chat.go +++ b/internal/embedder/api/transport/chat.go @@ -14,6 +14,12 @@ import ( "github.com/ultravioletrs/cube/internal/embedder/domain" ) +// maxChatRecordIDs bounds the explicit record scope a single chat request may +// carry. A larger allowlist bloats the request and the SQL IN (...) clause +// (Postgres caps bind parameters at 65535, and the plan degrades well before). +// Clients that want "all records" send no record_ids, which searches unscoped. +const maxChatRecordIDs = 1000 + // MountChat registers the streaming chat endpoint. func MountChat(r chi.Router, svc domain.ChatService, conversations domain.ConversationRepository) { r.Post("/api/v1/chat", chatHandler(svc, conversations)) @@ -41,6 +47,10 @@ func chatHandler(svc domain.ChatService, conversations domain.ConversationReposi writeJSON(w, http.StatusBadRequest, errBody("messages is required")) return } + if len(req.RecordIDs) > maxChatRecordIDs { + writeJSON(w, http.StatusBadRequest, errBody(fmt.Sprintf("record_ids exceeds limit of %d", maxChatRecordIDs))) + return + } // Ensure we have a conversation to save messages into. convID := req.ConversationID diff --git a/internal/embedder/api/transport/records.go b/internal/embedder/api/transport/records.go index dde9f841..d0229c35 100644 --- a/internal/embedder/api/transport/records.go +++ b/internal/embedder/api/transport/records.go @@ -6,6 +6,7 @@ package transport import ( "errors" "net/http" + "strings" "github.com/go-chi/chi/v5" "github.com/ultravioletrs/cube/internal/embedder/auth" @@ -224,6 +225,9 @@ func parseRecordFilter(r *http.Request) domain.RecordFilter { fmt := domain.RecordFormat(s) f.Format = &fmt } + if s := strings.TrimSpace(r.URL.Query().Get("q")); s != "" { + f.Name = &s + } return f } diff --git a/internal/embedder/domain/record.go b/internal/embedder/domain/record.go index 4583bfdd..d197830f 100644 --- a/internal/embedder/domain/record.go +++ b/internal/embedder/domain/record.go @@ -99,6 +99,8 @@ type RecordFilter struct { SourceID *string Status *RecordStatus Format *RecordFormat + // Name is a case-insensitive substring matched against the record name. + Name *string } // IngestResult holds post-ingestion metadata written back to the record. diff --git a/internal/embedder/postgres/records.go b/internal/embedder/postgres/records.go index d9a2d271..298c2f6e 100644 --- a/internal/embedder/postgres/records.go +++ b/internal/embedder/postgres/records.go @@ -72,6 +72,10 @@ func (r *recordsRepo) List( args = append(args, string(*f.Format)) conds = append(conds, fmt.Sprintf("r.format = $%d", len(args))) } + if f.Name != nil { + args = append(args, "%"+*f.Name+"%") + conds = append(conds, fmt.Sprintf("r.name ILIKE $%d", len(args))) + } where := strings.Join(conds, " AND ") diff --git a/internal/embedder/postgres/sql/009_records_name_trgm.sql b/internal/embedder/postgres/sql/009_records_name_trgm.sql new file mode 100644 index 00000000..0bad3219 --- /dev/null +++ b/internal/embedder/postgres/sql/009_records_name_trgm.sql @@ -0,0 +1,8 @@ +-- Copyright (c) Ultraviolet +-- SPDX-License-Identifier: Apache-2.0 + +-- Trigram index on record name to support fast case-insensitive ILIKE +-- substring search from the chat "Customize records" panel at scale. +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE INDEX IF NOT EXISTS records_name_trgm_idx + ON records USING GIN (name gin_trgm_ops); diff --git a/ui/README.md b/ui/README.md index 46b6e67a..96d8c4fb 100644 --- a/ui/README.md +++ b/ui/README.md @@ -67,6 +67,26 @@ npm run dev -- --host 0.0.0.0 | `npm run preview` | Serve the production build locally | | `npm run lint` | Run ESLint | +## Chat record selection + +Chat answers are grounded in indexed records (RAG). The records panel on the +right of the chat controls which records the model can retrieve from: + +- **All records** (default) — every indexed record is active. No selection + needed; the panel header shows `N active · all`. The chat sends no + `record_ids`, so the backend searches all records via its index (scales to + large corpora without enumerating the list). +- **Customized** — click **Customize** to limit the chat to a chosen allowlist. + The search box queries records by name on the server (paginated, so it works + past the client load cap). Toggle individual records, or **Select all** / + **Clear** to (de)select every record matching the current search. **Reset** + returns to all records. A selection is capped (1000 records); "all" is the way + to use more. +- **Scoped** — opening chat from a Source or Record pins it to that scope: + - A **source** scope limits the pool to that source's records, and can still + be narrowed further with the customize filter. + - A **single record** scope is locked and cannot be customized. + ## Tech stack - [React 19](https://react.dev/) + [TypeScript](https://www.typescriptlang.org/) diff --git a/ui/package-lock.json b/ui/package-lock.json index dcd2b6ae..ece49b7e 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -13,12 +13,14 @@ "@react-oauth/google": "^0.13.5", "@tailwindcss/vite": "^4.2.2", "@tanstack/react-query": "^5.99.2", + "@types/react-window": "^1.8.8", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^1.8.0", "react": "^19.2.5", "react-dom": "^19.2.5", "react-router-dom": "^7.14.1", + "react-window": "^2.2.7", "shadcn": "^4.3.1", "tailwind-merge": "^3.5.0", "tailwindcss": "^4.2.2", @@ -899,7 +901,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -916,7 +917,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -933,7 +933,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -950,7 +949,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -967,7 +965,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -984,7 +981,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1001,7 +997,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1018,7 +1013,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1035,7 +1029,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1052,7 +1045,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1069,7 +1061,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1086,7 +1077,6 @@ "cpu": [ "loong64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1103,7 +1093,6 @@ "cpu": [ "mips64el" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1120,7 +1109,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1137,7 +1125,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1154,7 +1141,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1171,7 +1157,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1188,7 +1173,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1205,7 +1189,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1222,7 +1205,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1239,7 +1221,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1256,7 +1237,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1273,7 +1253,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1290,7 +1269,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1307,7 +1285,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1324,7 +1301,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3224,7 +3200,6 @@ "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", - "devOptional": true, "license": "MIT", "dependencies": { "csstype": "^3.2.2" @@ -3240,6 +3215,15 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/react-window": { + "version": "1.8.8", + "resolved": "https://registry.npmjs.org/@types/react-window/-/react-window-1.8.8.tgz", + "integrity": "sha512-8Ls660bHR1AUA2kuRvVG9D/4XpRC6wjAaPT9dil7Ckc76eP9TKWZwwmgfq8Q1LANX3QNDnoU4Zp48A3w+zK69Q==", + "license": "MIT", + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/set-cookie-parser": { "version": "2.4.10", "resolved": "https://registry.npmjs.org/@types/set-cookie-parser/-/set-cookie-parser-2.4.10.tgz", @@ -4414,7 +4398,6 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "devOptional": true, "license": "MIT" }, "node_modules/data-uri-to-buffer": { @@ -7801,6 +7784,16 @@ "react-dom": ">=18" } }, + "node_modules/react-window": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/react-window/-/react-window-2.2.7.tgz", + "integrity": "sha512-SH5nvfUQwGHYyriDUAOt7wfPsfG9Qxd6OdzQxl5oQ4dsSsUicqQvjV7dR+NqZ4coY0fUn3w1jnC5PwzIUWEg5w==", + "license": "MIT", + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, "node_modules/recast": { "version": "0.23.11", "resolved": "https://registry.npmjs.org/recast/-/recast-0.23.11.tgz", diff --git a/ui/package.json b/ui/package.json index 0c22b470..6c46cb63 100644 --- a/ui/package.json +++ b/ui/package.json @@ -19,12 +19,14 @@ "@react-oauth/google": "^0.13.5", "@tailwindcss/vite": "^4.2.2", "@tanstack/react-query": "^5.99.2", + "@types/react-window": "^1.8.8", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^1.8.0", "react": "^19.2.5", "react-dom": "^19.2.5", "react-router-dom": "^7.14.1", + "react-window": "^2.2.7", "shadcn": "^4.3.1", "tailwind-merge": "^3.5.0", "tailwindcss": "^4.2.2", @@ -32,9 +34,9 @@ }, "devDependencies": { "@eslint/js": "^9.39.4", + "@playwright/test": "^1.57.0", "@testing-library/react": "^16.3.0", "@testing-library/user-event": "^14.6.1", - "@playwright/test": "^1.57.0", "@types/node": "^24.12.2", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", diff --git a/ui/src/components/AddSourceModal.tsx b/ui/src/components/AddSourceModal.tsx index 3a5c6b24..737d87bf 100644 --- a/ui/src/components/AddSourceModal.tsx +++ b/ui/src/components/AddSourceModal.tsx @@ -103,6 +103,9 @@ export default function AddSourceModal({ const [files, setFiles] = useState([]) const [selectedFileIDs, setSelectedFileIDs] = useState([]) const [selectedFileMetaByID, setSelectedFileMetaByID] = useState>({}) + const [selectedFolderIDs, setSelectedFolderIDs] = useState([]) + const [selectedFolderMetaByID, setSelectedFolderMetaByID] = useState>({}) + const [pickerFolderSelectionIDs, setPickerFolderSelectionIDs] = useState([]) const [fileSearch, setFileSearch] = useState('') const [pickerBrowseTab, setPickerBrowseTab] = useState('folders') @@ -207,6 +210,22 @@ export default function AddSourceModal({ clearSelectionError() } + function removeSelectedFolder(id: string) { + setSelectedFolderIDs(prev => prev.filter(folderID => folderID !== id)) + setSelectedFolderMetaByID(prev => { + if (!prev[id]) return prev + const next = { ...prev } + delete next[id] + return next + }) + clearSelectionError() + } + + function togglePickerFolder(folder: DriveFolderOption, checked: boolean) { + setPickerFolderSelectionIDs(prev => checked ? Array.from(new Set([...prev, folder.id])) : prev.filter(id => id !== folder.id)) + setSelectedFolderMetaByID(prev => ({ ...prev, [folder.id]: { id: folder.id, name: folder.name } })) + } + async function loadFolder( folderID: string, nextStack: Array<{ id: string; name: string }>, @@ -306,6 +325,9 @@ export default function AddSourceModal({ setFolderStack([]) setSelectedFileIDs([]) setSelectedFileMetaByID({}) + setSelectedFolderIDs([]) + setSelectedFolderMetaByID({}) + setPickerFolderSelectionIDs([]) await loadFolder('', []) } catch (err) { setFormError(err instanceof Error ? err.message : 'Failed to finish Google OAuth') @@ -326,8 +348,8 @@ export default function AddSourceModal({ if (!name.trim()) e.name = 'Required' if (providerTab === 'google') { if (!oauthConnected || !googleAccessToken) e.google = 'Connect Google Drive first' - if (importMode === 'selected' && selectedFileIDs.length === 0) { - e.selectedFileIDs = 'Select at least one file to sync' + if (importMode === 'selected' && selectedFileIDs.length === 0 && selectedFolderIDs.length === 0) { + e.selectedFileIDs = 'Select at least one file or folder to sync' } } else { if (!rcloneRemote.trim()) e.rcloneRemote = 'Rclone remote is required' @@ -359,7 +381,9 @@ export default function AddSourceModal({ clientId: '', clientSecret: '', selectedFileIDs: importMode === 'selected' ? selectedFileIDs : [], - selectedFolderIDs: importMode === 'all' && currentFolderID ? [currentFolderID] : [], + selectedFolderIDs: importMode === 'selected' + ? selectedFolderIDs + : (currentFolderID ? [currentFolderID] : []), syncEnabled, autoSyncInterval: syncEnabled ? Number.parseInt(autoSyncInterval, 10) : 0, rcloneRemote: '', @@ -409,6 +433,7 @@ export default function AddSourceModal({ async function openFilePicker() { setPickerSelectionIDs(selectedFileIDs) + setPickerFolderSelectionIDs(selectedFolderIDs) setShowFilePicker(true) if (!filesLoaded) { if (pickerBrowseTab === 'recent') { @@ -430,6 +455,7 @@ export default function AddSourceModal({ function savePickerSelection() { setSelectedFileIDs(Array.from(new Set(pickerSelectionIDs))) + setSelectedFolderIDs(Array.from(new Set(pickerFolderSelectionIDs))) clearSelectionError() setShowFilePicker(false) } @@ -719,37 +745,57 @@ export default function AddSourceModal({ -
0 ? '104px' : '84px', borderRadius: '8px', border: '1px dashed var(--border)', background: 'rgba(255,255,255,0.02)', padding: '10px', display: 'flex', flexDirection: 'column', gap: '8px' }}> - {selectedFiles.length === 0 ? ( +
0 || selectedFolderIDs.length > 0 ? '104px' : '84px', borderRadius: '8px', border: '1px dashed var(--border)', background: 'rgba(255,255,255,0.02)', padding: '10px', display: 'flex', flexDirection: 'column', gap: '8px' }}> + {selectedFiles.length === 0 && selectedFolderIDs.length === 0 ? (
- No files selected yet. + No files or folders selected yet.
) : ( - selectedFiles.map(file => ( -
- FILE - - {file.name} - - -
- )) + <> + {selectedFolderIDs.map(id => ( +
+ DIR + + {selectedFolderMetaByID[id]?.name ?? id} + recursive + + +
+ ))} + {selectedFiles.map(file => ( +
+ FILE + + {file.name} + + +
+ ))} + )}
-
{selectedFileIDs.length} files selected
+
{selectedFileIDs.length} files · {selectedFolderIDs.length} folders selected
- ))} + {shownFolders.map(folder => { + const isSharedDriveRoot = pickerBrowseTab === 'shared_drives' && !pickerSharedDriveID && folder.mimeType === 'application/vnd.google-apps.drive' + const folderChecked = pickerFolderSelectionIDs.includes(folder.id) + return ( +
+ {!isSharedDriveRoot && ( + togglePickerFolder(folder, e.target.checked)} + title="Select folder (recursive import)" + style={{ accentColor: 'var(--accent)', flexShrink: 0, cursor: 'pointer' }} + /> + )} + +
+ ) + })} {folders.length > shownFolders.length && ( - ))} + {shownFolders.map(folder => { + const isSharedDriveRoot = pickerBrowseTab === 'shared_drives' && !pickerSharedDriveID && folder.mimeType === 'application/vnd.google-apps.drive' + const folderChecked = selectedFolderIDs.includes(folder.id) + return ( +
+ {!isSharedDriveRoot && ( + toggleFolder(folder.id, e.target.checked)} + title="Select folder (recursive import)" + style={{ accentColor: 'var(--accent)', flexShrink: 0, cursor: 'pointer' }} + /> + )} + +
+ ) + })} {folders.length > shownFolders.length && ( +
+ ) +} interface ChatRouteState { source?: AppRecord @@ -238,7 +273,12 @@ function SourcesPanel({ indexedSources, activeSources, canCustomizeSources, + isCustomized, onToggle, + onSetActive, + onResetAll, + searchRecords, + recordCap, citations, debug, debugEnabled, @@ -250,7 +290,12 @@ function SourcesPanel({ indexedSources: AppRecord[] activeSources: string[] canCustomizeSources: boolean + isCustomized: boolean onToggle: (id: string) => void + onSetActive: (ids: string[]) => void + onResetAll: () => void + searchRecords: (q: string, offset: number, limit: number) => Promise + recordCap: number citations: MsgSource[] debug?: ChatDebug | null debugEnabled?: boolean @@ -260,12 +305,88 @@ function SourcesPanel({ visibleSourceSyncNotice?: { kind: 'info' | 'error'; text: string } | null }) { const citationSummary = citationCounts(citations) + const [customizeOpen, setCustomizeOpen] = useState(false) + const [query, setQuery] = useState('') + const [debouncedQuery, setDebouncedQuery] = useState('') + const [results, setResults] = useState([]) + const [total, setTotal] = useState(0) + const [loading, setLoading] = useState(false) + const [busy, setBusy] = useState(false) + const [error, setError] = useState(null) + const activeSet = useMemo(() => new Set(activeSources), [activeSources]) + + // Debounce the search box so each keystroke doesn't hit the backend. + useEffect(() => { + const t = setTimeout(() => setDebouncedQuery(query), 250) + return () => clearTimeout(t) + }, [query]) + + // Load the first page whenever the panel opens or the query changes. + useEffect(() => { + if (!customizeOpen) return + let cancelled = false + // eslint-disable-next-line react-hooks/set-state-in-effect + setLoading(true) + setError(null) + searchRecords(debouncedQuery, 0, RECORD_PAGE) + .then(page => { if (!cancelled) { setResults(page.records); setTotal(page.total) } }) + .catch(e => { if (!cancelled) { setResults([]); setTotal(0); setError(e instanceof Error ? e.message : 'Failed to load records') } }) + .finally(() => { if (!cancelled) setLoading(false) }) + return () => { cancelled = true } + }, [customizeOpen, debouncedQuery, searchRecords]) + + const loadMore = () => { + setBusy(true) + searchRecords(debouncedQuery, results.length, RECORD_PAGE) + .then(page => { setResults(prev => [...prev, ...page.records]); setTotal(page.total) }) + .catch(e => setError(e instanceof Error ? e.message : 'Failed to load records')) + .finally(() => setBusy(false)) + } + + // Resolve every record id matching the current query, bounded by the cap. + const matchIDs = async () => (await searchRecords(debouncedQuery, 0, recordCap)).records.map(r => r.id) + + const selectAllMatches = async () => { + setBusy(true) + setError(null) + try { + const ids = await matchIDs() + const union = Array.from(new Set([...activeSources, ...ids])) + if (union.length > recordCap) { + onSetActive(union.slice(0, recordCap)) + setError(`Selection capped at ${recordCap} records.`) + } else { + onSetActive(union) + if (total > ids.length) setError(`Selected first ${ids.length} of ${total} matches (cap ${recordCap}).`) + } + } catch (e) { + setError(e instanceof Error ? e.message : 'Failed to select records') + } finally { + setBusy(false) + } + } + + const clearMatches = async () => { + setBusy(true) + setError(null) + try { + const ids = new Set(await matchIDs()) + onSetActive(activeSources.filter(id => !ids.has(id))) + } catch (e) { + setError(e instanceof Error ? e.message : 'Failed to clear records') + } finally { + setBusy(false) + } + } + + const totalCount = customizeOpen ? total : indexedSources.length + const listHeight = Math.min(Math.max(results.length, 1) * 34, 264) return (
{/* Files section */}
-
+
RECORDS · {indexedSources.length} @@ -279,36 +400,104 @@ function SourcesPanel({ )}
+
+ + {isCustomized ? `${activeSources.length} of ${totalCount} active` : `${totalCount} active · all`} + + {canCustomizeSources && ( +
+ {isCustomized && ( + + )} + +
+ )} +
{visibleSourceSyncNotice && ( -
+
{visibleSourceSyncNotice.text}
)} + {customizeOpen && ( + <> +
+ + + + + setQuery(e.target.value)} + style={{ width: '100%', boxSizing: 'border-box', padding: '6px 8px 6px 26px', background: 'rgba(255,255,255,0.04)', border: '1px solid var(--border)', borderRadius: '6px', color: 'var(--text)', fontFamily: 'JetBrains Mono, monospace', fontSize: '10px', outline: 'none' }} + /> +
+
+ + +
+
+ {isCustomized ? 'Chat limited to selected records.' : 'All records active — select to limit the chat to them.'} +
+ {error && ( +
{error}
+ )} + + )}
-
0 ? '0 0 auto' : '1', maxHeight: citations.length > 0 ? '45%' : undefined, overflowY: 'auto', padding: '0 8px 8px' }}> - {indexedSources.length === 0 && ( -
No indexed files
- )} - {indexedSources.map(s => { - const active = activeSources.includes(s.id) - return ( + {customizeOpen && ( +
0 ? '0 0 auto' : '1', maxHeight: citations.length > 0 ? '45%' : undefined, overflowY: 'auto', padding: '0 8px 8px' }}> + {loading && results.length === 0 && ( +
Loading…
+ )} + {!loading && results.length === 0 && ( +
{debouncedQuery.trim() ? 'No matches' : 'No indexed records'}
+ )} + {results.length > 0 && ( + + )} + {results.length < total && ( - ) - })} -
+ )} +
+ )} {/* Citations section */} {citations.length > 0 && ( @@ -503,6 +692,19 @@ export default function ChatPage() { } }, [accessToken, domainID, conversationId, clearChatMessages, setConversations]) + // Scope-aware paged search of indexed records for the customize panel. + const searchIndexedRecords = useCallback( + (q: string, offset: number, limit: number) => + listRecordsPage(accessToken ?? '', domainID, { + status: 'indexed', + sourceID: selectedSourceID, + q, + offset, + limit, + }), + [accessToken, domainID, selectedSourceID], + ) + const scopedSourceState = selectedSourceID && sourceScopeState?.sourceID === selectedSourceID ? sourceScopeState : null @@ -520,19 +722,27 @@ export default function ChatPage() { () => records.filter(s => s.status === 'indexed' && (s.chunks ?? 0) > 0).map(s => s.id), [records], ) + // Pool of records the user may pick from: a scoped source's records, otherwise all indexed. + const customizableSourceIDs = useMemo( + () => (selectedSourceID ? selectedSourceRecordIDs : allIndexedSourceIDs), + [selectedSourceID, selectedSourceRecordIDs, allIndexedSourceIDs], + ) const activeSources = useMemo(() => { if (selectedRecord) { if (selectedRecordNotIndexed) return [] return [selectedRecord.id] } - if (selectedSourceID) return selectedSourceRecordIDs - if (!manualActiveSources) return allIndexedSourceIDs - return manualActiveSources.filter(id => allIndexedSourceIDs.includes(id)) - }, [allIndexedSourceIDs, manualActiveSources, selectedRecord, selectedRecordNotIndexed, selectedSourceID, selectedSourceRecordIDs]) + // null = all records (default); an explicit allowlist (possibly empty) is + // trusted as-is, since selections may include records beyond the page the + // client has loaded. + if (!manualActiveSources) return customizableSourceIDs + return manualActiveSources + }, [customizableSourceIDs, manualActiveSources, selectedRecord, selectedRecordNotIndexed]) const visibleSourceSyncNotice = sourceSyncNotice && selectedSourceID === sourceSyncNotice.sourceID ? sourceSyncNotice : null - const canCustomizeSources = !selectedRecord && !selectedSourceID + // Single-record scope stays locked; a source scope can still be narrowed. + const canCustomizeSources = !selectedRecord useEffect(() => { if (!selectedSourceID || !accessToken) return @@ -618,7 +828,14 @@ export default function ChatPage() { return } const userContent = input.trim() - const targetRecordIDs = activeSources + // When unscoped and not customized, send no record_ids so the backend searches + // all records via its index instead of enumerating the (client-capped) record list. + const isUnscoped = !selectedRecord && !selectedSourceID && manualActiveSources === null + const targetRecordIDs = isUnscoped ? [] : activeSources + if (targetRecordIDs.length > RECORD_SELECTION_CAP) { + setRetrievalWarning(`Too many records selected (${targetRecordIDs.length}). Narrow to ${RECORD_SELECTION_CAP} or fewer.`) + return + } setInput('') setLoading(true) setRetrievalWarning(null) @@ -691,9 +908,13 @@ export default function ChatPage() { } setLoading(false) }) - }, [input, loading, chatMessages, setChatMessages, accessToken, domainID, activeSources, selectedRecord, selectedRecordNotIndexed, selectedSourceID, selectedSourceHasNoIndexedRecords, conversationId, setConversationId, setConversations, modelConfig, debugEnabled]) + }, [input, loading, chatMessages, setChatMessages, accessToken, domainID, activeSources, manualActiveSources, selectedRecord, selectedRecordNotIndexed, selectedSourceID, selectedSourceHasNoIndexedRecords, conversationId, setConversationId, setConversations, modelConfig, debugEnabled]) - const indexedSources = records.filter(s => s.status === 'indexed') + const indexedSources = useMemo(() => { + const indexed = records.filter(s => s.status === 'indexed' && (s.chunks ?? 0) > 0) + if (selectedSourceID) return indexed.filter(s => selectedSourceRecordIDs.includes(s.id)) + return indexed + }, [records, selectedSourceID, selectedSourceRecordIDs]) const modelStatusInfo = modelStatusDetails(modelConfig, modelStatus) const configuredExternalProvider = externalProviderConfig?.provider ?? null @@ -711,9 +932,11 @@ export default function ChatPage() { } function handleToggleSource(id: string) { + // Toggling enters allowlist mode starting from empty, so the first pick + // narrows the chat to that record rather than to the client-loaded page. setManualActiveSources(prev => { - const base = prev ?? allIndexedSourceIDs - return activeSources.includes(id) ? base.filter(x => x !== id) : [...base, id] + const base = prev ?? [] + return base.includes(id) ? base.filter(x => x !== id) : [...base, id] }) } @@ -933,7 +1156,12 @@ export default function ChatPage() { indexedSources={indexedSources} activeSources={activeSources} canCustomizeSources={canCustomizeSources} + isCustomized={manualActiveSources !== null} onToggle={handleToggleSource} + onSetActive={setManualActiveSources} + onResetAll={() => setManualActiveSources(null)} + searchRecords={searchIndexedRecords} + recordCap={RECORD_SELECTION_CAP} citations={panelCitations} debug={panelDebug} debugEnabled={debugEnabled} diff --git a/ui/src/pages/HomePage.tsx b/ui/src/pages/HomePage.tsx index 28bb5a8b..4380e13e 100644 --- a/ui/src/pages/HomePage.tsx +++ b/ui/src/pages/HomePage.tsx @@ -374,10 +374,10 @@ export default function HomePage() { } } - async function handleSaveSourceSelection(source: DriveSource, selectedFileIDs: string[]) { + async function handleSaveSourceSelection(source: DriveSource, selectedFileIDs: string[], selectedFolderIDs: string[]) { if (!accessToken) return try { - await updateGoogleSourceSelection(accessToken, domainID, source.id, selectedFileIDs, source.selectedFolderIDs ?? []) + await updateGoogleSourceSelection(accessToken, domainID, source.id, selectedFileIDs, selectedFolderIDs) await handleRetrySourceSync(source.id) setLoadError('') await refreshData() @@ -754,8 +754,8 @@ export default function HomePage() { authToken={accessToken} source={editingSource} onClose={() => setEditingSource(null)} - onSave={async selectedFileIDs => { - await handleSaveSourceSelection(editingSource, selectedFileIDs) + onSave={async (selectedFileIDs, selectedFolderIDs) => { + await handleSaveSourceSelection(editingSource, selectedFileIDs, selectedFolderIDs) setEditingSource(null) }} /> From 0442ed44203addea38107aaab9dfb2124afafa44 Mon Sep 17 00:00:00 2001 From: dusan Date: Tue, 9 Jun 2026 21:56:24 +0200 Subject: [PATCH 2/4] feat(embedder): add Google Drive folder structure Capture each record's containing-folder path during the Drive recursive walk and persist it so records can be filtered and grouped by directory. - ingest: ListFilesRecursive carries the folder path via BFS; SourceFile and Record gain FolderPath/FolderID - schema: folder_path/folder_id columns + prefix index (migration 010); RecordFilter folder-prefix matching (folder and nested) - api: expose folder_path/folder_id; add ?folder= list filter - ui: folder filter in the chat customize panel; group-by-folder toggle on the records page Paths populate on the next sync; existing records stay unfiled until re-synced. --- internal/embedder/api/transport/records.go | 7 + internal/embedder/domain/record.go | 9 ++ internal/embedder/ingest/drive.go | 47 ++++++- internal/embedder/ingest/source_providers.go | 4 + .../ingest/sources/google/provider.go | 6 + internal/embedder/postgres/records.go | 38 +++++- .../postgres/sql/010_records_folder.sql | 13 ++ internal/embedder/service/source_sync.go | 11 ++ ui/src/lib/embedder/service.ts | 6 + ui/src/pages/ChatPage.tsx | 45 ++++-- ui/src/pages/RecordsPage.tsx | 129 ++++++++++++------ ui/src/types/index.ts | 2 + 12 files changed, 261 insertions(+), 56 deletions(-) create mode 100644 internal/embedder/postgres/sql/010_records_folder.sql diff --git a/internal/embedder/api/transport/records.go b/internal/embedder/api/transport/records.go index d0229c35..af00464b 100644 --- a/internal/embedder/api/transport/records.go +++ b/internal/embedder/api/transport/records.go @@ -47,6 +47,8 @@ type recordResponse struct { ExternalURL string `json:"external_url"` ExternalRef string `json:"external_ref,omitempty"` MimeType string `json:"mime_type,omitempty"` + FolderPath *string `json:"folder_path,omitempty"` + FolderID *string `json:"folder_id,omitempty"` Description string `json:"description,omitempty"` ChunkCount *int `json:"chunks,omitempty"` IngestTotalChunks *int `json:"ingest_total_chunks,omitempty"` @@ -79,6 +81,8 @@ func toRecordResponse(rec domain.Record) recordResponse { ExternalURL: rec.ExternalURL, ExternalRef: rec.ExternalRef, MimeType: rec.MimeType, + FolderPath: rec.FolderPath, + FolderID: rec.FolderID, Description: rec.Description, ChunkCount: rec.ChunkCount, IngestTotalChunks: rec.IngestTotalChunks, @@ -228,6 +232,9 @@ func parseRecordFilter(r *http.Request) domain.RecordFilter { if s := strings.TrimSpace(r.URL.Query().Get("q")); s != "" { f.Name = &s } + if s := strings.TrimSpace(r.URL.Query().Get("folder")); s != "" { + f.FolderPrefix = &s + } return f } diff --git a/internal/embedder/domain/record.go b/internal/embedder/domain/record.go index d197830f..0bd5ff94 100644 --- a/internal/embedder/domain/record.go +++ b/internal/embedder/domain/record.go @@ -57,6 +57,12 @@ type Record struct { ExternalRef string MimeType string + // FolderPath is the human-readable containing-folder path within the source + // (e.g. /Docs/2024/Q3); FolderID is the immediate parent folder ID. Both are + // populated for folder-tree ingests (Google Drive); nil otherwise. + FolderPath *string + FolderID *string + // Content metadata populated after successful ingestion. Description string ChunkCount *int @@ -101,6 +107,9 @@ type RecordFilter struct { Format *RecordFormat // Name is a case-insensitive substring matched against the record name. Name *string + // FolderPrefix matches records whose folder_path equals or is nested under + // the given path (prefix match), e.g. "/Docs/2024". + FolderPrefix *string } // IngestResult holds post-ingestion metadata written back to the record. diff --git a/internal/embedder/ingest/drive.go b/internal/embedder/ingest/drive.go index 8977dcbc..e540871e 100644 --- a/internal/embedder/ingest/drive.go +++ b/internal/embedder/ingest/drive.go @@ -64,6 +64,13 @@ type DriveFile struct { ModifiedTime string `json:"modifiedTime"` WebViewLink string `json:"webViewLink"` Parents []string `json:"parents"` + + // FolderPath and FolderID are populated by ListFilesRecursive while walking + // the folder tree. FolderPath is the human-readable path of the containing + // folder relative to the walked root (e.g. /Docs/2024/Q3); FolderID is the + // immediate parent folder ID. Empty for flat (whole-drive) listings. + FolderPath string `json:"-"` + FolderID string `json:"-"` } // ImageIngestMode describes which signals should be indexed for an image. @@ -234,7 +241,14 @@ func (d *DriveReader) ListFilesRecursive(ctx context.Context, folderID string) ( return d.ListFiles(ctx, "") } - queue := []string{rootID} + // BFS the folder tree carrying each folder's path so files can record the + // human-readable folder path of their container without extra lookups. + type folderNode struct { + id string + path string + } + rootPath := "/" + d.folderName(ctx, rootID) + queue := []folderNode{{id: rootID, path: rootPath}} seenFolders := map[string]struct{}{rootID: {}} filesByID := make(map[string]DriveFile) @@ -242,7 +256,7 @@ func (d *DriveReader) ListFilesRecursive(ctx context.Context, folderID string) ( current := queue[0] queue = queue[1:] - folders, files, err := d.ListFolderContent(ctx, current) + folders, files, err := d.ListFolderContent(ctx, current.id) if err != nil { return nil, err } @@ -251,9 +265,11 @@ func (d *DriveReader) ListFilesRecursive(ctx context.Context, folderID string) ( continue } seenFolders[folder.ID] = struct{}{} - queue = append(queue, folder.ID) + queue = append(queue, folderNode{id: folder.ID, path: current.path + "/" + folder.Name}) } for _, file := range files { + file.FolderPath = current.path + file.FolderID = current.id filesByID[file.ID] = file } } @@ -268,6 +284,31 @@ func (d *DriveReader) ListFilesRecursive(ctx context.Context, folderID string) ( return out, nil } +// folderName fetches a folder's display name. On any error it falls back to the +// folder ID so path construction stays best-effort and never blocks ingest. +func (d *DriveReader) folderName(ctx context.Context, folderID string) string { + params := url.Values{"fields": {"name"}, "supportsAllDrives": {"true"}} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, driveFilesURL+"/"+folderID+"?"+params.Encode(), http.NoBody) + if err != nil { + return folderID + } + resp, err := d.httpClient.Do(req) + if err != nil { + return folderID + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return folderID + } + var meta struct { + Name string `json:"name"` + } + if err := json.NewDecoder(resp.Body).Decode(&meta); err != nil || strings.TrimSpace(meta.Name) == "" { + return folderID + } + return meta.Name +} + // ListFolderContent returns direct children of a folder split into folders and // supported files. Empty folderID resolves to Drive root. func (d *DriveReader) ListFolderContent(ctx context.Context, folderID string) ([]DriveFile, []DriveFile, error) { diff --git a/internal/embedder/ingest/source_providers.go b/internal/embedder/ingest/source_providers.go index a0cb464b..2358169f 100644 --- a/internal/embedder/ingest/source_providers.go +++ b/internal/embedder/ingest/source_providers.go @@ -19,6 +19,10 @@ type SourceFile struct { MimeType string SourceVersion string SourceModifiedAt *time.Time + // FolderPath is the human-readable containing-folder path (e.g. /Docs/2024); + // FolderID is the immediate parent folder ID. Both optional / best-effort. + FolderPath string + FolderID string } // SourceProviderCapabilities describes what integration operations are supported. diff --git a/internal/embedder/ingest/sources/google/provider.go b/internal/embedder/ingest/sources/google/provider.go index c3e265de..17dd9c85 100644 --- a/internal/embedder/ingest/sources/google/provider.go +++ b/internal/embedder/ingest/sources/google/provider.go @@ -64,6 +64,10 @@ func (p *sourceProvider) ListFiles( out := make([]ingest.SourceFile, 0, len(files)) for _, file := range files { + folderID := file.FolderID + if folderID == "" && len(file.Parents) > 0 { + folderID = file.Parents[0] + } out = append(out, ingest.SourceFile{ ExternalID: file.ID, Name: file.Name, @@ -72,6 +76,8 @@ func (p *sourceProvider) ListFiles( MimeType: file.MimeType, SourceVersion: file.Version, SourceModifiedAt: parseRFC3339Ptr(file.ModifiedTime), + FolderPath: file.FolderPath, + FolderID: folderID, }) } return out, nil diff --git a/internal/embedder/postgres/records.go b/internal/embedder/postgres/records.go index 298c2f6e..78e218f0 100644 --- a/internal/embedder/postgres/records.go +++ b/internal/embedder/postgres/records.go @@ -28,6 +28,7 @@ func (r *recordsRepo) GetByID(ctx context.Context, id, domainID string) (domain. const q = ` SELECT r.id, r.domain_id, r.user_id, r.source_id, r.name, r.format, r.status, r.external_id, r.external_url, r.external_ref, r.mime_type, + r.folder_path, r.folder_id, r.description, r.chunk_count, r.size_bytes, r.page_count, r.ingest_total_chunks, r.ingest_indexed_chunks, r.ingest_stage, r.source_version, r.source_modified_at, r.error, @@ -76,6 +77,12 @@ func (r *recordsRepo) List( args = append(args, "%"+*f.Name+"%") conds = append(conds, fmt.Sprintf("r.name ILIKE $%d", len(args))) } + if f.FolderPrefix != nil { + // Match the folder itself and any nested folder via prefix on folder_path. + args = append(args, *f.FolderPrefix) + args = append(args, *f.FolderPrefix+"/%") + conds = append(conds, fmt.Sprintf("(r.folder_path = $%d OR r.folder_path LIKE $%d)", len(args)-1, len(args))) + } where := strings.Join(conds, " AND ") @@ -87,6 +94,7 @@ func (r *recordsRepo) List( q := fmt.Sprintf(` SELECT r.id, r.domain_id, r.user_id, r.source_id, r.name, r.format, r.status, r.external_id, r.external_url, r.external_ref, r.mime_type, + r.folder_path, r.folder_id, r.description, r.chunk_count, r.size_bytes, r.page_count, r.ingest_total_chunks, r.ingest_indexed_chunks, r.ingest_stage, r.source_version, r.source_modified_at, r.error, @@ -143,6 +151,8 @@ func scanRecord(row interface { externalURL pgtype.Text externalRef pgtype.Text mimeType pgtype.Text + folderPath pgtype.Text + folderID pgtype.Text description pgtype.Text chunkCount pgtype.Int4 sizeBytes pgtype.Int8 @@ -161,6 +171,7 @@ func scanRecord(row interface { if err := row.Scan( &rec.ID, &rec.DomainID, &rec.UserID, &sourceID, &rec.Name, &format, &status, &externalID, &externalURL, &externalRef, &mimeType, + &folderPath, &folderID, &description, &chunkCount, &sizeBytes, &pageCount, &ingestTotalChunks, &ingestIndexedChunks, &ingestStage, &sourceVersion, &sourceModifiedAt, &recError, @@ -189,6 +200,14 @@ func scanRecord(row interface { if mimeType.Valid { rec.MimeType = mimeType.String } + if folderPath.Valid { + s := folderPath.String + rec.FolderPath = &s + } + if folderID.Valid { + s := folderID.String + rec.FolderID = &s + } if description.Valid { rec.Description = description.String } @@ -242,10 +261,12 @@ func (r *recordsRepo) Create(ctx context.Context, rec domain.Record) (domain.Rec INSERT INTO records (domain_id, user_id, source_id, name, format, status, external_id, external_url, external_ref, mime_type, + folder_path, folder_id, source_version, source_modified_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING id, domain_id, user_id, source_id, name, format, status, external_id, external_url, external_ref, mime_type, + folder_path, folder_id, description, chunk_count, size_bytes, page_count, ingest_total_chunks, ingest_indexed_chunks, ingest_stage, source_version, source_modified_at, error, @@ -260,6 +281,7 @@ func (r *recordsRepo) Create(ctx context.Context, rec domain.Record) (domain.Rec row := r.pool.QueryRow(ctx, q, rec.DomainID, rec.UserID, rec.SourceID, rec.Name, string(rec.Format), string(rec.Status), rec.ExternalID, rec.ExternalURL, rec.ExternalRef, rec.MimeType, + rec.FolderPath, rec.FolderID, rec.SourceVersion, sourceModifiedAt, ) created, err := scanRecord(row) @@ -276,6 +298,7 @@ func (r *recordsRepo) ListQueued(ctx context.Context, limit int) ([]domain.Recor const q = ` SELECT r.id, r.domain_id, r.user_id, r.source_id, r.name, r.format, r.status, r.external_id, r.external_url, r.external_ref, r.mime_type, + r.folder_path, r.folder_id, r.description, r.chunk_count, r.size_bytes, r.page_count, r.ingest_total_chunks, r.ingest_indexed_chunks, r.ingest_stage, r.source_version, r.source_modified_at, r.error, @@ -386,6 +409,7 @@ func (r *recordsRepo) UpsertFromSource(ctx context.Context, rec domain.Record) ( const selectQ = ` SELECT r.id, r.domain_id, r.user_id, r.source_id, r.name, r.format, r.status, r.external_id, r.external_url, r.external_ref, r.mime_type, + r.folder_path, r.folder_id, r.description, r.chunk_count, r.size_bytes, r.page_count, r.ingest_total_chunks, r.ingest_indexed_chunks, r.ingest_stage, r.source_version, r.source_modified_at, r.error, @@ -440,10 +464,12 @@ func (r *recordsRepo) createInTx(ctx context.Context, tx pgx.Tx, rec domain.Reco INSERT INTO records (domain_id, user_id, source_id, name, format, status, external_id, external_url, external_ref, mime_type, + folder_path, folder_id, source_version, source_modified_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING id, domain_id, user_id, source_id, name, format, status, external_id, external_url, external_ref, mime_type, + folder_path, folder_id, description, chunk_count, size_bytes, page_count, ingest_total_chunks, ingest_indexed_chunks, ingest_stage, source_version, source_modified_at, error, @@ -458,6 +484,7 @@ func (r *recordsRepo) createInTx(ctx context.Context, tx pgx.Tx, rec domain.Reco created, err := scanRecord(tx.QueryRow(ctx, q, rec.DomainID, rec.UserID, rec.SourceID, rec.Name, string(rec.Format), string(rec.Status), rec.ExternalID, rec.ExternalURL, rec.ExternalRef, rec.MimeType, + rec.FolderPath, rec.FolderID, rec.SourceVersion, sourceModifiedAt, )) if err != nil { @@ -495,12 +522,15 @@ func (r *recordsRepo) updateFromSourceInTx( chunk_count = $9, size_bytes = $10, page_count = $11, + folder_path = $12, + folder_id = $13, ingest_total_chunks = NULL, ingest_indexed_chunks = NULL, updated_at = now() - WHERE id = $12 + WHERE id = $14 RETURNING id, domain_id, user_id, source_id, name, format, status, external_id, external_url, external_ref, mime_type, + folder_path, folder_id, description, chunk_count, size_bytes, page_count, ingest_total_chunks, ingest_indexed_chunks, ingest_stage, source_version, source_modified_at, error, @@ -529,7 +559,7 @@ func (r *recordsRepo) updateFromSourceInTx( updated, err := scanRecord(tx.QueryRow(ctx, baseQ, rec.Name, string(rec.Format), rec.ExternalURL, rec.ExternalRef, rec.MimeType, rec.SourceVersion, sourceModifiedAt, string(nextStatus), - chunkCount, sizeBytes, pageCount, id, + chunkCount, sizeBytes, pageCount, rec.FolderPath, rec.FolderID, id, )) if err != nil { return domain.Record{}, fmt.Errorf("update record from source: %w", err) diff --git a/internal/embedder/postgres/sql/010_records_folder.sql b/internal/embedder/postgres/sql/010_records_folder.sql new file mode 100644 index 00000000..50d411a9 --- /dev/null +++ b/internal/embedder/postgres/sql/010_records_folder.sql @@ -0,0 +1,13 @@ +-- Copyright (c) Ultraviolet +-- SPDX-License-Identifier: Apache-2.0 + +-- Folder structure for records. folder_path is the human-readable containing +-- folder path within the source (e.g. /Docs/2024/Q3); folder_id is the +-- immediate parent folder ID in the source system. Populated on sync for +-- folder-tree ingests (Google Drive); NULL for existing/flat records. +ALTER TABLE records ADD COLUMN IF NOT EXISTS folder_path TEXT; +ALTER TABLE records ADD COLUMN IF NOT EXISTS folder_id TEXT; + +-- text_pattern_ops supports fast prefix (LIKE '/Docs/2024%') folder filtering. +CREATE INDEX IF NOT EXISTS records_folder_path_idx + ON records (folder_path text_pattern_ops); diff --git a/internal/embedder/service/source_sync.go b/internal/embedder/service/source_sync.go index e7d668fd..30f42460 100644 --- a/internal/embedder/service/source_sync.go +++ b/internal/embedder/service/source_sync.go @@ -15,6 +15,15 @@ import ( embedmetrics "github.com/ultravioletrs/cube/internal/embedder/metrics" ) +// nonEmptyPtr returns a pointer to s, or nil when s is empty, so optional +// string columns stay NULL rather than storing empty strings. +func nonEmptyPtr(s string) *string { + if strings.TrimSpace(s) == "" { + return nil + } + return &s +} + type sourceSyncService struct { sources domain.SourceRepository records domain.RecordRepository @@ -100,6 +109,8 @@ func (s *sourceSyncService) Sync(ctx context.Context, id, domainID string) (res MimeType: file.MimeType, SourceVersion: file.SourceVersion, SourceModifiedAt: file.SourceModifiedAt, + FolderPath: nonEmptyPtr(file.FolderPath), + FolderID: nonEmptyPtr(file.FolderID), }) if err != nil { msg := err.Error() diff --git a/ui/src/lib/embedder/service.ts b/ui/src/lib/embedder/service.ts index d08b3a09..053103b0 100644 --- a/ui/src/lib/embedder/service.ts +++ b/ui/src/lib/embedder/service.ts @@ -18,6 +18,8 @@ interface RecordDTO { size_bytes?: number | null pages?: number | null external_url?: string + folder_path?: string | null + folder_id?: string | null } interface SourceDTO { @@ -215,6 +217,7 @@ export interface RecordListOptions { format?: RecordFormat | 'all' sourceID?: string q?: string + folder?: string limit?: number offset?: number } @@ -303,6 +306,8 @@ function mapRecord(dto: RecordDTO): AppRecord { pages: dto.pages ?? null, size: bytesToLabel(dto.size_bytes ?? undefined), url: dto.external_url || undefined, + folderPath: dto.folder_path ?? undefined, + folderID: dto.folder_id ?? undefined, } } @@ -419,6 +424,7 @@ function recordListPath(opts: RecordListOptions, status?: RawRecordStatus): stri if (status) params.set('status', status) if (opts.format && opts.format !== 'all') params.set('format', opts.format) if (opts.q && opts.q.trim()) params.set('q', opts.q.trim()) + if (opts.folder && opts.folder.trim()) params.set('folder', opts.folder.trim()) // A scoped list uses the per-source route; source_id stays off the query. const base = opts.sourceID ? `/api/v1/sources/${opts.sourceID}/records` : '/api/v1/records' diff --git a/ui/src/pages/ChatPage.tsx b/ui/src/pages/ChatPage.tsx index 3dcf3bfb..90a8b73b 100644 --- a/ui/src/pages/ChatPage.tsx +++ b/ui/src/pages/ChatPage.tsx @@ -294,7 +294,7 @@ function SourcesPanel({ onToggle: (id: string) => void onSetActive: (ids: string[]) => void onResetAll: () => void - searchRecords: (q: string, offset: number, limit: number) => Promise + searchRecords: (q: string, folder: string, offset: number, limit: number) => Promise recordCap: number citations: MsgSource[] debug?: ChatDebug | null @@ -308,6 +308,8 @@ function SourcesPanel({ const [customizeOpen, setCustomizeOpen] = useState(false) const [query, setQuery] = useState('') const [debouncedQuery, setDebouncedQuery] = useState('') + const [folder, setFolder] = useState('') + const [debouncedFolder, setDebouncedFolder] = useState('') const [results, setResults] = useState([]) const [total, setTotal] = useState(0) const [loading, setLoading] = useState(false) @@ -315,36 +317,40 @@ function SourcesPanel({ const [error, setError] = useState(null) const activeSet = useMemo(() => new Set(activeSources), [activeSources]) - // Debounce the search box so each keystroke doesn't hit the backend. + // Debounce the search box / folder filter so each keystroke doesn't hit the backend. useEffect(() => { - const t = setTimeout(() => setDebouncedQuery(query), 250) + const t = setTimeout(() => { setDebouncedQuery(query); setDebouncedFolder(folder) }, 250) return () => clearTimeout(t) - }, [query]) + }, [query, folder]) - // Load the first page whenever the panel opens or the query changes. + // Load the first page whenever the panel opens or the query/folder changes. useEffect(() => { if (!customizeOpen) return let cancelled = false // eslint-disable-next-line react-hooks/set-state-in-effect setLoading(true) setError(null) - searchRecords(debouncedQuery, 0, RECORD_PAGE) + searchRecords(debouncedQuery, debouncedFolder, 0, RECORD_PAGE) .then(page => { if (!cancelled) { setResults(page.records); setTotal(page.total) } }) .catch(e => { if (!cancelled) { setResults([]); setTotal(0); setError(e instanceof Error ? e.message : 'Failed to load records') } }) .finally(() => { if (!cancelled) setLoading(false) }) return () => { cancelled = true } - }, [customizeOpen, debouncedQuery, searchRecords]) + }, [customizeOpen, debouncedQuery, debouncedFolder, searchRecords]) const loadMore = () => { setBusy(true) - searchRecords(debouncedQuery, results.length, RECORD_PAGE) + searchRecords(debouncedQuery, debouncedFolder, results.length, RECORD_PAGE) .then(page => { setResults(prev => [...prev, ...page.records]); setTotal(page.total) }) .catch(e => setError(e instanceof Error ? e.message : 'Failed to load records')) .finally(() => setBusy(false)) } - // Resolve every record id matching the current query, bounded by the cap. - const matchIDs = async () => (await searchRecords(debouncedQuery, 0, recordCap)).records.map(r => r.id) + // Resolve every record id matching the current query/folder, bounded by the cap. + const matchIDs = async () => (await searchRecords(debouncedQuery, debouncedFolder, 0, recordCap)).records.map(r => r.id) + const folderOptions = useMemo( + () => Array.from(new Set(results.map(r => r.folderPath).filter((p): p is string => !!p))).sort(), + [results], + ) const selectAllMatches = async () => { setBusy(true) @@ -444,6 +450,22 @@ function SourcesPanel({ style={{ width: '100%', boxSizing: 'border-box', padding: '6px 8px 6px 26px', background: 'rgba(255,255,255,0.04)', border: '1px solid var(--border)', borderRadius: '6px', color: 'var(--text)', fontFamily: 'JetBrains Mono, monospace', fontSize: '10px', outline: 'none' }} />
+
+ + + + setFolder(e.target.value)} + style={{ width: '100%', boxSizing: 'border-box', padding: '6px 8px 6px 26px', background: 'rgba(255,255,255,0.04)', border: '1px solid var(--border)', borderRadius: '6px', color: 'var(--text)', fontFamily: 'JetBrains Mono, monospace', fontSize: '10px', outline: 'none' }} + /> + + {folderOptions.map(p => +
+
+ +
+
{record.name}
+
{recordSubtext(record)}
+
+
+
+
{recordDetail(record)}
+
{record.createdAt}
+ +
+ ) + return (
@@ -701,6 +764,16 @@ export default function RecordsPage() { Clear filters )} +
Showing {filtered.length} of {records.length} @@ -768,46 +841,26 @@ export default function RecordsPage() {
- {filtered.map(record => ( -
setSelectedId(selectedId === record.id ? null : record.id)} - style={{ display: 'flex', alignItems: 'center', padding: '14px 32px', borderBottom: '1px solid var(--border)', cursor: 'pointer', transition: 'all 0.15s ease', gap: '8px', background: selectedId === record.id ? 'rgba(0,212,180,0.05)' : 'transparent', borderLeft: selectedId === record.id ? '2px solid var(--accent)' : '2px solid transparent' }} - onMouseEnter={e => { if (selectedId !== record.id) (e.currentTarget as HTMLDivElement).style.background = 'rgba(255,255,255,0.03)' }} - onMouseLeave={e => { if (selectedId !== record.id) (e.currentTarget as HTMLDivElement).style.background = 'transparent' }} - > - -
- -
-
{record.name}
-
{recordSubtext(record)}
+ {!groupByFolder && filtered.map(renderRecordRow)} + + {groupByFolder && folderGroups.map(([key, recs]) => { + const collapsed = collapsedFolders.has(key) + return ( +
+
toggleFolderCollapsed(key)} + style={{ display: 'flex', alignItems: 'center', gap: '8px', padding: '8px 32px', borderBottom: '1px solid var(--border)', background: 'rgba(255,255,255,0.02)', cursor: 'pointer', position: 'sticky', top: 0, zIndex: 1 }} + > + + + + {key} + · {recs.length}
+ {!collapsed && recs.map(renderRecordRow)}
-
-
{recordDetail(record)}
-
{record.createdAt}
- -
- ))} + ) + })} {filtered.length === 0 && hasLoaded && (
diff --git a/ui/src/types/index.ts b/ui/src/types/index.ts index 7fdc9359..7e33584b 100644 --- a/ui/src/types/index.ts +++ b/ui/src/types/index.ts @@ -67,6 +67,8 @@ export interface AppRecord { size?: string pages?: number | null url?: string + folderPath?: string + folderID?: string } export interface Conversation { From 275df12b9e6972e8c210cc5e783ae1fb671c2a0f Mon Sep 17 00:00:00 2001 From: dusan Date: Tue, 9 Jun 2026 22:14:23 +0200 Subject: [PATCH 3/4] Fix prod and UI filter clear Signed-off-by: dusan --- docker/traefik/dynamic.toml | 2 +- internal/embedder/ingest/drive.go | 37 +++++++++ .../ingest/sources/google/provider.go | 6 ++ .../ingest/sources/google/provider_test.go | 83 +++++++++++++++++++ ui/src/pages/ChatPage.tsx | 7 +- 5 files changed, 133 insertions(+), 2 deletions(-) diff --git a/docker/traefik/dynamic.toml b/docker/traefik/dynamic.toml index d3e2257b..73ebbf14 100644 --- a/docker/traefik/dynamic.toml +++ b/docker/traefik/dynamic.toml @@ -28,7 +28,7 @@ [http.services.ui.loadBalancer] [[http.services.ui.loadBalancer.servers]] - url = "http://cube-ui:5173" + url = "http://ui:5173" [http.services.ui.loadBalancer.healthCheck] scheme = "http" path = "/health" diff --git a/internal/embedder/ingest/drive.go b/internal/embedder/ingest/drive.go index e540871e..9d2dc3c4 100644 --- a/internal/embedder/ingest/drive.go +++ b/internal/embedder/ingest/drive.go @@ -233,6 +233,43 @@ func (d *DriveReader) ListFiles(ctx context.Context, folderID string) ([]DriveFi return all, nil } +// GetFile returns Drive metadata for a single file ID. +func (d *DriveReader) GetFile(ctx context.Context, fileID string) (DriveFile, error) { + id := strings.TrimSpace(fileID) + if id == "" { + return DriveFile{}, fmt.Errorf("drive file id is required") + } + + params := url.Values{ + "fields": {"id,name,mimeType,version,modifiedTime,webViewLink,parents"}, + "supportsAllDrives": {"true"}, + } + reqURL := strings.TrimRight(driveFilesURL, "/") + "/" + url.PathEscape(id) + "?" + params.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, http.NoBody) + if err != nil { + return DriveFile{}, fmt.Errorf("drive get file request: %w", err) + } + resp, err := d.httpClient.Do(req) + if err != nil { + return DriveFile{}, fmt.Errorf("drive get file: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return DriveFile{}, fmt.Errorf("drive get file status %d: %s", resp.StatusCode, body) + } + + var file DriveFile + if err := json.NewDecoder(resp.Body).Decode(&file); err != nil { + return DriveFile{}, fmt.Errorf("drive get file decode: %w", err) + } + if !supportsDriveFile(file) { + return DriveFile{}, fmt.Errorf("drive file %s has unsupported MIME type %q", id, file.MimeType) + } + return file, nil +} + // ListFilesRecursive returns supported files contained in folderID and all of // its descendant folders. func (d *DriveReader) ListFilesRecursive(ctx context.Context, folderID string) ([]DriveFile, error) { diff --git a/internal/embedder/ingest/sources/google/provider.go b/internal/embedder/ingest/sources/google/provider.go index 17dd9c85..c985eab6 100644 --- a/internal/embedder/ingest/sources/google/provider.go +++ b/internal/embedder/ingest/sources/google/provider.go @@ -162,7 +162,13 @@ func applyDriveSelection( for _, id := range selectedFiles { if file, ok := baseByID[id]; ok { collected[file.ID] = file + continue + } + file, err := reader.GetFile(ctx, id) + if err != nil { + return nil, err } + collected[file.ID] = file } for _, folderID := range selectedFolders { diff --git a/internal/embedder/ingest/sources/google/provider_test.go b/internal/embedder/ingest/sources/google/provider_test.go index db63f2c8..a5a7796a 100644 --- a/internal/embedder/ingest/sources/google/provider_test.go +++ b/internal/embedder/ingest/sources/google/provider_test.go @@ -108,3 +108,86 @@ func TestGoogleSourceProvider_ListAndDownload_Smoke(t *testing.T) { t.Fatalf("expected nil page count, got %d", *pageCount) } } + +func TestGoogleSourceProvider_SelectedFileOutsideConfiguredFolder(t *testing.T) { + const accessToken = "google-test-token" + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := strings.TrimSpace(r.Header.Get("Authorization")); got != "Bearer "+accessToken { + http.Error(w, `{"error":"unauthorized"}`, http.StatusUnauthorized) + return + } + + switch { + case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files": + _, _ = io.WriteString(w, `{ + "files":[ + { + "id":"local-file", + "name":"local.txt", + "mimeType":"text/plain", + "version":"1", + "modifiedTime":"2026-05-12T10:00:00Z", + "webViewLink":"https://drive.example.local/local-file", + "parents":["configured-folder"] + } + ] + }`) + return + case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files/nested-file": + if r.URL.Query().Get("fields") == "" { + http.Error(w, "missing fields", http.StatusBadRequest) + return + } + _, _ = io.WriteString(w, `{ + "id":"nested-file", + "name":"nested.md", + "mimeType":"text/markdown", + "version":"7", + "modifiedTime":"2026-05-12T11:00:00Z", + "webViewLink":"https://drive.example.local/nested-file", + "parents":["other-folder"] + }`) + return + default: + http.Error(w, "not found", http.StatusNotFound) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + return + } + })) + defer srv.Close() + + restore := ingest.SetDriveAPIEndpoints( + srv.URL+"/drive/v3/files", + srv.URL+"/drive/v3/files/%s/export", + srv.URL+"/drive/v3/files/%s?alt=media", + ) + defer restore() + + cfgRaw, err := json.Marshal(domain.GoogleDriveConfig{ + AccessToken: accessToken, + FolderID: "configured-folder", + SelectedFileIDs: []string{"nested-file"}, + }) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + + files, err := google.NewSourceProvider().ListFiles(context.Background(), "user-1", domain.Source{ + ID: "src-g2", + Type: domain.SourceTypeGoogleDrive, + Config: cfgRaw, + }) + if err != nil { + t.Fatalf("ListFiles returned error: %v", err) + } + if len(files) != 1 { + t.Fatalf("expected 1 selected file, got %d: %#v", len(files), files) + } + if files[0].ExternalID != "nested-file" { + t.Fatalf("unexpected external_id: %q", files[0].ExternalID) + } + if files[0].FolderID != "other-folder" { + t.Fatalf("unexpected folder id: %q", files[0].FolderID) + } +} diff --git a/ui/src/pages/ChatPage.tsx b/ui/src/pages/ChatPage.tsx index 90a8b73b..fedd14c8 100644 --- a/ui/src/pages/ChatPage.tsx +++ b/ui/src/pages/ChatPage.tsx @@ -357,7 +357,8 @@ function SourcesPanel({ setError(null) try { const ids = await matchIDs() - const union = Array.from(new Set([...activeSources, ...ids])) + const base = isCustomized ? activeSources : [] + const union = Array.from(new Set([...base, ...ids])) if (union.length > recordCap) { onSetActive(union.slice(0, recordCap)) setError(`Selection capped at ${recordCap} records.`) @@ -851,6 +852,10 @@ export default function ChatPage() { return } const userContent = input.trim() + if (!selectedRecord && manualActiveSources !== null && manualActiveSources.length === 0) { + setRetrievalWarning('No records selected. Choose at least one record or reset to all records.') + return + } // When unscoped and not customized, send no record_ids so the backend searches // all records via its index instead of enumerating the (client-capped) record list. const isUnscoped = !selectedRecord && !selectedSourceID && manualActiveSources === null From 1830926a857143bedba7976aef51fcf6de56eff9 Mon Sep 17 00:00:00 2001 From: dusan Date: Wed, 10 Jun 2026 09:00:44 +0200 Subject: [PATCH 4/4] Improve stale ID handling Signed-off-by: dusan --- internal/embedder/ingest/drive.go | 14 +++- .../ingest/sources/google/provider.go | 6 ++ .../ingest/sources/google/provider_test.go | 67 +++++++++++++++++++ ui/README.md | 12 ++-- 4 files changed, 92 insertions(+), 7 deletions(-) diff --git a/internal/embedder/ingest/drive.go b/internal/embedder/ingest/drive.go index 9d2dc3c4..7542ca07 100644 --- a/internal/embedder/ingest/drive.go +++ b/internal/embedder/ingest/drive.go @@ -10,6 +10,7 @@ import ( "encoding/base64" "encoding/json" "encoding/xml" + "errors" "fmt" "io" "net/http" @@ -37,6 +38,14 @@ var ( driveDownloadURL = "https://www.googleapis.com/drive/v3/files/%s?alt=media" ) +var ( + // ErrDriveNotFound is returned when a Drive item no longer exists (404/410), + // so callers can skip stale references instead of failing the whole sync. + ErrDriveNotFound = errors.New("drive file not found") + // ErrUnsupportedDriveFile is returned when a file's MIME type cannot be ingested. + ErrUnsupportedDriveFile = errors.New("drive file has unsupported MIME type") +) + // SetDriveAPIEndpoints overrides Drive API endpoints and returns a restore function. // Intended for tests that need deterministic HTTP fixtures. func SetDriveAPIEndpoints(filesURL, exportURLFmt, downloadURLFmt string) func() { @@ -255,6 +264,9 @@ func (d *DriveReader) GetFile(ctx context.Context, fileID string) (DriveFile, er } defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone { + return DriveFile{}, fmt.Errorf("drive get file %s: %w", id, ErrDriveNotFound) + } if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) return DriveFile{}, fmt.Errorf("drive get file status %d: %s", resp.StatusCode, body) @@ -265,7 +277,7 @@ func (d *DriveReader) GetFile(ctx context.Context, fileID string) (DriveFile, er return DriveFile{}, fmt.Errorf("drive get file decode: %w", err) } if !supportsDriveFile(file) { - return DriveFile{}, fmt.Errorf("drive file %s has unsupported MIME type %q", id, file.MimeType) + return DriveFile{}, fmt.Errorf("drive file %s (%q): %w", id, file.MimeType, ErrUnsupportedDriveFile) } return file, nil } diff --git a/internal/embedder/ingest/sources/google/provider.go b/internal/embedder/ingest/sources/google/provider.go index c985eab6..d6e61455 100644 --- a/internal/embedder/ingest/sources/google/provider.go +++ b/internal/embedder/ingest/sources/google/provider.go @@ -6,6 +6,7 @@ package google import ( "context" "encoding/json" + "errors" "fmt" "strings" "time" @@ -166,6 +167,11 @@ func applyDriveSelection( } file, err := reader.GetFile(ctx, id) if err != nil { + // Skip a selected file that has gone away or can't be ingested rather + // than failing the entire sync over one stale reference. + if errors.Is(err, ingest.ErrDriveNotFound) || errors.Is(err, ingest.ErrUnsupportedDriveFile) { + continue + } return nil, err } collected[file.ID] = file diff --git a/internal/embedder/ingest/sources/google/provider_test.go b/internal/embedder/ingest/sources/google/provider_test.go index a5a7796a..6b5a18c2 100644 --- a/internal/embedder/ingest/sources/google/provider_test.go +++ b/internal/embedder/ingest/sources/google/provider_test.go @@ -191,3 +191,70 @@ func TestGoogleSourceProvider_SelectedFileOutsideConfiguredFolder(t *testing.T) t.Fatalf("unexpected folder id: %q", files[0].FolderID) } } + +func TestGoogleSourceProvider_SkipsStaleSelectedFile(t *testing.T) { + const accessToken = "google-test-token" + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := strings.TrimSpace(r.Header.Get("Authorization")); got != "Bearer "+accessToken { + http.Error(w, `{"error":"unauthorized"}`, http.StatusUnauthorized) + return + } + + switch { + case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files": + _, _ = io.WriteString(w, `{"files":[]}`) + return + case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files/live-file": + _, _ = io.WriteString(w, `{ + "id":"live-file", + "name":"live.txt", + "mimeType":"text/plain", + "version":"1", + "modifiedTime":"2026-05-12T11:00:00Z", + "webViewLink":"https://drive.example.local/live-file", + "parents":["some-folder"] + }`) + return + case r.Method == http.MethodGet && r.URL.Path == "/drive/v3/files/gone-file": + http.Error(w, `{"error":"not found"}`, http.StatusNotFound) + return + default: + http.Error(w, "not found", http.StatusNotFound) + t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) + return + } + })) + defer srv.Close() + + restore := ingest.SetDriveAPIEndpoints( + srv.URL+"/drive/v3/files", + srv.URL+"/drive/v3/files/%s/export", + srv.URL+"/drive/v3/files/%s?alt=media", + ) + defer restore() + + cfgRaw, err := json.Marshal(domain.GoogleDriveConfig{ + AccessToken: accessToken, + FolderID: "configured-folder", + SelectedFileIDs: []string{"live-file", "gone-file"}, + }) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + + files, err := google.NewSourceProvider().ListFiles(context.Background(), "user-1", domain.Source{ + ID: "src-g3", + Type: domain.SourceTypeGoogleDrive, + Config: cfgRaw, + }) + if err != nil { + t.Fatalf("ListFiles returned error: %v", err) + } + if len(files) != 1 { + t.Fatalf("expected 1 file (stale skipped), got %d: %#v", len(files), files) + } + if files[0].ExternalID != "live-file" { + t.Fatalf("unexpected external_id: %q", files[0].ExternalID) + } +} diff --git a/ui/README.md b/ui/README.md index 96d8c4fb..ba79f16b 100644 --- a/ui/README.md +++ b/ui/README.md @@ -60,12 +60,12 @@ npm run dev -- --host 0.0.0.0 ## Available scripts -| Script | Description | -|--------|-------------| -| `npm run dev` | Start development server with HMR | -| `npm run build` | Type-check and build for production (output: `dist/`) | -| `npm run preview` | Serve the production build locally | -| `npm run lint` | Run ESLint | +| Script | Description | +| ----------------- | ----------------------------------------------------- | +| `npm run dev` | Start development server with HMR | +| `npm run build` | Type-check and build for production (output: `dist/`) | +| `npm run preview` | Serve the production build locally | +| `npm run lint` | Run ESLint | ## Chat record selection