diff --git a/README.md b/README.md index ee952317..9e64bc63 100644 --- a/README.md +++ b/README.md @@ -372,9 +372,14 @@ This test uses your normal Claude Code auth and writes small test transcripts to Manual indexing tools for bulk operations and maintenance. See `episodic-memory index --help` for full options. Common operations: -- `--cleanup` - Index all unprocessed conversations -- `--verify` - Check index health -- `--repair` - Fix detected issues +- `--cleanup` - Index unprocessed exchanges; summaries may call an AI provider unless disabled +- `--verify` - Distinguish missing summaries, unindexed archives, appended exchanges, and archive mtime refreshes +- `--repair` - Re-embed and regenerate summaries for repair candidates; review the scope and provider cost first + +An archive mtime refresh without new exchanges is reported separately and is +not an outdated-index repair target. An unindexed archive is reported, but +that status alone does not select it for `--repair`; a separate overlapping +repair category may still cause it to be processed. ### `episodic-memory search` diff --git a/cli/index-conversations.js b/cli/index-conversations.js index 4d5f5bf5..77ea6f10 100755 --- a/cli/index-conversations.js +++ b/cli/index-conversations.js @@ -50,16 +50,16 @@ USAGE: COMMANDS: (default) Index all conversations - --cleanup Process only unindexed conversations (fast, cheap) + --cleanup Process unindexed exchanges (may generate AI summaries) --session ID Index specific session (used by hook) - --verify Check index health - --repair Fix detected issues + --verify Classify index and summary state + --repair Re-embed and summarize repair candidates (review first) --rebuild Delete DB and re-index everything (requires confirmation) OPTIONS: --concurrency N Parallel summarization (1-16, default: 1) -c N Short form of --concurrency - --no-summaries Skip AI summary generation (free, but no summaries in results) + --no-summaries Skip AI summary provider calls --help, -h Show this help EXAMPLES: @@ -69,13 +69,13 @@ EXAMPLES: # Index with 8 parallel summarizations (8x faster) index-conversations --cleanup --concurrency 8 - # Index without AI summaries (free, fast) + # Index without AI summary provider calls index-conversations --cleanup --no-summaries # Check index health index-conversations --verify - # Fix any issues found + # Review candidates and provider cost before broad repair index-conversations --repair # Nuclear option (deletes everything, re-indexes) @@ -85,7 +85,7 @@ WORKFLOW: 1. Initial setup: index-conversations --cleanup 2. Ongoing: Auto-indexed by sessionEnd hook 3. Health check: index-conversations --verify (weekly) - 4. Recovery: index-conversations --repair (if issues found) + 4. Recovery: review exact verifier categories before repair SEE ALSO: INDEXING.md - Setup and maintenance guide diff --git a/dist/index-cli.js b/dist/index-cli.js index 950a4c7d..fbdb3ba3 100644 --- a/dist/index-cli.js +++ b/dist/index-cli.js @@ -7,16 +7,16 @@ import path from 'path'; import { getSyncLockPath } from './logging.js'; import { acquireFileLock, readLockHolder, releaseFileLock } from './file-lock.js'; const command = process.argv[2]; -// Serialize the whole CLI with `sync-cli`: every command can initialize or -// migrate the database, and rebuild also deletes the database and summaries. +// Serialize the whole CLI with `sync-cli`: writers mutate the index/archive, +// and verification must not claim a consistent result while one is active. const syncLockPath = getSyncLockPath(); const syncLock = acquireFileLock(syncLockPath); if (!syncLock) { const holder = readLockHolder(syncLockPath); const holderLabel = holder !== null ? `pid ${holder}` : 'another process'; - // stderr keeps this out of stdout consumers; status 0 so hooks don't fail. + // A skipped write may be retried by a hook; a skipped verifier is not a result. console.error(`episodic-memory: sync already running (${holderLabel}); skipping`); - process.exit(0); + process.exit(command === 'verify' ? 2 : 0); } const releaseSyncLockOnce = () => { if (releaseSyncLockOnce.done) @@ -63,15 +63,25 @@ async function main() { const issues = await verifyIndex(); console.log('\n=== Verification Results ==='); console.log(`Missing summaries: ${issues.missing.length}`); + console.log(`Unindexed archives: ${issues.unindexed.length}`); console.log(`Orphaned entries: ${issues.orphaned.length}`); console.log(`Outdated files: ${issues.outdated.length}`); + console.log(`Archive refreshes (no new exchanges): ${issues.archiveRefreshes.length}`); console.log(`Corrupted files: ${issues.corrupted.length}`); if (issues.missing.length > 0) { console.log('\nMissing summaries:'); issues.missing.forEach(m => console.log(` ${m.path}`)); } - if (issues.missing.length + issues.orphaned.length + issues.outdated.length + issues.corrupted.length > 0) { - console.log('\nRun with --repair to fix these issues.'); + if (issues.unindexed.length > 0) { + console.log('\nUnindexed archives:'); + issues.unindexed.forEach(item => console.log(` ${item.path}`)); + } + if (issues.outdated.length > 0) { + console.log('\nArchives with appended exchanges:'); + issues.outdated.forEach(item => console.log(` ${item.path}`)); + } + if (issues.missing.length + issues.unindexed.length + issues.orphaned.length + issues.outdated.length + issues.corrupted.length > 0) { + console.log('\nReview exact candidates before repair; --repair may regenerate summaries and embeddings.'); process.exit(1); } else { @@ -81,10 +91,14 @@ async function main() { case 'repair': console.log('Verifying conversation index...'); const repairIssues = await verifyIndex(); + if (repairIssues.unindexed.length > 0) { + console.log(`${repairIssues.unindexed.length} unindexed archive(s) require separate review; --repair does not select them by this status alone.`); + process.exitCode = 1; + } if (repairIssues.missing.length + repairIssues.orphaned.length + repairIssues.outdated.length > 0) { await repairIndex(repairIssues); } - else { + else if (repairIssues.unindexed.length === 0) { console.log('āœ… No issues to repair!'); } break; diff --git a/dist/verify.d.ts b/dist/verify.d.ts index 9558c879..44208d16 100644 --- a/dist/verify.d.ts +++ b/dist/verify.d.ts @@ -3,6 +3,9 @@ export interface VerificationResult { path: string; reason: string; }>; + unindexed: Array<{ + path: string; + }>; orphaned: Array<{ uuid: string; path: string; @@ -12,10 +15,15 @@ export interface VerificationResult { fileTime: number; dbTime: number; }>; + archiveRefreshes: Array<{ + path: string; + fileTime: number; + dbTime: number; + }>; corrupted: Array<{ path: string; error: string; }>; } export declare function verifyIndex(): Promise; -export declare function repairIndex(issues: VerificationResult): Promise; +export declare function repairIndex(issues: Pick): Promise; diff --git a/dist/verify.js b/dist/verify.js index dd4535d2..c84ed676 100644 --- a/dist/verify.js +++ b/dist/verify.js @@ -1,14 +1,17 @@ import fs from 'fs'; import path from 'path'; +import Database from 'better-sqlite3'; import { parseConversation } from './parser.js'; -import { initDatabase, getAllExchanges, getFileLastIndexed } from './db.js'; -import { getArchiveDir, getExcludedProjects, findJsonlFiles, statIfExists } from './paths.js'; +import { getAllExchanges } from './db.js'; +import { getArchiveDir, getDbPath, getExcludedProjects, findJsonlFiles, statIfExists } from './paths.js'; import { isErroredSentinel } from './summary-sentinel.js'; export async function verifyIndex() { const result = { missing: [], + unindexed: [], orphaned: [], outdated: [], + archiveRefreshes: [], corrupted: [] }; const archiveDir = getArchiveDir(); @@ -18,8 +21,13 @@ export async function verifyIndex() { if (!fs.existsSync(archiveDir)) { return result; } - // Initialize database once for all checks - const db = initDatabase(); + // Verification must not create or migrate the index it is measuring. + const dbPath = getDbPath(); + const db = fs.existsSync(dbPath) ? new Database(dbPath, { readonly: true, fileMustExist: true }) : null; + const indexState = db?.prepare(` + SELECT MAX(line_end) AS maxLineEnd, MAX(last_indexed) AS lastIndexed + FROM exchanges WHERE archive_path = ? + `); const projects = fs.readdirSync(archiveDir); const excludedProjects = getExcludedProjects(); const excludedDirSet = new Set(excludedProjects); @@ -47,27 +55,28 @@ export async function verifyIndex() { // re-attempts it rather than reporting the conversation as healthy. if (!fs.existsSync(summaryPath)) { result.missing.push({ path: conversationPath, reason: 'No summary file' }); - continue; } - if (isErroredSentinel(fs.readFileSync(summaryPath, 'utf-8'))) { + else if (isErroredSentinel(fs.readFileSync(summaryPath, 'utf-8'))) { result.missing.push({ path: conversationPath, reason: 'Previous summarization failed (error sentinel)' }); - continue; } - // Check if file is outdated (modified after last_indexed) - const lastIndexed = getFileLastIndexed(db, conversationPath); - if (lastIndexed !== null) { - const fileStat = fs.statSync(conversationPath); - if (fileStat.mtimeMs > lastIndexed) { - result.outdated.push({ - path: conversationPath, - fileTime: fileStat.mtimeMs, - dbTime: lastIndexed - }); - } - } - // Try parsing to detect corruption + const state = indexState?.get(conversationPath); + const maxLineEnd = state?.maxLineEnd ?? null; + if (maxLineEnd === null) + result.unindexed.push({ path: conversationPath }); + // Parse independently of summary state and mtime. Only exchanges past + // the indexed high-water mark are a real index delta. try { - await parseConversation(conversationPath, project, conversationPath); + const exchanges = await parseConversation(conversationPath, project, conversationPath); + if (maxLineEnd !== null) { + const fileTime = fs.statSync(conversationPath).mtimeMs; + const dbTime = state?.lastIndexed ?? 0; + if (exchanges.some(exchange => exchange.lineEnd > maxLineEnd)) { + result.outdated.push({ path: conversationPath, fileTime, dbTime }); + } + else if (state?.lastIndexed != null && fileTime > state.lastIndexed) { + result.archiveRefreshes.push({ path: conversationPath, fileTime, dbTime }); + } + } } catch (error) { result.corrupted.push({ @@ -79,10 +88,10 @@ export async function verifyIndex() { } console.log(`Verified ${totalChecked} conversations.`); // Check for orphaned database entries - const dbExchanges = getAllExchanges(db); - db.close(); + const dbExchanges = db ? getAllExchanges(db) : []; + db?.close(); for (const exchange of dbExchanges) { - if (!foundFiles.has(exchange.archivePath)) { + if (!foundFiles.has(exchange.archivePath) && !statIfExists(exchange.archivePath)?.isFile()) { result.orphaned.push({ uuid: exchange.id, path: exchange.archivePath @@ -106,10 +115,10 @@ export async function repairIndex(issues) { deleteExchange(db, orphan.uuid); } // Re-index missing and outdated conversations - const toReindex = [ - ...issues.missing.map(m => m.path), - ...issues.outdated.map(o => o.path) - ]; + const toReindex = [...new Set([ + ...issues.missing.map(m => m.path), + ...issues.outdated.map(o => o.path) + ])]; for (const conversationPath of toReindex) { console.log(`Re-indexing: ${conversationPath}`); try { diff --git a/src/index-cli.ts b/src/index-cli.ts index 272ecdba..49773283 100644 --- a/src/index-cli.ts +++ b/src/index-cli.ts @@ -10,16 +10,16 @@ import { acquireFileLock, readLockHolder, releaseFileLock } from './file-lock.js const command = process.argv[2]; -// Serialize the whole CLI with `sync-cli`: every command can initialize or -// migrate the database, and rebuild also deletes the database and summaries. +// Serialize the whole CLI with `sync-cli`: writers mutate the index/archive, +// and verification must not claim a consistent result while one is active. const syncLockPath = getSyncLockPath(); const syncLock = acquireFileLock(syncLockPath); if (!syncLock) { const holder = readLockHolder(syncLockPath); const holderLabel = holder !== null ? `pid ${holder}` : 'another process'; - // stderr keeps this out of stdout consumers; status 0 so hooks don't fail. + // A skipped write may be retried by a hook; a skipped verifier is not a result. console.error(`episodic-memory: sync already running (${holderLabel}); skipping`); - process.exit(0); + process.exit(command === 'verify' ? 2 : 0); } const releaseSyncLockOnce = () => { if ((releaseSyncLockOnce as any).done) return; @@ -71,17 +71,27 @@ async function main() { console.log('\n=== Verification Results ==='); console.log(`Missing summaries: ${issues.missing.length}`); + console.log(`Unindexed archives: ${issues.unindexed.length}`); console.log(`Orphaned entries: ${issues.orphaned.length}`); console.log(`Outdated files: ${issues.outdated.length}`); + console.log(`Archive refreshes (no new exchanges): ${issues.archiveRefreshes.length}`); console.log(`Corrupted files: ${issues.corrupted.length}`); if (issues.missing.length > 0) { console.log('\nMissing summaries:'); issues.missing.forEach(m => console.log(` ${m.path}`)); } + if (issues.unindexed.length > 0) { + console.log('\nUnindexed archives:'); + issues.unindexed.forEach(item => console.log(` ${item.path}`)); + } + if (issues.outdated.length > 0) { + console.log('\nArchives with appended exchanges:'); + issues.outdated.forEach(item => console.log(` ${item.path}`)); + } - if (issues.missing.length + issues.orphaned.length + issues.outdated.length + issues.corrupted.length > 0) { - console.log('\nRun with --repair to fix these issues.'); + if (issues.missing.length + issues.unindexed.length + issues.orphaned.length + issues.outdated.length + issues.corrupted.length > 0) { + console.log('\nReview exact candidates before repair; --repair may regenerate summaries and embeddings.'); process.exit(1); } else { console.log('\nāœ… Index is healthy!'); @@ -92,9 +102,13 @@ async function main() { console.log('Verifying conversation index...'); const repairIssues = await verifyIndex(); + if (repairIssues.unindexed.length > 0) { + console.log(`${repairIssues.unindexed.length} unindexed archive(s) require separate review; --repair does not select them by this status alone.`); + process.exitCode = 1; + } if (repairIssues.missing.length + repairIssues.orphaned.length + repairIssues.outdated.length > 0) { await repairIndex(repairIssues); - } else { + } else if (repairIssues.unindexed.length === 0) { console.log('āœ… No issues to repair!'); } break; diff --git a/src/verify.ts b/src/verify.ts index 0dd2c5db..e70173ad 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -1,22 +1,27 @@ import fs from 'fs'; import path from 'path'; +import Database from 'better-sqlite3'; import { parseConversation } from './parser.js'; -import { initDatabase, getAllExchanges, getFileLastIndexed } from './db.js'; -import { getArchiveDir, getExcludedProjects, findJsonlFiles, statIfExists } from './paths.js'; +import { getAllExchanges } from './db.js'; +import { getArchiveDir, getDbPath, getExcludedProjects, findJsonlFiles, statIfExists } from './paths.js'; import { isErroredSentinel } from './summary-sentinel.js'; export interface VerificationResult { missing: Array<{ path: string; reason: string }>; + unindexed: Array<{ path: string }>; orphaned: Array<{ uuid: string; path: string }>; outdated: Array<{ path: string; fileTime: number; dbTime: number }>; + archiveRefreshes: Array<{ path: string; fileTime: number; dbTime: number }>; corrupted: Array<{ path: string; error: string }>; } export async function verifyIndex(): Promise { const result: VerificationResult = { missing: [], + unindexed: [], orphaned: [], outdated: [], + archiveRefreshes: [], corrupted: [] }; @@ -30,8 +35,13 @@ export async function verifyIndex(): Promise { return result; } - // Initialize database once for all checks - const db = initDatabase(); + // Verification must not create or migrate the index it is measuring. + const dbPath = getDbPath(); + const db = fs.existsSync(dbPath) ? new Database(dbPath, { readonly: true, fileMustExist: true }) : null; + const indexState = db?.prepare(` + SELECT MAX(line_end) AS maxLineEnd, MAX(last_indexed) AS lastIndexed + FROM exchanges WHERE archive_path = ? + `); const projects = fs.readdirSync(archiveDir); const excludedProjects = getExcludedProjects(); @@ -68,29 +78,27 @@ export async function verifyIndex(): Promise { // re-attempts it rather than reporting the conversation as healthy. if (!fs.existsSync(summaryPath)) { result.missing.push({ path: conversationPath, reason: 'No summary file' }); - continue; - } - if (isErroredSentinel(fs.readFileSync(summaryPath, 'utf-8'))) { + } else if (isErroredSentinel(fs.readFileSync(summaryPath, 'utf-8'))) { result.missing.push({ path: conversationPath, reason: 'Previous summarization failed (error sentinel)' }); - continue; } - // Check if file is outdated (modified after last_indexed) - const lastIndexed = getFileLastIndexed(db, conversationPath); - if (lastIndexed !== null) { - const fileStat = fs.statSync(conversationPath); - if (fileStat.mtimeMs > lastIndexed) { - result.outdated.push({ - path: conversationPath, - fileTime: fileStat.mtimeMs, - dbTime: lastIndexed - }); - } - } + const state = indexState?.get(conversationPath) as { maxLineEnd: number | null; lastIndexed: number | null } | undefined; + const maxLineEnd = state?.maxLineEnd ?? null; + if (maxLineEnd === null) result.unindexed.push({ path: conversationPath }); - // Try parsing to detect corruption + // Parse independently of summary state and mtime. Only exchanges past + // the indexed high-water mark are a real index delta. try { - await parseConversation(conversationPath, project, conversationPath); + const exchanges = await parseConversation(conversationPath, project, conversationPath); + if (maxLineEnd !== null) { + const fileTime = fs.statSync(conversationPath).mtimeMs; + const dbTime = state?.lastIndexed ?? 0; + if (exchanges.some(exchange => exchange.lineEnd > maxLineEnd)) { + result.outdated.push({ path: conversationPath, fileTime, dbTime }); + } else if (state?.lastIndexed != null && fileTime > state.lastIndexed) { + result.archiveRefreshes.push({ path: conversationPath, fileTime, dbTime }); + } + } } catch (error) { result.corrupted.push({ path: conversationPath, @@ -103,11 +111,11 @@ export async function verifyIndex(): Promise { console.log(`Verified ${totalChecked} conversations.`); // Check for orphaned database entries - const dbExchanges = getAllExchanges(db); - db.close(); + const dbExchanges = db ? getAllExchanges(db) : []; + db?.close(); for (const exchange of dbExchanges) { - if (!foundFiles.has(exchange.archivePath)) { + if (!foundFiles.has(exchange.archivePath) && !statIfExists(exchange.archivePath)?.isFile()) { result.orphaned.push({ uuid: exchange.id, path: exchange.archivePath @@ -118,7 +126,7 @@ export async function verifyIndex(): Promise { return result; } -export async function repairIndex(issues: VerificationResult): Promise { +export async function repairIndex(issues: Pick): Promise { console.log('Repairing index...'); // To avoid circular dependencies, we import the indexer functions dynamically @@ -137,10 +145,10 @@ export async function repairIndex(issues: VerificationResult): Promise { } // Re-index missing and outdated conversations - const toReindex = [ + const toReindex = [...new Set([ ...issues.missing.map(m => m.path), ...issues.outdated.map(o => o.path) - ]; + ])]; for (const conversationPath of toReindex) { console.log(`Re-indexing: ${conversationPath}`); diff --git a/test/index-cli-single-instance.test.ts b/test/index-cli-single-instance.test.ts index d41203a9..c9c9f23a 100644 --- a/test/index-cli-single-instance.test.ts +++ b/test/index-cli-single-instance.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { spawnSync } from 'child_process'; -import { mkdtempSync, mkdirSync, rmSync, existsSync } from 'fs'; +import { mkdtempSync, mkdirSync, rmSync, existsSync, writeFileSync } from 'fs'; import { fileURLToPath } from 'url'; import { dirname, join } from 'path'; import { tmpdir } from 'os'; @@ -75,18 +75,39 @@ describe('index-cli single-instance lock (#97)', () => { expect(existsSync(join(testDir, 'test.db'))).toBe(false); }); - it('skips `verify`, which can initialize and migrate the database', () => { + it('reports a lock-blocked `verify` as incomplete rather than healthy', () => { heldLock = acquireFileLock(lockPath); expect(heldLock).not.toBeNull(); const result = runIndexCli(['verify']); - expect(result.status).toBe(0); + expect(result.status).toBe(2); expect(result.stderr).toMatch(/sync already running.*skipping/); expect(result.stdout).not.toMatch(/Verifying conversation index/); expect(existsSync(join(testDir, 'test.db'))).toBe(false); }); + it('reports unindexed archives as actionable without calling them outdated', () => { + const projectDir = join(testDir, 'archive', 'project'); + mkdirSync(projectDir, { recursive: true }); + const archivePath = join(projectDir, 'session.jsonl'); + writeFileSync(archivePath, [ + JSON.stringify({ type: 'user', message: { role: 'user', content: 'Hello' }, timestamp: '2024-01-01T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'Hi' }, timestamp: '2024-01-01T00:00:01Z' }), + ].join('\n')); + writeFileSync(archivePath.replace('.jsonl', '-summary.txt'), 'Existing summary'); + + const result = runIndexCli(['verify']); + expect(result.status).toBe(1); + expect(result.stdout).toContain('Unindexed archives: 1'); + expect(result.stdout).toContain(archivePath); + expect(result.stdout).toContain('Outdated files: 0'); + + const repair = runIndexCli(['repair']); + expect(repair.status).toBe(1); + expect(repair.stdout).toContain('unindexed archive(s) require separate review'); + }); + it('releases the lock after an uncontended run', () => { const result = runIndexCli(['verify']); diff --git a/test/verify.test.ts b/test/verify.test.ts index ae70da81..14eb223d 100644 --- a/test/verify.test.ts +++ b/test/verify.test.ts @@ -6,6 +6,7 @@ import path from 'path'; import os from 'os'; import { initDatabase, insertExchange } from '../src/db.js'; import { ConversationExchange } from '../src/types.js'; +import { summarizeConversation } from '../src/summarizer.js'; vi.mock('../src/embeddings.js', () => ({ initEmbeddings: vi.fn(async () => {}), @@ -45,6 +46,21 @@ describe('verifyIndex', () => { delete process.env.TEST_DB_PATH; }); + it('does not create an index database merely by verifying an archive', async () => { + const projectArchive = path.join(archiveDir, 'read-only-project'); + fs.mkdirSync(projectArchive, { recursive: true }); + const conversationPath = path.join(projectArchive, 'conversation.jsonl'); + fs.writeFileSync(conversationPath, [ + JSON.stringify({ type: 'user', message: { role: 'user', content: 'Hello' }, timestamp: '2024-01-01T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'Hi' }, timestamp: '2024-01-01T00:00:01Z' }), + ].join('\n')); + + expect(fs.existsSync(dbPath)).toBe(false); + const result = await verifyIndex(); + expect(result.unindexed.map(item => item.path)).toContain(conversationPath); + expect(fs.existsSync(dbPath)).toBe(false); + }); + it('should skip excluded projects', async () => { // Create two projects in archive const projectA = path.join(archiveDir, 'project-a'); @@ -156,7 +172,7 @@ describe('verifyIndex', () => { expect(result.orphaned[0].path).toBe(exchange.archivePath); }); - it('detects outdated files (file modified after last_indexed)', async () => { + it('detects appended exchanges beyond the indexed high-water mark', async () => { // Create conversation file with summary const projectArchive = path.join(archiveDir, 'test-project'); fs.mkdirSync(projectArchive, { recursive: true }); @@ -193,15 +209,15 @@ describe('verifyIndex', () => { const lastIndexed = row.last_indexed; db.close(); - // Wait a bit, then modify the file - await new Promise(resolve => setTimeout(resolve, 10)); - - // Update the conversation file + // Append a complete exchange and make the timestamp comparison deterministic. const updatedMessages = [ ...messages, - JSON.stringify({ type: 'user', message: { role: 'user', content: 'New message' }, timestamp: '2024-01-01T00:00:02Z' }) + JSON.stringify({ type: 'user', message: { role: 'user', content: 'New message' }, timestamp: '2024-01-01T00:00:02Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'New response' }, timestamp: '2024-01-01T00:00:03Z' }) ]; fs.writeFileSync(conversationPath, updatedMessages.join('\n')); + const future = new Date(lastIndexed + 10_000); + fs.utimesSync(conversationPath, future, future); // Verify detects outdated file const result = await verifyIndex(); @@ -212,6 +228,75 @@ describe('verifyIndex', () => { expect(result.outdated[0].fileTime).toBeGreaterThan(lastIndexed); }); + it('does not call an mtime-only archive refresh an index delta', async () => { + const projectArchive = path.join(archiveDir, 'test-project'); + fs.mkdirSync(projectArchive, { recursive: true }); + const conversationPath = path.join(projectArchive, 'refreshed.jsonl'); + fs.writeFileSync(conversationPath, [ + JSON.stringify({ type: 'user', message: { role: 'user', content: 'Hello' }, timestamp: '2024-01-01T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'Hi' }, timestamp: '2024-01-01T00:00:01Z' }) + ].join('\n')); + fs.writeFileSync(conversationPath.replace('.jsonl', '-summary.txt'), 'Existing summary'); + const db = initDatabase(); + insertExchange(db, { id: 'refreshed', project: 'test-project', timestamp: '2024-01-01T00:00:00Z', + userMessage: 'Hello', assistantMessage: 'Hi', archivePath: conversationPath, lineStart: 1, lineEnd: 2 }, new Array(384).fill(0.1)); + db.close(); + const future = new Date(Date.now() + 10_000); + fs.utimesSync(conversationPath, future, future); + + const result = await verifyIndex(); + expect(result.outdated).toEqual([]); + expect(result.archiveRefreshes.map(item => item.path)).toEqual([conversationPath]); + }); + + it('reports a missing summary and an appended exchange independently', async () => { + const projectArchive = path.join(archiveDir, 'test-project'); + fs.mkdirSync(projectArchive, { recursive: true }); + const conversationPath = path.join(projectArchive, 'appended.jsonl'); + const messages = [ + JSON.stringify({ type: 'user', message: { role: 'user', content: 'Hello' }, timestamp: '2024-01-01T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'Hi' }, timestamp: '2024-01-01T00:00:01Z' }) + ]; + fs.writeFileSync(conversationPath, messages.join('\n')); + const db = initDatabase(); + insertExchange(db, { id: 'original', project: 'test-project', timestamp: '2024-01-01T00:00:00Z', + userMessage: 'Hello', assistantMessage: 'Hi', archivePath: conversationPath, lineStart: 1, lineEnd: 2 }, new Array(384).fill(0.1)); + const lastIndexed = (db.prepare('SELECT last_indexed FROM exchanges WHERE id = ?').get('original') as { last_indexed: number }).last_indexed; + db.close(); + fs.appendFileSync(conversationPath, '\n' + [ + JSON.stringify({ type: 'user', message: { role: 'user', content: 'Later' }, timestamp: '2024-01-01T00:00:02Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'Done' }, timestamp: '2024-01-01T00:00:03Z' }) + ].join('\n')); + const future = new Date(lastIndexed + 10_000); + fs.utimesSync(conversationPath, future, future); + + const result = await verifyIndex(); + expect(result.missing.map(item => item.path)).toEqual([conversationPath]); + expect(result.outdated.map(item => item.path)).toEqual([conversationPath]); + }); + + it('distinguishes an unindexed archive from an indexed external source', async () => { + const projectArchive = path.join(archiveDir, 'test-project'); + fs.mkdirSync(projectArchive, { recursive: true }); + const archivePath = path.join(projectArchive, 'unindexed.jsonl'); + const sourcePath = path.join(testDir, 'active.jsonl'); + const lines = [ + JSON.stringify({ type: 'user', message: { role: 'user', content: 'Hello' }, timestamp: '2024-01-01T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'Hi' }, timestamp: '2024-01-01T00:00:01Z' }) + ].join('\n'); + fs.writeFileSync(archivePath, lines); + fs.writeFileSync(archivePath.replace('.jsonl', '-summary.txt'), 'Existing summary'); + fs.writeFileSync(sourcePath, lines); + const db = initDatabase(); + insertExchange(db, { id: 'external', project: 'test-project', timestamp: '2024-01-01T00:00:00Z', + userMessage: 'Hello', assistantMessage: 'Hi', archivePath: sourcePath, lineStart: 1, lineEnd: 2 }, new Array(384).fill(0.1)); + db.close(); + + const result = await verifyIndex(); + expect(result.unindexed.map(item => item.path)).toEqual([archivePath]); + expect(result.orphaned).toEqual([]); + }); + // Note: Parser is resilient to malformed JSON - it skips bad lines // Corruption detection would require file system errors or permission issues // which are harder to test. Skipping for now as missing summaries is the @@ -244,6 +329,26 @@ describe('repairIndex', () => { delete process.env.TEST_DB_PATH; }); + it('repairs a path reported as both missing and outdated only once', async () => { + const projectArchive = path.join(archiveDir, 'test-project'); + fs.mkdirSync(projectArchive, { recursive: true }); + const conversationPath = path.join(projectArchive, 'duplicate.jsonl'); + fs.writeFileSync(conversationPath, [ + JSON.stringify({ type: 'user', message: { role: 'user', content: 'Hello' }, timestamp: '2024-01-01T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'Hi' }, timestamp: '2024-01-01T00:00:01Z' }) + ].join('\n')); + vi.mocked(summarizeConversation).mockClear(); + + await repairIndex({ + ...await verifyIndex(), + missing: [{ path: conversationPath, reason: 'No summary file' }], + outdated: [{ path: conversationPath, fileTime: 2, dbTime: 1 }], + orphaned: [], corrupted: [] + }); + + expect(summarizeConversation).toHaveBeenCalledTimes(1); + }); + it('deletes orphaned database entries during repair', async () => { // Initialize database with orphaned entry const db = initDatabase();