Consolidate freshness, semantic-index status, and MCP config onto the Overview

internal/embed/embed.go

@@ -43,6 +43,14 @@ type Summary struct {
 // batches, until none remain. It returns a summary. Individual batch failures
 // abort the run (the next run resumes where this one stopped, since stored
 // embeddings persist).
+//
+// Each run is also recorded in the store's embed_runs table (issue #1):
+// a row at start, a per-batch progress heartbeat, and a terminal write with
+// the totals (or the abort error). That log is how the web Overview shows
+// "last index run" and a live in-progress marker — this CLI and `msgbrowse
+// serve` are separate processes sharing one SQLite file. Recording is
+// best-effort bookkeeping: a failed recording write logs a warning and never
+// aborts the embedding work itself.
 func Run(ctx context.Context, st *store.Store, client llm.Client, opts Options) (Summary, error) {
 	log := opts.Logger
 	if log == nil {
@@ -56,11 +64,44 @@ func Run(ctx context.Context, st *store.Store, client llm.Client, opts Options)
 	if model == "" {
 		return Summary{}, fmt.Errorf("embed: model not configured (set llm.embed_model)")
 	}
+
+	start := time.Now()
+	runID, err := st.BeginEmbedRun(ctx, model, start)
+	if err != nil {
+		log.Warn("could not record embed run start", "error", err)
+		runID = 0
+	}
+	sum, err := run(ctx, st, client, opts, model, runID, start, log)
+	if runID != 0 {
+		errText := ""
+		if err != nil {
+			errText = err.Error()
+		}
+		// The terminal write must land even when the run was aborted by ctx
+		// cancellation — otherwise every Ctrl-C reads as a crashed run forever.
+		if ferr := st.FinishEmbedRun(context.WithoutCancel(ctx), store.EmbedRun{
+			ID:         runID,
+			FinishedAt: time.Now(),
+			DurationMS: time.Since(start).Milliseconds(),
+			Embedded:   sum.Embedded,
+			Pruned:     sum.Pruned,
+			Batches:    sum.Batches,
+			Error:      errText,
+		}); ferr != nil {
+			log.Warn("could not record embed run finish", "error", ferr)
+		}
+	}
+	return sum, err
+}
+
+// run is the embedding loop behind Run, separated so the caller can wrap it
+// with the begin/finish run-recording writes. runID 0 disables the per-batch
+// progress heartbeat (recording could not start).
+func run(ctx context.Context, st *store.Store, client llm.Client, opts Options, model string, runID int64, start time.Time, log *slog.Logger) (Summary, error) {
 	batch := opts.BatchSize
 	if batch <= 0 || batch > 512 {
 		batch = 64
 	}
-	start := time.Now()
 	var sum Summary
 
 	if opts.Prune {
@@ -125,6 +166,13 @@ func Run(ctx context.Context, st *store.Store, client llm.Client, opts Options)
 		}
 		sum.Embedded += len(targets)
 		sum.Batches++
+		if runID != 0 {
+			// The heartbeat readers use to distinguish a live run from a crashed
+			// one; best-effort like the rest of the recording.
+			if uerr := st.UpdateEmbedRunProgress(ctx, runID, sum.Embedded, sum.Batches, time.Now()); uerr != nil {
+				log.Warn("could not record embed run progress", "error", uerr)
+			}
+		}
 		log.Debug("embedded batch", "batch", sum.Batches, "embedded", sum.Embedded, "of", total)
 	}
 

internal/embed/embed_test.go

@@ -2,9 +2,11 @@ package embed
 
 import (
 	"context"
+	"errors"
 	"io"
 	"log/slog"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -130,6 +132,74 @@ func TestRunRespectsBatchSize(t *testing.T) {
 	}
 }
 
+// TestRunRecordsEmbedRun (issue #1): every Run leaves a finished embed_runs
+// row behind — the durable log the web Overview reads for "last index run" —
+// with the run's totals, and a no-op re-run records its own (zero-work) row.
+func TestRunRecordsEmbedRun(t *testing.T) {
+	st := newStore(t)
+	seed(t, st)
+	ctx := context.Background()
+	opts := Options{EmbedModel: "test-embed", Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
+
+	if _, err := Run(ctx, st, &fakeClient{}, opts); err != nil {
+		t.Fatal(err)
+	}
+	r, err := st.LatestEmbedRun(ctx)
+	if err != nil || r == nil {
+		t.Fatalf("LatestEmbedRun = %v, %v; want a recorded run", r, err)
+	}
+	if r.InFlight() {
+		t.Error("completed run still recorded as in flight")
+	}
+	if r.Model != "test-embed" || r.Embedded != 2 || r.Batches != 1 || r.Error != "" {
+		t.Errorf("recorded run = %+v, want model test-embed, 2 embedded in 1 batch, no error", r)
+	}
+
+	first := r.ID
+	if _, err := Run(ctx, st, &fakeClient{}, opts); err != nil {
+		t.Fatal(err)
+	}
+	r, err = st.LatestEmbedRun(ctx)
+	if err != nil || r == nil {
+		t.Fatalf("LatestEmbedRun after re-run = %v, %v", r, err)
+	}
+	if r.ID == first || r.Embedded != 0 || r.InFlight() {
+		t.Errorf("no-op re-run row = %+v, want a fresh finished row with 0 embedded", r)
+	}
+}
+
+// TestRunRecordsFailure: an aborted run's row is still finished (never left
+// dangling in-flight) and carries the abort reason.
+func TestRunRecordsFailure(t *testing.T) {
+	st := newStore(t)
+	seed(t, st)
+	ctx := context.Background()
+
+	_, err := Run(ctx, st, &failingClient{}, Options{
+		EmbedModel: "test-embed", Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
+	})
+	if err == nil {
+		t.Fatal("expected the failing client to abort the run")
+	}
+	r, lerr := st.LatestEmbedRun(ctx)
+	if lerr != nil || r == nil {
+		t.Fatalf("LatestEmbedRun = %v, %v; want the failed run recorded", r, lerr)
+	}
+	if r.InFlight() {
+		t.Error("failed run left dangling in flight")
+	}
+	if r.Error == "" || !strings.Contains(r.Error, "boom") {
+		t.Errorf("recorded error = %q, want the abort reason", r.Error)
+	}
+}
+
+// failingClient errors on every Embed call.
+type failingClient struct{ fakeClient }
+
+func (f *failingClient) Embed(context.Context, []string) ([][]float32, error) {
+	return nil, errors.New("boom")
+}
+
 func TestRunNoModel(t *testing.T) {
 	st := newStore(t)
 	if _, err := Run(context.Background(), st, &fakeClient{}, Options{}); err == nil {

internal/store/embedruns.go

@@ -0,0 +1,136 @@
+// Embedding-run bookkeeping and coverage (issue #1): the store-level queries
+// behind the Overview's "Semantic search index" card. internal/embed records a
+// row per indexing run here (begin → per-batch heartbeat → finish), and the
+// web layer reads the latest row plus the coverage aggregate to show
+// embedded-vs-total, the last completed run, and a live in-progress marker
+// — the embed CLI and `msgbrowse serve` are separate processes sharing one
+// SQLite file, so this table is their only communication channel.
+package store
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+)
+
+// EmbedRun is one semantic-search indexing run. FinishedAt is the zero time
+// while the run is still in flight (or died before its terminal write);
+// UpdatedAt is the per-batch heartbeat readers use to tell a live run from a
+// crashed one. Embedded/Batches are live counters during a run and the final
+// totals after it. Error carries the abort reason for a failed run ("" on
+// success).
+type EmbedRun struct {
+	ID         int64
+	Model      string
+	StartedAt  time.Time
+	UpdatedAt  time.Time
+	FinishedAt time.Time
+	DurationMS int64
+	Embedded   int
+	Pruned     int64
+	Batches    int
+	Error      string
+}
+
+// InFlight reports whether the run has not recorded its terminal write.
+func (r EmbedRun) InFlight() bool { return r.FinishedAt.IsZero() }
+
+// BeginEmbedRun records the start of an embedding run and returns the row id
+// the run's later progress/finish writes target. The heartbeat (updated_at)
+// starts equal to startedAt.
+func (s *Store) BeginEmbedRun(ctx context.Context, model string, startedAt time.Time) (int64, error) {
+	ts := startedAt.UTC().Format(time.RFC3339)
+	res, err := s.db.ExecContext(ctx,
+		`INSERT INTO embed_runs (model, started_at, updated_at) VALUES (?, ?, ?)`,
+		model, ts, ts)
+	if err != nil {
+		return 0, fmt.Errorf("begin embed run: %w", err)
+	}
+	return res.LastInsertId()
+}
+
+// UpdateEmbedRunProgress refreshes a run's live counters and heartbeat after a
+// batch. Readers treat an unfinished row with a fresh heartbeat as "indexing
+// in progress".
+func (s *Store) UpdateEmbedRunProgress(ctx context.Context, id int64, embedded, batches int, at time.Time) error {
+	if _, err := s.db.ExecContext(ctx,
+		`UPDATE embed_runs SET embedded = ?, batches = ?, updated_at = ? WHERE id = ?`,
+		embedded, batches, at.UTC().Format(time.RFC3339), id); err != nil {
+		return fmt.Errorf("update embed run progress: %w", err)
+	}
+	return nil
+}
+
+// FinishEmbedRun records a run's terminal state: finished_at (which flips the
+// row out of "in flight"), the final totals, and the abort error when the run
+// failed. r.ID selects the row; r.UpdatedAt is stamped to r.FinishedAt.
+func (s *Store) FinishEmbedRun(ctx context.Context, r EmbedRun) error {
+	ts := r.FinishedAt.UTC().Format(time.RFC3339)
+	if _, err := s.db.ExecContext(ctx,
+		`UPDATE embed_runs
+		    SET finished_at = ?, updated_at = ?, duration_ms = ?, embedded = ?, pruned = ?, batches = ?, error = ?
+		  WHERE id = ?`,
+		ts, ts, r.DurationMS, r.Embedded, r.Pruned, r.Batches, r.Error, r.ID); err != nil {
+		return fmt.Errorf("finish embed run: %w", err)
+	}
+	return nil
+}
+
+// LatestEmbedRun returns the most recently started embedding run, or nil when
+// none has ever been recorded. The caller decides what an unfinished row
+// means by its heartbeat age (live vs crashed).
+func (s *Store) LatestEmbedRun(ctx context.Context) (*EmbedRun, error) {
+	var (
+		r                          EmbedRun
+		started, updated, finished string
+	)
+	err := s.db.QueryRowContext(ctx,
+		`SELECT id, model, started_at, updated_at, finished_at, duration_ms, embedded, pruned, batches, error
+		   FROM embed_runs ORDER BY id DESC LIMIT 1`).
+		Scan(&r.ID, &r.Model, &started, &updated, &finished,
+			&r.DurationMS, &r.Embedded, &r.Pruned, &r.Batches, &r.Error)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("latest embed run: %w", err)
+	}
+	r.StartedAt = parseRFC3339(started)
+	r.UpdatedAt = parseRFC3339(updated)
+	if finished != "" {
+		r.FinishedAt = parseRFC3339(finished)
+	}
+	return &r, nil
+}
+
+// EmbeddingCoverage is the semantic-search index's footprint for one model:
+// how many embeddable messages exist and how many of them already have a
+// stored vector.
+type EmbeddingCoverage struct {
+	// Embedded is the number of embeddable messages with a vector for the model.
+	Embedded int
+	// Embeddable is the total number of messages semantic search can index:
+	// non-system with a non-blank body (the same predicate the embed pipeline's
+	// MessagesNeedingEmbedding/CountMissingEmbeddings use, so
+	// Embeddable - Embedded always equals the pending count).
+	Embeddable int
+}
+
+// EmbeddingCoverage returns the index coverage for model in one pass: a
+// single LEFT JOIN over messages ✕ the embeddings PK, counting every
+// embeddable message and the subset that already carries a vector. It shares
+// its WHERE predicate with CountMissingEmbeddings by construction.
+func (s *Store) EmbeddingCoverage(ctx context.Context, model string) (EmbeddingCoverage, error) {
+	var c EmbeddingCoverage
+	err := s.db.QueryRowContext(ctx, `
+SELECT COUNT(*), COUNT(e.message_hash)
+  FROM messages m
+  LEFT JOIN embeddings e ON e.message_hash = m.hash AND e.model = ?
+ WHERE m.is_system = 0 AND TRIM(m.body) <> ''`, model).
+		Scan(&c.Embeddable, &c.Embedded)
+	if err != nil {
+		return EmbeddingCoverage{}, fmt.Errorf("embedding coverage: %w", err)
+	}
+	return c, nil
+}