Paginate the cohort builder and add a cohort CSV export

CHANGELOG.md

@@ -28,6 +28,13 @@ minor versions can still change behaviour.
   go out. One check per address, cached for 30 days; changing the sender
   country, address source or product description triggers a fresh check. The
   receipt id is logged with each send. Native providers only. Off by default.
+- **Cohort builder pagination and CSV export.** The cohort table on the
+  Signals tab used to stop at the first 50 matches with no way to see the
+  rest. It now pages with Prev / Next ("Showing 51–100 of 147"), and an
+  Export CSV button downloads the whole cohort. `POST /api/cohort` accepts
+  `offset` and `limit` (default 50, max 200) and orders by name so pages never
+  overlap; `GET /api/cohort.csv?require=...&exclude=...` returns every match.
+  The cohort size is now counted up to 10,000 instead of 1,000.
 
 ## [0.3.0] — 2026-10-13
 

harvey/dashboard.py

@@ -1250,38 +1250,87 @@ async def set_signals_status(request: Request):
         return JSONResponse({"success": False, "message": str(e)}, status_code=500)
 
 
+COHORT_PAGE_SIZE = 50
+COHORT_MAX_PAGE_SIZE = 200
+# A cohort bigger than this is a query to narrow, not a list to page through.
+COHORT_MAX = 10_000
+
+
+async def _cohort_companies(require: list[str], exclude: list[str]) -> list[dict]:
+    """Every company in the cohort, ordered by name so pages never overlap."""
+    state = _state()
+    await state.init_db()
+    ids = await state.cohort(require, exclude, limit=COHORT_MAX)
+    rows: list[dict] = []
+    # Chunked: older SQLite builds cap a statement at 999 bound variables.
+    for i in range(0, len(ids), 500):
+        chunk = ids[i:i + 500]
+        rows += await query_db(
+            "SELECT id, name, domain, website, industry, location FROM companies "
+            f"WHERE id IN ({','.join('?' for _ in chunk)})",
+            tuple(chunk),
+        )
+    rows.sort(key=lambda r: ((r.get("name") or "").lower(), r["id"]))
+    return rows
+
+
+def _codes(raw) -> list[str]:
+    """Signal codes from a JSON list or a comma-separated query string."""
+    if isinstance(raw, str):
+        raw = raw.split(",")
+    return [c.strip() for c in (raw or []) if c and c.strip()]
+
+
 @app.post("/api/cohort")
 async def preview_cohort(request: Request):
     """How many companies carry ALL these signals and none of those.
 
     The payoff for confirming signals: a cohort is a query, not a list. Set
     intersection happens in SQL — intersecting in JS over a capped fetch
-    silently returns the wrong answer.
+    silently returns the wrong answer. ``offset``/``limit`` page through the
+    matches; ``size`` is always the whole cohort.
     """
     try:
         body = await request.json()
-        require = [c for c in (body.get("require") or []) if c]
-        exclude = [c for c in (body.get("exclude") or []) if c]
+        require = _codes(body.get("require"))
+        exclude = _codes(body.get("exclude"))
+        offset = max(0, int(body.get("offset") or 0))
+        limit = min(max(1, int(body.get("limit") or COHORT_PAGE_SIZE)),
+                    COHORT_MAX_PAGE_SIZE)
         if not require:
-            return {"size": 0, "companies": []}
+            return {"size": 0, "offset": 0, "limit": limit, "companies": []}
 
-        state = _state()
-        await state.init_db()
-        ids = await state.cohort(require, exclude, limit=1000)
-        if not ids:
-            return {"size": 0, "companies": []}
-
-        placeholders = ",".join("?" for _ in ids[:200])
-        rows = await query_db(
-            f"SELECT id, name, domain, industry, location FROM companies "
-            f"WHERE id IN ({placeholders})",
-            tuple(ids[:200]),
-        )
-        return {"size": len(ids), "companies": rows}
+        rows = await _cohort_companies(require, exclude)
+        return {"size": len(rows), "offset": offset, "limit": limit,
+                "companies": rows[offset:offset + limit]}
     except Exception as e:
         return JSONResponse({"size": 0, "companies": [], "error": str(e)}, status_code=500)
 
 
+@app.get("/api/cohort.csv")
+async def export_cohort(require: str = "", exclude: str = ""):
+    """The whole cohort as a CSV download — every match, not just one page."""
+    import csv
+    import io
+
+    buf = io.StringIO()
+    writer = csv.writer(buf)
+    writer.writerow(["name", "domain", "website", "industry", "location"])
+    try:
+        codes = _codes(require)
+        if codes:
+            for r in await _cohort_companies(codes, _codes(exclude)):
+                writer.writerow([r.get(k) or "" for k in
+                                 ("name", "domain", "website", "industry", "location")])
+    except Exception as e:
+        logger.warning("Cohort export failed: %s", e)
+    return PlainTextResponse(
+        buf.getvalue(),
+        media_type="text/csv",
+        headers={"Content-Disposition": 'attachment; filename="cohort.csv"'},
+    )
+
+
 @app.get("/api/runs")
 async def get_runs_api():
     """The collector run log — what ran, when, what it produced and cost."""

harvey/web/app.js

@@ -582,6 +582,8 @@ function renderDiscoverResult(report, running) {
 // ── Cohort builder: a prospect list is a query ──
 
 let _cohort = { require: new Set(), exclude: new Set() };
+let _cohortOffset = 0;
+const COHORT_PAGE = 50;
 
 function renderCohortBuilder() {
   const el = document.getElementById('cohort-builder');
@@ -620,6 +622,7 @@ function toggleCohort(key, code, on) {
   // A company can't be both required and excluded on the same signal.
   const other = key === 'require' ? 'exclude' : 'require';
   if (on) _cohort[other].delete(code);
+  _cohortOffset = 0;  // a new cohort starts on its first page
   runCohort();
   renderCohortBuilder();
 }
@@ -633,7 +636,8 @@ async function runCohort() {
   const data = await api('/api/cohort', {
     method: 'POST',
     headers: {'Content-Type': 'application/json'},
-    body: JSON.stringify({require: require, exclude: exclude}),
+    body: JSON.stringify({require: require, exclude: exclude,
+                          offset: _cohortOffset, limit: COHORT_PAGE}),
   });
   if (!data) { el.innerHTML = ''; return; }
 
@@ -643,25 +647,46 @@ async function runCohort() {
       (data.size === 1 ? 'y matches' : 'ies match') + ' this cohort right now</div>';
 
   if (data.companies && data.companies.length) {
+    const qs = 'require=' + encodeURIComponent(require.join(',')) +
+      '&exclude=' + encodeURIComponent(exclude.join(','));
     html += '<div class="table-card" style="margin-top:16px"><table><thead><tr>' +
       '<th>Company</th><th>Domain</th><th>Industry</th><th>Location</th>' +
       '</tr></thead><tbody>' +
-      data.companies.slice(0, 50).map(c =>
+      data.companies.map(c =>
         '<tr><td>' + escHtml(c.name) + '</td><td class="muted">' + escHtml(c.domain) + '</td>' +
         '<td class="muted">' + escHtml(c.industry) + '</td>' +
         '<td class="muted">' + escHtml(c.location) + '</td></tr>'
       ).join('') + '</tbody></table></div>';
-    if (data.size > 50) {
-      html += '<p class="muted" style="font-size:12px;margin-top:10px">Showing the first 50.</p>';
-    }
+
+    const first = data.offset + 1, last = data.offset + data.companies.length;
+    const hasPrev = data.offset > 0, hasNext = last < data.size;
+    html += '<div style="display:flex;align-items:center;gap:8px;margin-top:10px">' +
+      '<span class="muted" style="font-size:12px">Showing ' + first + '&ndash;' + last +
+        ' of ' + data.size + '</span>' +
+      '<span style="flex:1"></span>' +
+      '<button class="btn btn-secondary btn-sm" onclick="pageCohort(-1)"' +
+        (hasPrev ? '' : ' disabled') + '>Prev</button>' +
+      '<button class="btn btn-secondary btn-sm" onclick="pageCohort(1)"' +
+        (hasNext ? '' : ' disabled') + '>Next</button>' +
+      '<a class="btn btn-secondary btn-sm" href="/api/cohort.csv?' + qs + '" download>Export CSV</a>' +
+      '</div>';
   } else if (data.size === 0) {
     html += '<p class="muted" style="font-size:13px;margin-top:12px">' +
       'No company carries all of those yet. Either loosen the cohort, or run ' +
       'prospecting to collect more.</p>';
+  } else if (_cohortOffset > 0) {
+    // The cohort shrank under us (a re-profile, a delete): back to page one.
+    _cohortOffset = 0;
+    return runCohort();
   }
   el.innerHTML = html + '</div>';
 }
 
+function pageCohort(dir) {
+  _cohortOffset = Math.max(0, _cohortOffset + dir * COHORT_PAGE);
+  runCohort();
+}
+
 // ── Settings ──
 
 async function loadSettings() {

tests/test_dashboard_signals.py

@@ -143,6 +143,63 @@ def test_cohort_is_a_set_intersection(client):
     assert excluded["companies"][0]["name"] == "Alpha Roofing"
 
 
+async def _seed_large_cohort(db_path, n):
+    state = StateManager(str(db_path))
+    await state.init_db()
+    from harvey.signals import seed_signal_catalog
+    await seed_signal_catalog(state)
+    from harvey.models import Company
+
+    ids = [await state.add_company(Company(name=f"Co {i:03d}", domain=f"co{i:03d}.com"))
+           for i in range(n)]
+    await state.add_observations([
+        {"company_id": cid, "signal_code": "RUNNING_GOOGLE_ADS", "value_num": 1}
+        for cid in ids
+    ])
+
+
+def test_cohort_pages_through_every_match(client):
+    """Past the first page used to be unreachable: 147 matches showed 50."""
+    import asyncio
+    asyncio.run(_seed_large_cohort(client.db_path, 120))
+
+    seen = []
+    for offset in (0, 50, 100):
+        page = client.post("/api/cohort", json={
+            "require": ["RUNNING_GOOGLE_ADS"], "offset": offset, "limit": 50}).json()
+        assert page["size"] == 120
+        assert page["offset"] == offset
+        seen += [c["name"] for c in page["companies"]]
+
+    assert len(seen) == 120
+    assert len(set(seen)) == 120, "pages must not overlap"
+    assert seen == sorted(seen), "pages are in a stable order"
+
+
+def test_cohort_page_size_is_capped(client):
+    import asyncio
+    asyncio.run(_seed_large_cohort(client.db_path, 3))
+    page = client.post("/api/cohort", json={
+        "require": ["RUNNING_GOOGLE_ADS"], "limit": 100000}).json()
+    assert page["limit"] == dash.COHORT_MAX_PAGE_SIZE
+
+
+def test_cohort_csv_exports_the_whole_cohort(client):
+    import asyncio
+    asyncio.run(_seed_large_cohort(client.db_path, 120))
+    r = client.get("/api/cohort.csv", params={"require": "RUNNING_GOOGLE_ADS"})
+    assert r.status_code == 200
+    assert "text/csv" in r.headers["content-type"]
+    lines = r.text.strip().splitlines()
+    assert lines[0] == "name,domain,website,industry,location"
+    assert len(lines) == 121  # header + every match, not one page
+
+
+def test_cohort_csv_with_no_requirements_is_header_only(client):
+    r = client.get("/api/cohort.csv")
+    assert r.text.strip() == "name,domain,website,industry,location"
+
+
 def test_cohort_with_no_requirements_is_empty_not_everything(client):
     """An empty cohort must never mean 'select all' — that would silently
     turn a mis-click into a campaign against the entire database."""