fix(search): rank results by page title, path, and heading

.changeset/search-ranking-page-match.md

@@ -0,0 +1,11 @@
+---
+"leadtype": minor
+---
+
+Rank docs search results by the page they come from, not only by term counts. Regenerate search artifacts with this release: the index format is now version 4. The browser search client, MCP artifact loading and NLWeb `/ask` reject a version 3 index with an error that says to regenerate, instead of searching it with the new tokenizer and quietly missing matches.
+
+- A chunk's score is multiplied by how much of the query its page title, URL path and own heading cover. A page titled "Consent categories" now outranks a release note that repeats both words, and `vue quickstart` finds `/docs/frameworks/vue/quickstart` when the title is only "Quickstart". URL path words a page doesn't already use are indexed once, so a page that matches only through its URL is still found.
+- Camel-case names index their parts and short dotted or hyphenated names index their joined form, so `consent banner` matches a page titled `ConsentBanner` and `nextjs` matches "Next.js".
+- A page's description is indexed with its first chunk only. Repeating it in every chunk let pages with long descriptions outrank the sections that answer the query. The index is also smaller.
+- `searchDocs` returns at most two chunks per page before applying `limit`. Set `maxResultsPerPage` to change this. NLWeb `/ask` uses `maxResultsPerPage: 1`, so it now fills its result limit instead of returning fewer results after removing duplicate pages.
+- Queries with two or more words skip phrase matching for chunks that cannot reach the results, which makes broad queries several times faster on large indexes.

docs/reference/search.mdx

@@ -20,6 +20,8 @@ and [Agent search tools](/docs/search/agent-tools).
 - **Index** (`search-index.json`) — compact term postings + document/chunk refs. Loaded on every search.
 - **Content store** (`search-content.json`) — chunk text, separate from the index so search stays cheap and content loads lazily.
 - **BM25** — the ranking function. Weights: title 4×, headings 2×, body 1×, code 0.35×.
+- **Page match** — a multiplier on each chunk's BM25 score for how much of the query its page title, URL path and own heading cover. A short title that matches the query beats a long title that only contains it. The path brings in context titles often leave out, such as the framework in `/docs/frameworks/react/quickstart`. URL path words a chunk doesn't already contain are indexed once as body text, so a page that matches only through its URL is still found.
+- **Tokens** — words split on letters and digits, lowercased, accents and stopwords removed. Camel-case names also index their parts (`ConsentBanner` → `consentbanner`, `consent`, `banner`), and short dotted or hyphenated names also index their joined form (`Next.js` → `next`, `js`, `nextjs`).
 
 ## Build time — `leadtype/search/node`
 
@@ -46,9 +48,14 @@ searchDocs(index: DocsSearchIndex, query: string, options?: {
   content?: DocsSearchContentStore;  // enables excerpts
   synonyms?: Record<string, string[]>;
   limit?: number;                    // default 8
+  maxResultsPerPage?: number;        // default 2, applied before limit
 }): DocsSearchResult[]
 ```
 
+`maxResultsPerPage` keeps one long page from filling every result. Set it to `1`
+for a list of pages. Filter or merge results after the call only with a larger
+`limit`, or the list can come back shorter than you asked for.
+
 `DocsSearchResult` carries `title`, `urlPath`, `urlWithHash`, `headingPath`,
 `score`, and (with `content`) `excerpt`. The runtime is edge-safe — no Node APIs.
 
@@ -134,6 +141,7 @@ Objects) in production. Defaults live in `docsSearchDefaults`:
 | `maxContextChars` | 12000 |
 | `maxSources` | 6 |
 | `searchLimit` | 8 |
+| `maxResultsPerPage` | 2 |
 | `maxBodyBytes` | 16 KB |
 
 ## When to add embeddings

docs/search/add-search.mdx

@@ -49,7 +49,7 @@ For server routes or any code that already has the JSON, query directly with
 import type { DocsSearchContentStore } from "leadtype/search";
 
 const contentJson: DocsSearchContentStore = {
-  version: 3,
+  version: 4,
   generatedAt: "2026-06-15T10:30:00.000Z",
   chunks: [],
   codeChunks: [],
@@ -61,7 +61,7 @@ export default contentJson;
 import type { DocsSearchIndex } from "leadtype/search";
 
 const indexJson: DocsSearchIndex = {
-  version: 3,
+  version: 4,
   generatedAt: "2026-06-15T10:30:00.000Z",
   documents: [],
   chunks: [],

docs/search/ai-answers.mdx

@@ -134,14 +134,14 @@ import type {
 } from "leadtype/search";
 
 export const docsSearchContent: DocsSearchContentStore = {
-  version: 3,
+  version: 4,
   generatedAt: "2026-06-15T10:30:00.000Z",
   chunks: [],
   codeChunks: [],
 };
 
 export const docsSearchIndex: DocsSearchIndex = {
-  version: 3,
+  version: 4,
   generatedAt: "2026-06-15T10:30:00.000Z",
   documents: [],
   chunks: [],

packages/leadtype/src/mcp/artifacts.ts

@@ -15,7 +15,10 @@ import type {
   DocsSearchContentStore,
   DocsSearchIndex,
 } from "../search/index.js";
-import { isDocsSearchContentStoreCompatible } from "../search/search.js";
+import {
+  docsSearchIndexVersionError,
+  isDocsSearchContentStoreCompatible,
+} from "../search/search.js";
 
 /**
  * Subdirectory (under the artifacts base) that `generate` writes docs artifacts
@@ -89,6 +92,13 @@ export async function loadDocsArtifacts(
   if (!index) {
     throw missingArtifactError(baseDir, SEARCH_INDEX_FILE);
   }
+  const versionError = docsSearchIndexVersionError(
+    index,
+    path.join(docsDir, SEARCH_INDEX_FILE)
+  );
+  if (versionError) {
+    throw versionError;
+  }
 
   const rawManifest = await readOptionalJson<unknown>(
     path.join(docsDir, MANIFEST_FILE)

packages/leadtype/src/mcp/mcp.test.ts

@@ -307,6 +307,35 @@ describe("loadDocsArtifacts (from disk)", () => {
     expect(textOf(result)).toBe(QUICKSTART_MARKDOWN);
   });
 
+  it("rejects a search index from an older leadtype", async () => {
+    const staleDir = await mkdtemp(join(tmpdir(), "leadtype-mcp-stale-"));
+    try {
+      const docsDir = join(staleDir, "docs");
+      await mkdir(docsDir, { recursive: true });
+      const index = createDocsSearchIndex(docs);
+      await Promise.all([
+        writeFile(
+          join(docsDir, "search-index.json"),
+          JSON.stringify({
+            ...index,
+            version: 3,
+            content: index.content && { ...index.content, version: 3 },
+          })
+        ),
+        writeFile(
+          join(docsDir, "agent-readability.json"),
+          JSON.stringify(buildManifest())
+        ),
+      ]);
+
+      await expect(loadDocsArtifacts({ artifacts: staleDir })).rejects.toThrow(
+        "uses an unsupported version (3)"
+      );
+    } finally {
+      await rm(staleDir, { recursive: true, force: true });
+    }
+  });
+
   it("ignores legacy split content and keeps index-only search available", async () => {
     const legacyDir = await mkdtemp(join(tmpdir(), "leadtype-mcp-legacy-"));
     try {

packages/leadtype/src/next/client.test.ts

@@ -105,6 +105,30 @@ describe("createSearchClient", () => {
     expect(results[0]?.title).toBe("Install");
   });
 
+  it("rejects a search index from an older leadtype", async () => {
+    const bundle = buildSearchBundle();
+    const staleIndex = { ...bundle.index, version: 3 };
+    const staleContent = { ...bundle.content, version: 3 };
+    const scope = nextScope();
+    const fetchImpl = ((input: RequestInfo | URL) => {
+      const url = typeof input === "string" ? input : input.toString();
+      if (url.endsWith(scope.indexUrl)) {
+        return Promise.resolve(Response.json(staleIndex));
+      }
+      return Promise.resolve(Response.json(staleContent));
+    }) as typeof fetch;
+
+    const client = createSearchClient("docs", {
+      indexUrl: scope.indexUrl,
+      contentUrl: scope.contentUrl,
+      fetch: fetchImpl,
+    });
+
+    await expect(client.search("install")).rejects.toThrow(
+      "uses an unsupported version (3)"
+    );
+  });
+
   it("treats a missing content file as a soft failure", async () => {
     const bundle = buildSearchBundle();
     const scope = nextScope();

packages/leadtype/src/nlweb/ask.ts

@@ -9,7 +9,7 @@ import {
   searchDocs,
 } from "../search/index.js";
 import {
-  countDocsSearchTerms,
+  countDocsSearchChunkTerms,
   DOCS_SEARCH_INDEX_VERSION,
   tokenizeDocsSearchText,
 } from "../search/search.js";
@@ -593,13 +593,20 @@ function assertNlwebArtifacts(artifacts: DocsArtifacts): void {
           ) {
             return false;
           }
+          const counts = countDocsSearchChunkTerms({
+            title: document[1],
+            headingPath: chunk[3],
+            text: contentText,
+            codeText,
+            urlPath: document[3],
+          });
           addExpectedVisiblePostings(
             expectedVisiblePostings,
             chunkIndex,
-            countDocsSearchTerms(document[1]),
-            countDocsSearchTerms(chunk[3].join(" ")),
-            countDocsSearchTerms([document[2], contentText].join(" ")),
-            countDocsSearchTerms(codeText)
+            counts.title,
+            counts.heading,
+            counts.body,
+            counts.code
           );
           chunkIds.add(chunkId);
           totalChunkLength += chunk[4];
@@ -688,20 +695,6 @@ function toNlwebResult(
   };
 }
 
-/** Search hits are heading-level; keep the best hit per page. */
-function dedupeByPage(results: DocsSearchResult[]): DocsSearchResult[] {
-  const seen = new Set<string>();
-  const deduped: DocsSearchResult[] = [];
-  for (const result of results) {
-    if (seen.has(result.urlPath)) {
-      continue;
-    }
-    seen.add(result.urlPath);
-    deduped.push(result);
-  }
-  return deduped;
-}
-
 function sseEvent(event: string, data: unknown): string {
   return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
 }
@@ -838,12 +831,11 @@ export function createAskHandler(
 
       const site = resolveSite(config, artifacts);
       const productName = artifacts.manifest.product.name;
-      const hits = dedupeByPage(
-        searchDocs(artifacts.index, parsed.query, {
-          limit,
-          content: artifacts.content,
-        })
-      );
+      const hits = searchDocs(artifacts.index, parsed.query, {
+        limit,
+        maxResultsPerPage: 1,
+        content: artifacts.content,
+      });
       const results = hits.map((hit) => toNlwebResult(hit, site, productName));
       failureCode = NLWEB_ERROR_CODES.internalError;
 

packages/leadtype/src/search/client.ts

@@ -5,6 +5,7 @@ import type {
   DocsSearchResult,
 } from "./search";
 import {
+  docsSearchIndexVersionError,
   isDocsSearchContentStoreCompatible,
   listDocsSearchDocuments,
   searchDocs,
@@ -100,6 +101,10 @@ async function loadArtifacts(
         () => undefined
       ),
     ]);
+    const versionError = docsSearchIndexVersionError(index, indexUrl);
+    if (versionError) {
+      throw versionError;
+    }
     const compatibleContent = isDocsSearchContentStoreCompatible(
       index,
       content