From 214f17d7247177e0523a23a66ef0b992d81e95b8 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:14:43 -0700 Subject: [PATCH 1/6] fix(search): rank results by page title, path, and heading Docs sites with one copy of each guide per framework got poor results: title matches saturated into one more BM25 occurrence, the framework only lived in the URL, camel-case names were single tokens, every chunk repeated the page description, and one page could fill every result. - Multiply each chunk's score by how much of the query its page title, URL path, and heading cover, weighted by title precision. - Index camel-case parts and joined dotted/hyphenated compounds. - Index the page description with its first chunk only. - Add maxResultsPerPage (default 2), applied before limit; NLWeb uses 1. - Skip phrase matching for chunks that cannot reach the results. Bumps the search index format to version 4. --- .changeset/search-ranking-page-match.md | 11 + docs/reference/search.mdx | 8 + docs/search/add-search.mdx | 4 +- docs/search/ai-answers.mdx | 4 +- packages/leadtype/src/nlweb/ask.ts | 26 +- packages/leadtype/src/search/search.test.ts | 116 +++++++- packages/leadtype/src/search/search.ts | 306 ++++++++++++++++++-- 7 files changed, 420 insertions(+), 55 deletions(-) create mode 100644 .changeset/search-ranking-page-match.md diff --git a/.changeset/search-ranking-page-match.md b/.changeset/search-ranking-page-match.md new file mode 100644 index 000000000..1aa1a8ba0 --- /dev/null +++ b/.changeset/search-ranking-page-match.md @@ -0,0 +1,11 @@ +--- +"leadtype": minor +--- + +Rank docs search results by the page they come from, not only by term counts. Regenerate search artifacts with this release: the index format is now version 4, and version 3 artifacts fail validation. + +- A chunk's score is multiplied by how much of the query its page title, URL path and own heading cover. A page titled "Consent categories" now outranks a release note that repeats both words, and `vue quickstart` finds `/docs/frameworks/vue/quickstart` when the title is only "Quickstart". +- Camel-case names index their parts and short dotted or hyphenated names index their joined form, so `consent banner` matches a page titled `ConsentBanner` and `nextjs` matches "Next.js". +- A page's description is indexed with its first chunk only. Repeating it in every chunk let pages with long descriptions outrank the sections that answer the query. The index is also smaller. +- `searchDocs` returns at most two chunks per page before applying `limit`. Set `maxResultsPerPage` to change this. NLWeb `/ask` uses `maxResultsPerPage: 1`, so it now fills its result limit instead of returning fewer results after removing duplicate pages. +- Queries with two or more words skip phrase matching for chunks that cannot reach the results, which makes broad queries several times faster on large indexes. diff --git a/docs/reference/search.mdx b/docs/reference/search.mdx index 7262030ae..b516f0c96 100644 --- a/docs/reference/search.mdx +++ b/docs/reference/search.mdx @@ -20,6 +20,8 @@ and [Agent search tools](/docs/search/agent-tools). - **Index** (`search-index.json`) — compact term postings + document/chunk refs. Loaded on every search. - **Content store** (`search-content.json`) — chunk text, separate from the index so search stays cheap and content loads lazily. - **BM25** — the ranking function. Weights: title 4×, headings 2×, body 1×, code 0.35×. +- **Page match** — a multiplier on each chunk's BM25 score for how much of the query its page title, URL path and own heading cover. A short title that matches the query beats a long title that only contains it. The path brings in context titles often leave out, such as the framework in `/docs/frameworks/react/quickstart`. +- **Tokens** — words split on letters and digits, lowercased, accents and stopwords removed. Camel-case names also index their parts (`ConsentBanner` → `consentbanner`, `consent`, `banner`), and short dotted or hyphenated names also index their joined form (`Next.js` → `next`, `js`, `nextjs`). ## Build time — `leadtype/search/node` @@ -46,9 +48,14 @@ searchDocs(index: DocsSearchIndex, query: string, options?: { content?: DocsSearchContentStore; // enables excerpts synonyms?: Record; limit?: number; // default 8 + maxResultsPerPage?: number; // default 2, applied before limit }): DocsSearchResult[] ``` +`maxResultsPerPage` keeps one long page from filling every result. Set it to `1` +for a list of pages. Filter or merge results after the call only with a larger +`limit`, or the list can come back shorter than you asked for. + `DocsSearchResult` carries `title`, `urlPath`, `urlWithHash`, `headingPath`, `score`, and (with `content`) `excerpt`. The runtime is edge-safe — no Node APIs. @@ -134,6 +141,7 @@ Objects) in production. Defaults live in `docsSearchDefaults`: | `maxContextChars` | 12000 | | `maxSources` | 6 | | `searchLimit` | 8 | +| `maxResultsPerPage` | 2 | | `maxBodyBytes` | 16 KB | ## When to add embeddings diff --git a/docs/search/add-search.mdx b/docs/search/add-search.mdx index d0dbeceff..fa95e7c34 100644 --- a/docs/search/add-search.mdx +++ b/docs/search/add-search.mdx @@ -49,7 +49,7 @@ For server routes or any code that already has the JSON, query directly with import type { DocsSearchContentStore } from "leadtype/search"; const contentJson: DocsSearchContentStore = { - version: 3, + version: 4, generatedAt: "2026-06-15T10:30:00.000Z", chunks: [], codeChunks: [], @@ -61,7 +61,7 @@ export default contentJson; import type { DocsSearchIndex } from "leadtype/search"; const indexJson: DocsSearchIndex = { - version: 3, + version: 4, generatedAt: "2026-06-15T10:30:00.000Z", documents: [], chunks: [], diff --git a/docs/search/ai-answers.mdx b/docs/search/ai-answers.mdx index dbb61a895..5d2a0775b 100644 --- a/docs/search/ai-answers.mdx +++ b/docs/search/ai-answers.mdx @@ -134,14 +134,14 @@ import type { } from "leadtype/search"; export const docsSearchContent: DocsSearchContentStore = { - version: 3, + version: 4, generatedAt: "2026-06-15T10:30:00.000Z", chunks: [], codeChunks: [], }; export const docsSearchIndex: DocsSearchIndex = { - version: 3, + version: 4, generatedAt: "2026-06-15T10:30:00.000Z", documents: [], chunks: [], diff --git a/packages/leadtype/src/nlweb/ask.ts b/packages/leadtype/src/nlweb/ask.ts index a1113b671..d87dbb49d 100644 --- a/packages/leadtype/src/nlweb/ask.ts +++ b/packages/leadtype/src/nlweb/ask.ts @@ -598,7 +598,7 @@ function assertNlwebArtifacts(artifacts: DocsArtifacts): void { chunkIndex, countDocsSearchTerms(document[1]), countDocsSearchTerms(chunk[3].join(" ")), - countDocsSearchTerms([document[2], contentText].join(" ")), + countDocsSearchTerms(contentText), countDocsSearchTerms(codeText) ); chunkIds.add(chunkId); @@ -689,19 +689,6 @@ function toNlwebResult( } /** Search hits are heading-level; keep the best hit per page. */ -function dedupeByPage(results: DocsSearchResult[]): DocsSearchResult[] { - const seen = new Set(); - const deduped: DocsSearchResult[] = []; - for (const result of results) { - if (seen.has(result.urlPath)) { - continue; - } - seen.add(result.urlPath); - deduped.push(result); - } - return deduped; -} - function sseEvent(event: string, data: unknown): string { return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; } @@ -838,12 +825,11 @@ export function createAskHandler( const site = resolveSite(config, artifacts); const productName = artifacts.manifest.product.name; - const hits = dedupeByPage( - searchDocs(artifacts.index, parsed.query, { - limit, - content: artifacts.content, - }) - ); + const hits = searchDocs(artifacts.index, parsed.query, { + limit, + maxResultsPerPage: 1, + content: artifacts.content, + }); const results = hits.map((hit) => toNlwebResult(hit, site, productName)); failureCode = NLWEB_ERROR_CODES.internalError; diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 42b1cde75..668b5335f 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -17,6 +17,7 @@ import { slugifyDocsHeading, validateDocsQuery, } from "./index"; +import { DOCS_SEARCH_INDEX_VERSION, tokenizeDocsSearchText } from "./search"; const CHUNK_ANCHOR_INDEX = 2; @@ -99,7 +100,7 @@ describe("createDocsSearchIndex and searchDocs", () => { generatedAt: "2026-01-01T00:00:00.000Z", }); - expect(index.version).toBe(3); + expect(index.version).toBe(DOCS_SEARCH_INDEX_VERSION); expect(index.documents[0]).toEqual([ "quickstart", "Quickstart", @@ -110,7 +111,7 @@ describe("createDocsSearchIndex and searchDocs", () => { ]); expect(index.chunks[0]).toHaveLength(6); expect(index.chunks[0]).not.toHaveProperty("text"); - expect(index.content?.version).toBe(3); + expect(index.content?.version).toBe(DOCS_SEARCH_INDEX_VERSION); expect(index.content?.chunks[0]).toContain("Install the package"); expect(index.content?.codeChunks).toHaveLength(index.chunks.length); expect( @@ -309,6 +310,117 @@ describe("createDocsSearchIndex and searchDocs", () => { expect(searchDocs(index, "unreleased")).toEqual([]); }); + it("splits camel-case and dotted names into searchable words", () => { + expect(tokenizeDocsSearchText("ConsentBanner in Next.js")).toEqual([ + "consentbanner", + "consent", + "banner", + "next", + "js", + "nextjs", + ]); + }); + + it("ranks the page titled for the query above a release that repeats it", () => { + const filler = ["Scripts", "Callbacks", "Hooks", "Theming", "Iframes"].map( + (title): DocsSearchDocument => ({ + id: title, + title, + urlPath: `/docs/${title}`, + absoluteUrl: `https://leadtype.dev/docs/${title}`, + relativePath: title, + content: `## About ${title}\n\n${title} control what loads before a visitor chooses.`, + }) + ); + const index = createDocsSearchIndex( + [ + { + id: "release", + title: "v1.3.0: Translations package, SSR consent, CLI telemetry", + description: + "Adds the translations package, backend translations and translations for the CLI.", + urlPath: "/changelog/1.3.0", + absoluteUrl: "https://leadtype.dev/changelog/1.3.0", + relativePath: "changelog/1.3.0", + content: [ + "## New translations package\n\nTranslations ship as their own package.", + "## Backend\n\nThe backend serves translations.", + "## CLI\n\nThe CLI writes translations.", + ].join("\n\n"), + }, + { + id: "translations", + title: "Translations", + description: "Change the copy and add languages.", + urlPath: "/docs/customization/translations", + absoluteUrl: "https://leadtype.dev/docs/customization/translations", + relativePath: "customization/translations", + content: + "## Change the copy\n\nOverride any string the banner and dialog show, then add a language your visitors read. Strings fall back to English when a key is missing.", + }, + ...filler, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "translations")[0]?.documentId).toBe( + "translations" + ); + }); + + it("matches words that only appear in the page URL", () => { + const quickstart = (framework: string): DocsSearchDocument => ({ + id: framework, + title: "Quickstart", + urlPath: `/docs/frameworks/${framework}/quickstart`, + absoluteUrl: `https://leadtype.dev/docs/frameworks/${framework}/quickstart`, + relativePath: `frameworks/${framework}/quickstart`, + content: "## Install the package\n\nAdd the provider to your app.", + }); + const index = createDocsSearchIndex( + [quickstart("react"), quickstart("vue")], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "vue quickstart")[0]?.documentId).toBe("vue"); + }); + + it("caps how many sections one page contributes before the limit", () => { + const sections = Array.from( + { length: 6 }, + (_, section) => + `## Analytics setup ${section}\n\nConfigure analytics step ${section}.` + ).join("\n\n"); + const index = createDocsSearchIndex( + [ + { + id: "long", + title: "Analytics", + urlPath: "/docs/analytics", + absoluteUrl: "https://leadtype.dev/docs/analytics", + relativePath: "analytics", + content: sections, + }, + { + id: "other", + title: "Tag managers", + urlPath: "/docs/tag-managers", + absoluteUrl: "https://leadtype.dev/docs/tag-managers", + relativePath: "tag-managers", + content: "## Load analytics\n\nA tag manager can load analytics.", + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + const pages = (maxResultsPerPage?: number) => + searchDocs(index, "analytics", { limit: 3, maxResultsPerPage }).map( + (result) => result.documentId + ); + expect(pages()).toEqual(["long", "long", "other"]); + expect(pages(1)).toEqual(["long", "other"]); + }); + it("falls back to prefix and typo-tolerant matches", () => { const index = createDocsSearchIndex(docs, { generatedAt: "2026-01-01T00:00:00.000Z", diff --git a/packages/leadtype/src/search/search.ts b/packages/leadtype/src/search/search.ts index 9cb17323e..7e66f25e1 100644 --- a/packages/leadtype/src/search/search.ts +++ b/packages/leadtype/src/search/search.ts @@ -14,12 +14,13 @@ import { const DEFAULT_MAX_CHUNK_CHARS = 1200; const DEFAULT_OVERLAP_CHARS = 160; const DEFAULT_SEARCH_LIMIT = 8; +const DEFAULT_MAX_RESULTS_PER_PAGE = 2; const DEFAULT_MAX_QUERY_CHARS = 400; const DEFAULT_ASK_MAX_QUERY_CHARS = 600; const DEFAULT_MAX_BODY_BYTES = 16 * 1024; const DEFAULT_MAX_SOURCES = 6; const DEFAULT_MAX_CONTEXT_CHARS = 12_000; -export const DOCS_SEARCH_INDEX_VERSION = 3; +export const DOCS_SEARCH_INDEX_VERSION = 4; const TITLE_WEIGHT = 4; const HEADING_WEIGHT = 2; const BODY_WEIGHT = 1; @@ -33,6 +34,10 @@ const PREFIX_TERM_WEIGHT = 0.55; const TYPO_TERM_WEIGHT = 0.45; const PHRASE_MATCH_BOOST = 1.4; const PROXIMITY_MATCH_BOOST = 0.8; +const TITLE_MATCH_BOOST = 1; +const TITLE_PRECISION_WEIGHT = 2; +const PATH_MATCH_BOOST = 0.5; +const HEADING_MATCH_BOOST = 0.5; const MAX_PREFIX_EXPANSIONS = 24; const MAX_TYPO_EXPANSIONS = 16; const PROXIMITY_WINDOW = 8; @@ -42,7 +47,18 @@ const WHITESPACE_PATTERN = /\s+/g; // Tokenizer behavior is serialized into chunk lengths and postings. Changes // to word matching, normalization, stopwords, or token filtering must bump // DOCS_SEARCH_INDEX_VERSION so older artifacts fail with a version mismatch. -const WORD_CHARACTER_PATTERN = /[\p{L}\p{N}]+/gu; +// A word plus any `.`, `-` or `_` joined continuation, so `Next.js`, +// `tanstack-start` and `snake_case` reach the tokenizer as one compound. +const COMPOUND_WORD_PATTERN = /[\p{L}\p{N}]+(?:[._-][\p{L}\p{N}]+)*/gu; +const COMPOUND_SEPARATOR_PATTERN = /[._-]/; +// `ConsentBanner` → `Consent|Banner`, `IABConsent` → `IAB|Consent`. +const CAMEL_CASE_BOUNDARY_PATTERN = + /(?<=\p{Ll})(?=\p{Lu})|(?<=\p{Lu})(?=\p{Lu}\p{Ll})/u; +// Same boundaries without lookbehind: a cheap test that skips the split for +// the plain words that make up most text. +const HAS_CAMEL_CASE_BOUNDARY_PATTERN = /\p{Ll}\p{Lu}|\p{Lu}\p{Lu}\p{Ll}/u; +const DIGITS_ONLY_PATTERN = /^\p{N}+$/u; +const MAX_JOINED_COMPOUND_PARTS = 3; const DIACRITIC_PATTERN = /[\u0300-\u036f]/g; const SHARED_ROUTE_SEGMENTS = new Set(["shared", "_shared"]); @@ -218,6 +234,11 @@ export type CreateDocsSearchIndexOptions = { export type SearchDocsOptions = ContentStoreOptions & { limit?: number; + /** + * Most sections returned from one page, applied before `limit`. Defaults + * to 2 so a long page whose title matches cannot fill every result. + */ + maxResultsPerPage?: number; synonyms?: Record; }; @@ -340,13 +361,47 @@ function withHash(url: string, anchor: string): string { return anchor ? `${url}#${anchor}` : url; } +function pushSearchToken(tokens: string[], raw: string): void { + const token = raw.toLowerCase(); + if (token.length > 1 && !STOPWORDS.has(token)) { + tokens.push(token); + } +} + +/** + * Tokens for one word, its camel-case parts, and the joined form of short + * dotted or hyphenated compounds. `ConsentBanner` yields `consentbanner`, + * `consent`, `banner`; `Next.js` yields `next`, `js`, `nextjs`. Parts come + * before the joined form so phrase matching sees `next js nextjs app`. + */ +function pushCompoundTokens(tokens: string[], compound: string): void { + const parts = COMPOUND_SEPARATOR_PATTERN.test(compound) + ? compound.split(COMPOUND_SEPARATOR_PATTERN) + : [compound]; + for (const part of parts) { + pushSearchToken(tokens, part); + if (HAS_CAMEL_CASE_BOUNDARY_PATTERN.test(part)) { + for (const word of part.split(CAMEL_CASE_BOUNDARY_PATTERN)) { + pushSearchToken(tokens, word); + } + } + } + if ( + parts.length > 1 && + parts.length <= MAX_JOINED_COMPOUND_PARTS && + !parts.every((part) => DIGITS_ONLY_PATTERN.test(part)) + ) { + pushSearchToken(tokens, parts.join("")); + } +} + export function tokenizeDocsSearchText(input: string): string[] { const tokens: string[] = []; - for (const match of normalizeText(input).matchAll(WORD_CHARACTER_PATTERN)) { - const token = match[0]; - if (token.length > 1 && !STOPWORDS.has(token)) { - tokens.push(token); - } + // Case survives until each token is pushed so camel-case boundaries are + // still visible. + const folded = input.normalize("NFKD").replace(DIACRITIC_PATTERN, ""); + for (const match of folded.matchAll(COMPOUND_WORD_PATTERN)) { + pushCompoundTokens(tokens, match[0]); } return tokens; } @@ -404,16 +459,22 @@ function synonymTokensFor( ); } -function collectWeightedSearchTerms( +/** + * Index terms each distinct query token expands to, with the weight of the + * strongest way it matched (exact, stem, synonym, prefix, typo). + */ +function expandQueryTokens( index: DocsSearchIndex, queryTokens: string[], synonyms?: Record -): WeightedSearchTerm[] { - const weightedTerms = new Map(); +): Map> { + const expansions = new Map>(); const indexTerms = Object.keys(index.terms); const indexTermSet = new Set(indexTerms); - for (const token of queryTokens) { + for (const token of new Set(queryTokens)) { + const weightedTerms = new Map(); + expansions.set(token, weightedTerms); if (indexTermSet.has(token)) { addWeightedTerm(weightedTerms, token, EXACT_TERM_WEIGHT); } @@ -464,9 +525,69 @@ function collectWeightedSearchTerms( } } + return expansions; +} + +function mergeQueryExpansions( + expansions: Map> +): WeightedSearchTerm[] { + const weightedTerms = new Map(); + for (const terms of expansions.values()) { + for (const [term, weight] of terms) { + addWeightedTerm(weightedTerms, term, weight); + } + } return Array.from(weightedTerms, ([term, weight]) => ({ term, weight })); } +/** + * Share of the query a short field answers, from 0 to 1: each query token + * contributes the weight of its best expansion that appears in the field. + */ +function queryCoverage( + expansions: Map>, + fieldTokens: Set +): number { + if (expansions.size === 0 || fieldTokens.size === 0) { + return 0; + } + let covered = 0; + for (const [token, terms] of expansions) { + let best = fieldTokens.has(token) ? EXACT_TERM_WEIGHT : 0; + for (const [term, weight] of terms) { + if (weight > best && fieldTokens.has(term)) { + best = weight; + } + } + covered += best; + } + return covered / expansions.size; +} + +/** + * Share of a field's tokens the query accounts for, from 0 to 1. Separates a + * page titled "Translations" from a release titled "Translations Package, + * SSR Consent, CLI Telemetry, and Next.js Restructure". + */ +function fieldPrecision( + expansions: Map>, + fieldTokens: Set +): number { + if (fieldTokens.size === 0) { + return 0; + } + let matched = 0; + for (const fieldToken of fieldTokens) { + for (const [token, terms] of expansions) { + if (token === fieldToken || terms.has(fieldToken)) { + matched += 1; + break; + } + } + } + return matched / fieldTokens.size; +} + export function countDocsSearchTerms(input: string): Map { const counts = new Map(); for (const token of tokenizeDocsSearchText(input)) { @@ -748,6 +869,129 @@ function scoreContentMatchBoost(text: string, queryTokens: string[]): number { : 0; } +type PageMatchTokens = { + title: Set; + titleAndPath: Set; +}; + +/** + * Score multiplier for chunks whose page title, URL path or own heading + * answers the query. BM25 alone saturates a title match into one more term + * occurrence, so a page titled "Consent categories" loses to a changelog + * that repeats both words. The path carries context titles often leave out, + * such as the framework in `/docs/frameworks/react/quickstart`. + */ +function pageMatchMultiplier( + chunk: DocsSearchChunk, + expansions: Map>, + cache: Map +): number { + let page = cache.get(chunk.documentId); + if (!page) { + const title = new Set(tokenizeDocsSearchText(chunk.title)); + page = { + title, + titleAndPath: new Set([ + ...title, + ...tokenizeDocsSearchText(chunk.urlPath), + ]), + }; + cache.set(chunk.documentId, page); + } + const heading = new Set( + tokenizeDocsSearchText(chunk.headingPath.at(-1) ?? "") + ); + return ( + 1 + + TITLE_MATCH_BOOST * + queryCoverage(expansions, page.title) * + (1 + TITLE_PRECISION_WEIGHT * fieldPrecision(expansions, page.title)) + + PATH_MATCH_BOOST * queryCoverage(expansions, page.titleAndPath) + + HEADING_MATCH_BOOST * queryCoverage(expansions, heading) + ); +} + +type ScoredChunk = { + chunk: DocsSearchChunk; + termScore: number; + multiplier: number; + score: number; +}; + +const MAX_PAGE_MATCH_MULTIPLIER = + 1 + + TITLE_MATCH_BOOST * (1 + TITLE_PRECISION_WEIGHT) + + PATH_MATCH_BOOST + + HEADING_MATCH_BOOST; +const CONTENT_BOOST_BATCH_SIZE = 32; + +function compareScoredChunks(left: ScoredChunk, right: ScoredChunk): number { + if (right.score !== left.score) { + return right.score - left.score; + } + return left.chunk.absoluteUrl.localeCompare(right.chunk.absoluteUrl); +} + +/** Best chunks in score order, at most `maxResultsPerPage` from each page. */ +function selectTopChunks( + sorted: ScoredChunk[], + limit: number, + maxResultsPerPage: number +): ScoredChunk[] { + const resultsPerPage = new Map(); + const selected: ScoredChunk[] = []; + for (const scored of sorted) { + if (selected.length >= limit) { + break; + } + const pageCount = resultsPerPage.get(scored.chunk.documentId) ?? 0; + if (pageCount < maxResultsPerPage) { + resultsPerPage.set(scored.chunk.documentId, pageCount + 1); + selected.push(scored); + } + } + return selected; +} + +/** + * Adds the phrase and proximity boost in place. Matching tokenizes the whole + * chunk text, the costliest step of a query, so chunks are boosted in score + * order and the rest are skipped once even the largest boost could not lift + * them into the results. Adding chunks never lowers the score of the last + * selected result, so a chunk below that bound cannot be selected. + */ +function applyContentMatchBoosts( + scoredChunks: ScoredChunk[], + queryTokens: string[], + limit: number, + maxResultsPerPage: number +): void { + if (queryTokens.length < 2) { + return; + } + scoredChunks.sort(compareScoredChunks); + const maxBoost = PHRASE_MATCH_BOOST * MAX_PAGE_MATCH_MULTIPLIER; + let lastSelectedScore = Number.NEGATIVE_INFINITY; + for (const [position, scored] of scoredChunks.entries()) { + if (scored.score + maxBoost < lastSelectedScore) { + return; + } + scored.score = + (scored.termScore + + scoreContentMatchBoost(scored.chunk.text, queryTokens)) * + scored.multiplier; + if ((position + 1) % CONTENT_BOOST_BATCH_SIZE === 0) { + const boosted = scoredChunks + .slice(0, position + 1) + .sort(compareScoredChunks); + const selected = selectTopChunks(boosted, limit, maxResultsPerPage); + if (selected.length >= limit) { + lastSelectedScore = selected.at(-1)?.score ?? lastSelectedScore; + } + } + } +} + function requestError(message: string, status: number): never { throw new DocsSearchRequestError(message, status); } @@ -961,6 +1205,10 @@ export function createDocsSearchIndex( } documents.push(entry); + // Only the page's first chunk carries its description. Repeating it in + // every chunk let a long description outrank the sections that answer + // the query. + let chunkDescription = description; for (const block of collectSectionBlocks(doc.content)) { const bodyParts = splitWithOverlap( block.text, @@ -978,7 +1226,7 @@ export function createDocsSearchIndex( const codeText = codeParts[partIndex] ?? ""; const chunkText = createChunkText( doc.title, - description, + chunkDescription, block.headingPath, [text, codeText].filter(Boolean).join("\n\n") ); @@ -1020,6 +1268,7 @@ export function createDocsSearchIndex( | undefined ); + chunkDescription = ""; const chunkIndex = mutableChunks.length; const chunkId = `chunk-${chunkIndex}`; const length = countDocsSearchTokens(transformedChunk.text); @@ -1036,9 +1285,7 @@ export function createDocsSearchIndex( chunkTermCounts.set(chunkIndex, { title: countDocsSearchTerms(doc.title), heading: countDocsSearchTerms(transformedChunk.headingPath.join(" ")), - body: countDocsSearchTerms( - [description, transformedChunk.text].join(" ") - ), + body: countDocsSearchTerms(transformedChunk.text), code: countDocsSearchTerms(transformedChunk.codeText), }); } @@ -1105,11 +1352,8 @@ export function searchDocs( return []; } - const weightedTerms = collectWeightedSearchTerms( - index, - queryTokens, - options.synonyms - ); + const expansions = expandQueryTokens(index, queryTokens, options.synonyms); + const weightedTerms = mergeQueryExpansions(expansions); if (weightedTerms.length === 0) { return []; } @@ -1158,30 +1402,33 @@ export function searchDocs( } const limit = options.limit ?? DEFAULT_SEARCH_LIMIT; - const scoredChunks: Array<{ chunk: DocsSearchChunk; score: number }> = []; + const maxResultsPerPage = + options.maxResultsPerPage ?? DEFAULT_MAX_RESULTS_PER_PAGE; + const scoredChunks: ScoredChunk[] = []; + const pageTokens = new Map(); for (const [chunkId, score] of scores) { const chunk = readDocsContentChunk(index, chunkId, options.content); if (!chunk) { continue; } + const multiplier = pageMatchMultiplier(chunk, expansions, pageTokens); scoredChunks.push({ chunk, - score: score + scoreContentMatchBoost(chunk.text, queryTokens), + termScore: score, + multiplier, + score: score * multiplier, }); } - scoredChunks.sort((left, right) => { - if (right.score !== left.score) { - return right.score - left.score; - } - return left.chunk.absoluteUrl.localeCompare(right.chunk.absoluteUrl); - }); + applyContentMatchBoosts(scoredChunks, queryTokens, limit, maxResultsPerPage); + scoredChunks.sort(compareScoredChunks); // Excerpts are expensive string scans, so build them only for results // that survive the limit. const excerptTokens = Array.from( new Set([...queryTokens, ...weightedTerms.map(({ term }) => term)]) ); - return scoredChunks.slice(0, limit).map(({ chunk, score }) => { + const topChunks = selectTopChunks(scoredChunks, limit, maxResultsPerPage); + return topChunks.map(({ chunk, score }) => { const excerptText = chunk.text || [chunk.title, chunk.description, ...chunk.headingPath].join(" "); @@ -1489,6 +1736,7 @@ export const docsSearchDefaults = { maxChunkChars: DEFAULT_MAX_CHUNK_CHARS, maxContextChars: DEFAULT_MAX_CONTEXT_CHARS, maxQueryChars: DEFAULT_MAX_QUERY_CHARS, + maxResultsPerPage: DEFAULT_MAX_RESULTS_PER_PAGE, maxSources: DEFAULT_MAX_SOURCES, overlapChars: DEFAULT_OVERLAP_CHARS, searchLimit: DEFAULT_SEARCH_LIMIT, From b05655322d161b3c3451404aafebbc1eb2f280b4 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:30:15 -0700 Subject: [PATCH 2/6] fix(search): address review on title precision, phrases, and pruning cost - Count title precision per written word, so TypeScript is fully matched by typescript instead of scoring 1/3 for its camel-case parts. - Also compare phrases on the words as written, so the joined nextjs token in Next.js app no longer breaks the phrase next js app. - Refresh the pruning bound at doubling checkpoints, so refreshing stays O(n log n) when scores are flat and nothing is pruned. - Drop the dedupeByPage JSDoc left above sseEvent. --- packages/leadtype/src/nlweb/ask.ts | 1 - packages/leadtype/src/search/search.test.ts | 85 ++++++++++-- packages/leadtype/src/search/search.ts | 139 +++++++++++++++----- 3 files changed, 182 insertions(+), 43 deletions(-) diff --git a/packages/leadtype/src/nlweb/ask.ts b/packages/leadtype/src/nlweb/ask.ts index d87dbb49d..82121b7ee 100644 --- a/packages/leadtype/src/nlweb/ask.ts +++ b/packages/leadtype/src/nlweb/ask.ts @@ -688,7 +688,6 @@ function toNlwebResult( }; } -/** Search hits are heading-level; keep the best hit per page. */ function sseEvent(event: string, data: unknown): string { return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; } diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 668b5335f..65f5efd28 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -94,6 +94,22 @@ const cafe = "café"; }, ]; +/** Unrelated pages, so query terms have realistic document frequencies */ +const fillerDocs: DocsSearchDocument[] = [ + "Scripts", + "Callbacks", + "Hooks", + "Theming", + "Iframes", +].map((title) => ({ + id: title, + title, + urlPath: `/docs/${title}`, + absoluteUrl: `https://leadtype.dev/docs/${title}`, + relativePath: title, + content: `## About ${title}\n\n${title} control what loads before a visitor chooses.`, +})); + describe("createDocsSearchIndex and searchDocs", () => { it("stores compact metadata separately from answer content", () => { const index = createDocsSearchIndex(docs, { @@ -322,16 +338,6 @@ describe("createDocsSearchIndex and searchDocs", () => { }); it("ranks the page titled for the query above a release that repeats it", () => { - const filler = ["Scripts", "Callbacks", "Hooks", "Theming", "Iframes"].map( - (title): DocsSearchDocument => ({ - id: title, - title, - urlPath: `/docs/${title}`, - absoluteUrl: `https://leadtype.dev/docs/${title}`, - relativePath: title, - content: `## About ${title}\n\n${title} control what loads before a visitor chooses.`, - }) - ); const index = createDocsSearchIndex( [ { @@ -358,7 +364,7 @@ describe("createDocsSearchIndex and searchDocs", () => { content: "## Change the copy\n\nOverride any string the banner and dialog show, then add a language your visitors read. Strings fall back to English when a key is missing.", }, - ...filler, + ...fillerDocs, ], { generatedAt: "2026-01-01T00:00:00.000Z" } ); @@ -368,6 +374,63 @@ describe("createDocsSearchIndex and searchDocs", () => { ); }); + it("counts a camel-case title word once when matching the title", () => { + const index = createDocsSearchIndex( + [ + { + id: "typescript", + title: "TypeScript", + urlPath: "/docs/typescript", + absoluteUrl: "https://leadtype.dev/docs/typescript", + relativePath: "typescript", + content: "## Types\n\nThe package ships its own types.", + }, + { + id: "typescript-setup", + title: "Typescript setup", + urlPath: "/docs/typescript-setup", + absoluteUrl: "https://leadtype.dev/docs/typescript-setup", + relativePath: "typescript-setup", + content: + "## Configure\n\nSet strict in tsconfig for typescript projects.", + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "typescript")[0]?.documentId).toBe("typescript"); + }); + + it("matches a phrase written as separate words across a dotted name", () => { + const index = createDocsSearchIndex( + [ + { + id: "phrase", + title: "Routing", + urlPath: "/docs/routing", + absoluteUrl: "https://leadtype.dev/docs/routing", + relativePath: "routing", + content: + "## Routers\n\nThe Next.js app router renders on the server.", + }, + { + id: "scattered", + title: "Routing guide", + urlPath: "/docs/routing-guide", + absoluteUrl: "https://leadtype.dev/docs/routing-guide", + relativePath: "routing-guide", + content: + "## Routers\n\nNext js routes the app. Next js renders the app.", + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "next js app")[0]?.documentId).toBe("phrase"); + }); + it("matches words that only appear in the page URL", () => { const quickstart = (framework: string): DocsSearchDocument => ({ id: framework, diff --git a/packages/leadtype/src/search/search.ts b/packages/leadtype/src/search/search.ts index 7e66f25e1..538014a12 100644 --- a/packages/leadtype/src/search/search.ts +++ b/packages/leadtype/src/search/search.ts @@ -371,22 +371,27 @@ function pushSearchToken(tokens: string[], raw: string): void { /** * Tokens for one word, its camel-case parts, and the joined form of short * dotted or hyphenated compounds. `ConsentBanner` yields `consentbanner`, - * `consent`, `banner`; `Next.js` yields `next`, `js`, `nextjs`. Parts come - * before the joined form so phrase matching sees `next js nextjs app`. + * `consent`, `banner`; `Next.js` yields `next`, `js`, `nextjs`. Without + * `derived`, only the words as written: `consentbanner`; `next`, `js`. */ -function pushCompoundTokens(tokens: string[], compound: string): void { +function pushCompoundTokens( + tokens: string[], + compound: string, + derived = true +): void { const parts = COMPOUND_SEPARATOR_PATTERN.test(compound) ? compound.split(COMPOUND_SEPARATOR_PATTERN) : [compound]; for (const part of parts) { pushSearchToken(tokens, part); - if (HAS_CAMEL_CASE_BOUNDARY_PATTERN.test(part)) { + if (derived && HAS_CAMEL_CASE_BOUNDARY_PATTERN.test(part)) { for (const word of part.split(CAMEL_CASE_BOUNDARY_PATTERN)) { pushSearchToken(tokens, word); } } } if ( + derived && parts.length > 1 && parts.length <= MAX_JOINED_COMPOUND_PARTS && !parts.every((part) => DIGITS_ONLY_PATTERN.test(part)) @@ -395,17 +400,53 @@ function pushCompoundTokens(tokens: string[], compound: string): void { } } +// Case survives until each token is pushed so camel-case boundaries are +// still visible. +function foldForTokenizing(input: string): string { + return input.normalize("NFKD").replace(DIACRITIC_PATTERN, ""); +} + export function tokenizeDocsSearchText(input: string): string[] { const tokens: string[] = []; - // Case survives until each token is pushed so camel-case boundaries are - // still visible. - const folded = input.normalize("NFKD").replace(DIACRITIC_PATTERN, ""); - for (const match of folded.matchAll(COMPOUND_WORD_PATTERN)) { + for (const match of foldForTokenizing(input).matchAll( + COMPOUND_WORD_PATTERN + )) { pushCompoundTokens(tokens, match[0]); } return tokens; } +/** + * Tokens for the words as written, without camel-case parts or joined + * compounds. Phrase matching also compares these, because the joined + * `nextjs` in `next js nextjs app` would otherwise break the phrase + * `next js app`. + */ +function tokenizeWrittenWords(input: string): string[] { + const tokens: string[] = []; + for (const match of foldForTokenizing(input).matchAll( + COMPOUND_WORD_PATTERN + )) { + pushCompoundTokens(tokens, match[0], false); + } + return tokens; +} + +/** Tokens grouped by the word they came from; stopword-only words are dropped. */ +function tokenizeWordGroups(input: string): string[][] { + const words: string[][] = []; + for (const match of foldForTokenizing(input).matchAll( + COMPOUND_WORD_PATTERN + )) { + const tokens: string[] = []; + pushCompoundTokens(tokens, match[0]); + if (tokens.length > 0) { + words.push(tokens); + } + } + return words; +} + export function countDocsSearchTokens(input: string): number { return tokenizeDocsSearchText(input).length; } @@ -564,28 +605,39 @@ function queryCoverage( return covered / expansions.size; } +function isQueryMatch( + expansions: Map>, + fieldToken: string +): boolean { + for (const [token, terms] of expansions) { + if (token === fieldToken || terms.has(fieldToken)) { + return true; + } + } + return false; +} + /** - * Share of a field's tokens the query accounts for, from 0 to 1. Separates a + * Share of a field's words the query accounts for, from 0 to 1. Separates a * page titled "Translations" from a release titled "Translations Package, - * SSR Consent, CLI Telemetry, and Next.js Restructure". + * SSR Consent, CLI Telemetry, and Next.js Restructure". A word counts once + * however many tokens it yields, so `TypeScript` is fully matched by + * `typescript` alone. */ function fieldPrecision( expansions: Map>, - fieldTokens: Set + fieldWords: string[][] ): number { - if (fieldTokens.size === 0) { + if (fieldWords.length === 0) { return 0; } let matched = 0; - for (const fieldToken of fieldTokens) { - for (const [token, terms] of expansions) { - if (token === fieldToken || terms.has(fieldToken)) { - matched += 1; - break; - } + for (const word of fieldWords) { + if (word.some((token) => isQueryMatch(expansions, token))) { + matched += 1; } } - return matched / fieldTokens.size; + return matched / fieldWords.length; } export function countDocsSearchTerms(input: string): Map { @@ -852,14 +904,24 @@ function hasOrderedProximityMatch( return false; } -function scoreContentMatchBoost(text: string, queryTokens: string[]): number { +type QueryPhrase = { + tokens: string[]; + writtenWords: string; +}; + +function scoreContentMatchBoost(text: string, query: QueryPhrase): number { + const queryTokens = query.tokens; if (queryTokens.length < 2) { return 0; } const normalizedQuery = queryTokens.join(" "); const normalizedText = normalizeForPhraseMatch(text); - if (normalizedText.includes(normalizedQuery)) { + if ( + normalizedText.includes(normalizedQuery) || + (query.writtenWords.includes(" ") && + tokenizeWrittenWords(text).join(" ").includes(query.writtenWords)) + ) { return PHRASE_MATCH_BOOST; } @@ -871,6 +933,7 @@ function scoreContentMatchBoost(text: string, queryTokens: string[]): number { type PageMatchTokens = { title: Set; + titleWords: string[][]; titleAndPath: Set; }; @@ -888,9 +951,11 @@ function pageMatchMultiplier( ): number { let page = cache.get(chunk.documentId); if (!page) { - const title = new Set(tokenizeDocsSearchText(chunk.title)); + const titleWords = tokenizeWordGroups(chunk.title); + const title = new Set(titleWords.flat()); page = { title, + titleWords, titleAndPath: new Set([ ...title, ...tokenizeDocsSearchText(chunk.urlPath), @@ -905,7 +970,8 @@ function pageMatchMultiplier( 1 + TITLE_MATCH_BOOST * queryCoverage(expansions, page.title) * - (1 + TITLE_PRECISION_WEIGHT * fieldPrecision(expansions, page.title)) + + (1 + + TITLE_PRECISION_WEIGHT * fieldPrecision(expansions, page.titleWords)) + PATH_MATCH_BOOST * queryCoverage(expansions, page.titleAndPath) + HEADING_MATCH_BOOST * queryCoverage(expansions, heading) ); @@ -923,7 +989,7 @@ const MAX_PAGE_MATCH_MULTIPLIER = TITLE_MATCH_BOOST * (1 + TITLE_PRECISION_WEIGHT) + PATH_MATCH_BOOST + HEADING_MATCH_BOOST; -const CONTENT_BOOST_BATCH_SIZE = 32; +const FIRST_CONTENT_BOOST_CHECKPOINT = 32; function compareScoredChunks(left: ScoredChunk, right: ScoredChunk): number { if (right.score !== left.score) { @@ -958,29 +1024,32 @@ function selectTopChunks( * chunk text, the costliest step of a query, so chunks are boosted in score * order and the rest are skipped once even the largest boost could not lift * them into the results. Adding chunks never lowers the score of the last - * selected result, so a chunk below that bound cannot be selected. + * selected result, so a chunk below that bound cannot be selected. The bound + * is refreshed at doubling checkpoints, so refreshing costs O(n log n) even + * when it never prunes. */ function applyContentMatchBoosts( scoredChunks: ScoredChunk[], - queryTokens: string[], + query: QueryPhrase, limit: number, maxResultsPerPage: number ): void { - if (queryTokens.length < 2) { + if (query.tokens.length < 2) { return; } scoredChunks.sort(compareScoredChunks); const maxBoost = PHRASE_MATCH_BOOST * MAX_PAGE_MATCH_MULTIPLIER; let lastSelectedScore = Number.NEGATIVE_INFINITY; + let checkpoint = FIRST_CONTENT_BOOST_CHECKPOINT; for (const [position, scored] of scoredChunks.entries()) { if (scored.score + maxBoost < lastSelectedScore) { return; } scored.score = - (scored.termScore + - scoreContentMatchBoost(scored.chunk.text, queryTokens)) * + (scored.termScore + scoreContentMatchBoost(scored.chunk.text, query)) * scored.multiplier; - if ((position + 1) % CONTENT_BOOST_BATCH_SIZE === 0) { + if (position + 1 === checkpoint) { + checkpoint *= 2; const boosted = scoredChunks .slice(0, position + 1) .sort(compareScoredChunks); @@ -1419,7 +1488,15 @@ export function searchDocs( score: score * multiplier, }); } - applyContentMatchBoosts(scoredChunks, queryTokens, limit, maxResultsPerPage); + applyContentMatchBoosts( + scoredChunks, + { + tokens: queryTokens, + writtenWords: tokenizeWrittenWords(query).join(" "), + }, + limit, + maxResultsPerPage + ); scoredChunks.sort(compareScoredChunks); // Excerpts are expensive string scans, so build them only for results From d27d100d7eead7999fc40333029866cb538c17a3 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:36:30 -0700 Subject: [PATCH 3/6] perf(search): tokenize chunk text once for phrase and proximity matching Reuse the phrase tokens for the proximity check, and test that a phrase match lifted only by the content boost survives pruning. --- packages/leadtype/src/search/search.test.ts | 38 +++++++++++++++++++++ packages/leadtype/src/search/search.ts | 9 ++--- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 65f5efd28..d63d547df 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -431,6 +431,44 @@ describe("createDocsSearchIndex and searchDocs", () => { expect(searchDocs(index, "next js app")[0]?.documentId).toBe("phrase"); }); + it("keeps a phrase match that only the phrase boost lifts into the results", () => { + // 60 pages that mention both words apart rank above the phrase match on + // term scores alone, enough for the content boost to set its bound. + const scattered = Array.from( + { length: 60 }, + (_, page): DocsSearchDocument => ({ + id: `guide-${page}`, + title: `Guide ${page}`, + urlPath: `/docs/guide-${page}`, + absoluteUrl: `https://leadtype.dev/docs/guide-${page}`, + relativePath: `guide-${page}`, + content: `## Setup\n\nThe banner ${page} opens first, then a visitor reads the consent text.`, + }) + ); + const index = createDocsSearchIndex( + [ + ...scattered, + { + id: "phrase", + title: "Reference", + urlPath: "/docs/reference", + absoluteUrl: "https://leadtype.dev/docs/reference", + relativePath: "reference", + content: + "## Setup\n\nShow the consent banner once and the dialog lists every choice for each visitor who returns.", + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const top = (limit: number) => + searchDocs(index, "consent banner", { limit }).map( + (result) => result.documentId + ); + + expect(top(5)[0]).toBe("phrase"); + expect(top(5)).toEqual(top(1000).slice(0, 5)); + }); + it("matches words that only appear in the page URL", () => { const quickstart = (framework: string): DocsSearchDocument => ({ id: framework, diff --git a/packages/leadtype/src/search/search.ts b/packages/leadtype/src/search/search.ts index 538014a12..7b9ece59b 100644 --- a/packages/leadtype/src/search/search.ts +++ b/packages/leadtype/src/search/search.ts @@ -858,10 +858,6 @@ function buildExcerpt(text: string, queryTokens: string[]): string { return `${prefix}${text.slice(start, end).trim()}${suffix}`; } -function normalizeForPhraseMatch(input: string): string { - return tokenizeDocsSearchText(input).join(" "); -} - function hasOrderedProximityMatch( textTokens: string[], queryTokens: string[] @@ -916,16 +912,15 @@ function scoreContentMatchBoost(text: string, query: QueryPhrase): number { } const normalizedQuery = queryTokens.join(" "); - const normalizedText = normalizeForPhraseMatch(text); + const textTokens = tokenizeDocsSearchText(text); if ( - normalizedText.includes(normalizedQuery) || + textTokens.join(" ").includes(normalizedQuery) || (query.writtenWords.includes(" ") && tokenizeWrittenWords(text).join(" ").includes(query.writtenWords)) ) { return PHRASE_MATCH_BOOST; } - const textTokens = tokenizeDocsSearchText(text); return hasOrderedProximityMatch(textTokens, queryTokens) ? PROXIMITY_MATCH_BOOST : 0; From df2c484d7c8207216dbfd420a3aa67cdf893fff8 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:41:22 -0700 Subject: [PATCH 4/6] fix(search): reject older search index versions in every loader The browser search client and MCP artifact loading only checked that the index and content matched each other, so a cached v3 pair loaded and was searched with the v4 tokenizer, quietly missing matches. Both now reject an index whose version differs from DOCS_SEARCH_INDEX_VERSION, with the same regenerate message NLWeb uses. --- .changeset/search-ranking-page-match.md | 2 +- packages/leadtype/src/mcp/artifacts.ts | 12 +++++++++- packages/leadtype/src/mcp/mcp.test.ts | 29 +++++++++++++++++++++++ packages/leadtype/src/next/client.test.ts | 24 +++++++++++++++++++ packages/leadtype/src/search/client.ts | 5 ++++ packages/leadtype/src/search/search.ts | 17 +++++++++++++ 6 files changed, 87 insertions(+), 2 deletions(-) diff --git a/.changeset/search-ranking-page-match.md b/.changeset/search-ranking-page-match.md index 1aa1a8ba0..768a37ea9 100644 --- a/.changeset/search-ranking-page-match.md +++ b/.changeset/search-ranking-page-match.md @@ -2,7 +2,7 @@ "leadtype": minor --- -Rank docs search results by the page they come from, not only by term counts. Regenerate search artifacts with this release: the index format is now version 4, and version 3 artifacts fail validation. +Rank docs search results by the page they come from, not only by term counts. Regenerate search artifacts with this release: the index format is now version 4. The browser search client, MCP artifact loading and NLWeb `/ask` reject a version 3 index with an error that says to regenerate, instead of searching it with the new tokenizer and quietly missing matches. - A chunk's score is multiplied by how much of the query its page title, URL path and own heading cover. A page titled "Consent categories" now outranks a release note that repeats both words, and `vue quickstart` finds `/docs/frameworks/vue/quickstart` when the title is only "Quickstart". - Camel-case names index their parts and short dotted or hyphenated names index their joined form, so `consent banner` matches a page titled `ConsentBanner` and `nextjs` matches "Next.js". diff --git a/packages/leadtype/src/mcp/artifacts.ts b/packages/leadtype/src/mcp/artifacts.ts index 054c122c6..d2b195cb2 100644 --- a/packages/leadtype/src/mcp/artifacts.ts +++ b/packages/leadtype/src/mcp/artifacts.ts @@ -15,7 +15,10 @@ import type { DocsSearchContentStore, DocsSearchIndex, } from "../search/index.js"; -import { isDocsSearchContentStoreCompatible } from "../search/search.js"; +import { + docsSearchIndexVersionError, + isDocsSearchContentStoreCompatible, +} from "../search/search.js"; /** * Subdirectory (under the artifacts base) that `generate` writes docs artifacts @@ -89,6 +92,13 @@ export async function loadDocsArtifacts( if (!index) { throw missingArtifactError(baseDir, SEARCH_INDEX_FILE); } + const versionError = docsSearchIndexVersionError( + index, + path.join(docsDir, SEARCH_INDEX_FILE) + ); + if (versionError) { + throw versionError; + } const rawManifest = await readOptionalJson( path.join(docsDir, MANIFEST_FILE) diff --git a/packages/leadtype/src/mcp/mcp.test.ts b/packages/leadtype/src/mcp/mcp.test.ts index cefae2d59..ffcdde3e5 100644 --- a/packages/leadtype/src/mcp/mcp.test.ts +++ b/packages/leadtype/src/mcp/mcp.test.ts @@ -307,6 +307,35 @@ describe("loadDocsArtifacts (from disk)", () => { expect(textOf(result)).toBe(QUICKSTART_MARKDOWN); }); + it("rejects a search index from an older leadtype", async () => { + const staleDir = await mkdtemp(join(tmpdir(), "leadtype-mcp-stale-")); + try { + const docsDir = join(staleDir, "docs"); + await mkdir(docsDir, { recursive: true }); + const index = createDocsSearchIndex(docs); + await Promise.all([ + writeFile( + join(docsDir, "search-index.json"), + JSON.stringify({ + ...index, + version: 3, + content: index.content && { ...index.content, version: 3 }, + }) + ), + writeFile( + join(docsDir, "agent-readability.json"), + JSON.stringify(buildManifest()) + ), + ]); + + await expect(loadDocsArtifacts({ artifacts: staleDir })).rejects.toThrow( + "uses an unsupported version (3)" + ); + } finally { + await rm(staleDir, { recursive: true, force: true }); + } + }); + it("ignores legacy split content and keeps index-only search available", async () => { const legacyDir = await mkdtemp(join(tmpdir(), "leadtype-mcp-legacy-")); try { diff --git a/packages/leadtype/src/next/client.test.ts b/packages/leadtype/src/next/client.test.ts index 9b0a2ed01..3de4d4be5 100644 --- a/packages/leadtype/src/next/client.test.ts +++ b/packages/leadtype/src/next/client.test.ts @@ -105,6 +105,30 @@ describe("createSearchClient", () => { expect(results[0]?.title).toBe("Install"); }); + it("rejects a search index from an older leadtype", async () => { + const bundle = buildSearchBundle(); + const staleIndex = { ...bundle.index, version: 3 }; + const staleContent = { ...bundle.content, version: 3 }; + const scope = nextScope(); + const fetchImpl = ((input: RequestInfo | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url.endsWith(scope.indexUrl)) { + return Promise.resolve(Response.json(staleIndex)); + } + return Promise.resolve(Response.json(staleContent)); + }) as typeof fetch; + + const client = createSearchClient("docs", { + indexUrl: scope.indexUrl, + contentUrl: scope.contentUrl, + fetch: fetchImpl, + }); + + await expect(client.search("install")).rejects.toThrow( + "uses an unsupported version (3)" + ); + }); + it("treats a missing content file as a soft failure", async () => { const bundle = buildSearchBundle(); const scope = nextScope(); diff --git a/packages/leadtype/src/search/client.ts b/packages/leadtype/src/search/client.ts index 88f32953a..73127cbf7 100644 --- a/packages/leadtype/src/search/client.ts +++ b/packages/leadtype/src/search/client.ts @@ -5,6 +5,7 @@ import type { DocsSearchResult, } from "./search"; import { + docsSearchIndexVersionError, isDocsSearchContentStoreCompatible, listDocsSearchDocuments, searchDocs, @@ -100,6 +101,10 @@ async function loadArtifacts( () => undefined ), ]); + const versionError = docsSearchIndexVersionError(index, indexUrl); + if (versionError) { + throw versionError; + } const compatibleContent = isDocsSearchContentStoreCompatible( index, content diff --git a/packages/leadtype/src/search/search.ts b/packages/leadtype/src/search/search.ts index 7b9ece59b..52548b1ee 100644 --- a/packages/leadtype/src/search/search.ts +++ b/packages/leadtype/src/search/search.ts @@ -1060,6 +1060,23 @@ function requestError(message: string, status: number): never { throw new DocsSearchRequestError(message, status); } +/** + * Error for an index this leadtype cannot search, or `undefined` when it can. + * A version bump means the tokenizer changed, so an older index would load + * and quietly miss matches the current rules expect. + */ +export function docsSearchIndexVersionError( + index: { version?: unknown }, + source: string +): Error | undefined { + if (index.version === DOCS_SEARCH_INDEX_VERSION) { + return; + } + return new Error( + `leadtype: ${source} uses an unsupported version (${String(index.version)}); this leadtype reads search index version ${DOCS_SEARCH_INDEX_VERSION}. Run \`leadtype generate\` with this leadtype version and deploy the complete docs output.` + ); +} + export function isDocsSearchContentStoreCompatible( index: DocsSearchIndex, content: unknown From b2c035854a3fd730a115cf5faac10ba402c57b09 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:52:10 -0700 Subject: [PATCH 5/6] fix(search): index URL path words a chunk does not already contain The page-match boost read the URL path, but only chunks with a posting could be scored, so a page whose only match was its URL never appeared: `vue` missed a page titled Quickstart at /docs/frameworks/vue/quickstart. Path words now count once as body text when the chunk lacks them, using the words as written so hyphenated segments add no joined prefix terms. countDocsSearchChunkTerms keeps the indexer and NLWeb validation on one rule. --- .changeset/search-ranking-page-match.md | 2 +- docs/reference/search.mdx | 2 +- packages/leadtype/src/nlweb/ask.ts | 17 ++++-- packages/leadtype/src/search/search.test.ts | 29 +++++++++ packages/leadtype/src/search/search.ts | 68 ++++++++++++++++----- 5 files changed, 97 insertions(+), 21 deletions(-) diff --git a/.changeset/search-ranking-page-match.md b/.changeset/search-ranking-page-match.md index 768a37ea9..452cee51d 100644 --- a/.changeset/search-ranking-page-match.md +++ b/.changeset/search-ranking-page-match.md @@ -4,7 +4,7 @@ Rank docs search results by the page they come from, not only by term counts. Regenerate search artifacts with this release: the index format is now version 4. The browser search client, MCP artifact loading and NLWeb `/ask` reject a version 3 index with an error that says to regenerate, instead of searching it with the new tokenizer and quietly missing matches. -- A chunk's score is multiplied by how much of the query its page title, URL path and own heading cover. A page titled "Consent categories" now outranks a release note that repeats both words, and `vue quickstart` finds `/docs/frameworks/vue/quickstart` when the title is only "Quickstart". +- A chunk's score is multiplied by how much of the query its page title, URL path and own heading cover. A page titled "Consent categories" now outranks a release note that repeats both words, and `vue quickstart` finds `/docs/frameworks/vue/quickstart` when the title is only "Quickstart". URL path words a page doesn't already use are indexed once, so a page that matches only through its URL is still found. - Camel-case names index their parts and short dotted or hyphenated names index their joined form, so `consent banner` matches a page titled `ConsentBanner` and `nextjs` matches "Next.js". - A page's description is indexed with its first chunk only. Repeating it in every chunk let pages with long descriptions outrank the sections that answer the query. The index is also smaller. - `searchDocs` returns at most two chunks per page before applying `limit`. Set `maxResultsPerPage` to change this. NLWeb `/ask` uses `maxResultsPerPage: 1`, so it now fills its result limit instead of returning fewer results after removing duplicate pages. diff --git a/docs/reference/search.mdx b/docs/reference/search.mdx index b516f0c96..7c4c0bbea 100644 --- a/docs/reference/search.mdx +++ b/docs/reference/search.mdx @@ -20,7 +20,7 @@ and [Agent search tools](/docs/search/agent-tools). - **Index** (`search-index.json`) — compact term postings + document/chunk refs. Loaded on every search. - **Content store** (`search-content.json`) — chunk text, separate from the index so search stays cheap and content loads lazily. - **BM25** — the ranking function. Weights: title 4×, headings 2×, body 1×, code 0.35×. -- **Page match** — a multiplier on each chunk's BM25 score for how much of the query its page title, URL path and own heading cover. A short title that matches the query beats a long title that only contains it. The path brings in context titles often leave out, such as the framework in `/docs/frameworks/react/quickstart`. +- **Page match** — a multiplier on each chunk's BM25 score for how much of the query its page title, URL path and own heading cover. A short title that matches the query beats a long title that only contains it. The path brings in context titles often leave out, such as the framework in `/docs/frameworks/react/quickstart`. URL path words a chunk doesn't already contain are indexed once as body text, so a page that matches only through its URL is still found. - **Tokens** — words split on letters and digits, lowercased, accents and stopwords removed. Camel-case names also index their parts (`ConsentBanner` → `consentbanner`, `consent`, `banner`), and short dotted or hyphenated names also index their joined form (`Next.js` → `next`, `js`, `nextjs`). ## Build time — `leadtype/search/node` diff --git a/packages/leadtype/src/nlweb/ask.ts b/packages/leadtype/src/nlweb/ask.ts index 82121b7ee..275e7b5a4 100644 --- a/packages/leadtype/src/nlweb/ask.ts +++ b/packages/leadtype/src/nlweb/ask.ts @@ -9,7 +9,7 @@ import { searchDocs, } from "../search/index.js"; import { - countDocsSearchTerms, + countDocsSearchChunkTerms, DOCS_SEARCH_INDEX_VERSION, tokenizeDocsSearchText, } from "../search/search.js"; @@ -593,13 +593,20 @@ function assertNlwebArtifacts(artifacts: DocsArtifacts): void { ) { return false; } + const counts = countDocsSearchChunkTerms({ + title: document[1], + headingPath: chunk[3], + text: contentText, + codeText, + urlPath: document[3], + }); addExpectedVisiblePostings( expectedVisiblePostings, chunkIndex, - countDocsSearchTerms(document[1]), - countDocsSearchTerms(chunk[3].join(" ")), - countDocsSearchTerms(contentText), - countDocsSearchTerms(codeText) + counts.title, + counts.heading, + counts.body, + counts.code ); chunkIds.add(chunkId); totalChunkLength += chunk[4]; diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index d63d547df..285b04b0e 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -486,6 +486,35 @@ describe("createDocsSearchIndex and searchDocs", () => { expect(searchDocs(index, "vue quickstart")[0]?.documentId).toBe("vue"); }); + it("finds a page whose only match is its URL", () => { + const index = createDocsSearchIndex( + [ + { + id: "vue-quickstart", + title: "Quickstart", + urlPath: "/docs/frameworks/vue/quickstart", + absoluteUrl: "https://leadtype.dev/docs/frameworks/vue/quickstart", + relativePath: "frameworks/vue/quickstart", + content: "## Install the package\n\nAdd the plugin to your app.", + }, + { + id: "comparison", + title: "Framework support", + urlPath: "/docs/frameworks", + absoluteUrl: "https://leadtype.dev/docs/frameworks", + relativePath: "frameworks", + content: "## Supported\n\nThe library works with Vue and React.", + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect( + searchDocs(index, "vue").map((result) => result.documentId) + ).toContain("vue-quickstart"); + }); + it("caps how many sections one page contributes before the limit", () => { const sections = Array.from( { length: 6 }, diff --git a/packages/leadtype/src/search/search.ts b/packages/leadtype/src/search/search.ts index 52548b1ee..8622ccae2 100644 --- a/packages/leadtype/src/search/search.ts +++ b/packages/leadtype/src/search/search.ts @@ -317,13 +317,6 @@ export class DocsSearchRequestError extends Error { } } -type MutableTermCounts = { - title: Map; - heading: Map; - body: Map; - code: Map; -}; - type MutableChunk = { id: string; documentIndex: number; @@ -648,6 +641,49 @@ export function countDocsSearchTerms(input: string): Map { return counts; } +export type DocsSearchChunkTermCounts = { + title: Map; + heading: Map; + body: Map; + code: Map; +}; + +/** + * Term counts a chunk's postings store. A URL path word the chunk does not + * already contain counts once as body text, so a page found only through its + * path, such as `vue` in `/docs/frameworks/vue/quickstart`, still becomes a + * candidate, without adding weight to words the page already uses. + */ +export function countDocsSearchChunkTerms(chunk: { + title: string; + headingPath: string[]; + text: string; + codeText: string; + urlPath: string; +}): DocsSearchChunkTermCounts { + const counts = { + title: countDocsSearchTerms(chunk.title), + heading: countDocsSearchTerms(chunk.headingPath.join(" ")), + body: countDocsSearchTerms(chunk.text), + code: countDocsSearchTerms(chunk.codeText), + }; + // Written words only: a joined `typescriptsetup` from `typescript-setup` + // would become a prefix match for every `typescript` query. + for (const token of tokenizeWrittenWords(chunk.urlPath)) { + if ( + !( + counts.title.has(token) || + counts.heading.has(token) || + counts.body.has(token) || + counts.code.has(token) + ) + ) { + counts.body.set(token, 1); + } + } + return counts; +} + function hasUnsupportedControlCharacter(input: string): boolean { for (const character of input) { const codePoint = character.codePointAt(0); @@ -1247,7 +1283,7 @@ export function createDocsSearchIndex( ); const documents: DocsSearchIndex["documents"] = []; const mutableChunks: MutableChunk[] = []; - const chunkTermCounts = new Map(); + const chunkTermCounts = new Map(); const docs = runTransformersSync( options.transformers, @@ -1363,12 +1399,16 @@ export function createDocsSearchIndex( codeText: transformedChunk.codeText, length, }); - chunkTermCounts.set(chunkIndex, { - title: countDocsSearchTerms(doc.title), - heading: countDocsSearchTerms(transformedChunk.headingPath.join(" ")), - body: countDocsSearchTerms(transformedChunk.text), - code: countDocsSearchTerms(transformedChunk.codeText), - }); + chunkTermCounts.set( + chunkIndex, + countDocsSearchChunkTerms({ + title: doc.title, + headingPath: transformedChunk.headingPath, + text: transformedChunk.text, + codeText: transformedChunk.codeText, + urlPath: doc.urlPath, + }) + ); } } } From 14f4e3afc29adf23e36458d55f7df7d67133909f Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:54:28 -0700 Subject: [PATCH 6/6] fix(search): center excerpts on matches found through joined compounds A query such as nextjs matched Next.js through its joined token, but the excerpt looked for nextjs in text that still had the dot and fell back to the first 220 characters. Excerpts now also search the text with compound separators removed, mapped back to the original offsets. --- packages/leadtype/src/search/search.test.ts | 23 ++++++++++ packages/leadtype/src/search/search.ts | 49 +++++++++++++++++---- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 285b04b0e..efc024382 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -486,6 +486,29 @@ describe("createDocsSearchIndex and searchDocs", () => { expect(searchDocs(index, "vue quickstart")[0]?.documentId).toBe("vue"); }); + it("centers the excerpt on a dotted name matched through its joined form", () => { + const intro = + "The consent runtime keeps a record of every choice a visitor makes. ".repeat( + 5 + ); + const index = createDocsSearchIndex( + [ + { + id: "frameworks", + title: "Frameworks", + urlPath: "/docs/frameworks", + absoluteUrl: "https://leadtype.dev/docs/frameworks", + relativePath: "frameworks", + content: `## Support\n\n${intro}Next.js renders the banner on the server.`, + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "nextjs")[0]?.excerpt).toContain("Next.js"); + }); + it("finds a page whose only match is its URL", () => { const index = createDocsSearchIndex( [ diff --git a/packages/leadtype/src/search/search.ts b/packages/leadtype/src/search/search.ts index 8622ccae2..238ce2ed5 100644 --- a/packages/leadtype/src/search/search.ts +++ b/packages/leadtype/src/search/search.ts @@ -58,6 +58,7 @@ const CAMEL_CASE_BOUNDARY_PATTERN = // the plain words that make up most text. const HAS_CAMEL_CASE_BOUNDARY_PATTERN = /\p{Ll}\p{Lu}|\p{Lu}\p{Lu}\p{Ll}/u; const DIGITS_ONLY_PATTERN = /^\p{N}+$/u; +const WORD_CHARACTER_PATTERN = /^[\p{L}\p{N}]$/u; const MAX_JOINED_COMPOUND_PARTS = 3; const DIACRITIC_PATTERN = /[\u0300-\u036f]/g; @@ -871,22 +872,54 @@ function normalizeTextWithOffsets(input: string): { return { normalized, offsets }; } -function buildExcerpt(text: string, queryTokens: string[]): string { - const { normalized: normalizedText, offsets } = - normalizeTextWithOffsets(text); - let matchIndex = -1; +/** + * The normalized text without `.`, `-` or `_` between two word characters, + * so a joined token such as `nextjs` can be found in `next.js`. Offsets keep + * pointing into the original text. + */ +function joinCompoundsWithOffsets( + normalized: string, + offsets: number[] +): { joined: string; offsets: number[] } { + let joined = ""; + const joinedOffsets: number[] = []; + for (let index = 0; index < normalized.length; index += 1) { + const character = normalized.charAt(index); + const isJoin = + COMPOUND_SEPARATOR_PATTERN.test(character) && + WORD_CHARACTER_PATTERN.test(normalized.charAt(index - 1)) && + WORD_CHARACTER_PATTERN.test(normalized.charAt(index + 1)); + if (!isJoin) { + joined += character; + joinedOffsets.push(offsets[index] ?? index); + } + } + return { joined, offsets: joinedOffsets }; +} + +function findOriginalMatchIndex(text: string, queryTokens: string[]): number { + const { normalized, offsets } = normalizeTextWithOffsets(text); + let compounds: { joined: string; offsets: number[] } | undefined; for (const token of queryTokens) { - matchIndex = normalizedText.indexOf(token); + const matchIndex = normalized.indexOf(token); if (matchIndex >= 0) { - break; + return offsets[matchIndex] ?? matchIndex; + } + compounds ??= joinCompoundsWithOffsets(normalized, offsets); + const joinedIndex = compounds.joined.indexOf(token); + if (joinedIndex >= 0) { + return compounds.offsets[joinedIndex] ?? joinedIndex; } } + return -1; +} - if (matchIndex < 0) { +function buildExcerpt(text: string, queryTokens: string[]): string { + const originalMatchIndex = findOriginalMatchIndex(text, queryTokens); + if (originalMatchIndex < 0) { return text.slice(0, 220).trim(); } - const originalMatchIndex = offsets[matchIndex] ?? matchIndex; const start = Math.max(0, originalMatchIndex - 80); const end = Math.min(text.length, originalMatchIndex + 160); const prefix = start > 0 ? "..." : "";