diff --git a/.changeset/search-ranking-page-match.md b/.changeset/search-ranking-page-match.md new file mode 100644 index 00000000..452cee51 --- /dev/null +++ b/.changeset/search-ranking-page-match.md @@ -0,0 +1,11 @@ +--- +"leadtype": minor +--- + +Rank docs search results by the page they come from, not only by term counts. Regenerate search artifacts with this release: the index format is now version 4. The browser search client, MCP artifact loading and NLWeb `/ask` reject a version 3 index with an error that says to regenerate, instead of searching it with the new tokenizer and quietly missing matches. + +- A chunk's score is multiplied by how much of the query its page title, URL path and own heading cover. A page titled "Consent categories" now outranks a release note that repeats both words, and `vue quickstart` finds `/docs/frameworks/vue/quickstart` when the title is only "Quickstart". URL path words a page doesn't already use are indexed once, so a page that matches only through its URL is still found. +- Camel-case names index their parts and short dotted or hyphenated names index their joined form, so `consent banner` matches a page titled `ConsentBanner` and `nextjs` matches "Next.js". +- A page's description is indexed with its first chunk only. Repeating it in every chunk let pages with long descriptions outrank the sections that answer the query. The index is also smaller. +- `searchDocs` returns at most two chunks per page before applying `limit`. Set `maxResultsPerPage` to change this. NLWeb `/ask` uses `maxResultsPerPage: 1`, so it now fills its result limit instead of returning fewer results after removing duplicate pages. +- Queries with two or more words skip phrase matching for chunks that cannot reach the results, which makes broad queries several times faster on large indexes. diff --git a/docs/reference/search.mdx b/docs/reference/search.mdx index 7262030a..7c4c0bbe 100644 --- a/docs/reference/search.mdx +++ b/docs/reference/search.mdx @@ -20,6 +20,8 @@ and [Agent search tools](/docs/search/agent-tools). - **Index** (`search-index.json`) — compact term postings + document/chunk refs. Loaded on every search. - **Content store** (`search-content.json`) — chunk text, separate from the index so search stays cheap and content loads lazily. - **BM25** — the ranking function. Weights: title 4×, headings 2×, body 1×, code 0.35×. +- **Page match** — a multiplier on each chunk's BM25 score for how much of the query its page title, URL path and own heading cover. A short title that matches the query beats a long title that only contains it. The path brings in context titles often leave out, such as the framework in `/docs/frameworks/react/quickstart`. URL path words a chunk doesn't already contain are indexed once as body text, so a page that matches only through its URL is still found. +- **Tokens** — words split on letters and digits, lowercased, accents and stopwords removed. Camel-case names also index their parts (`ConsentBanner` → `consentbanner`, `consent`, `banner`), and short dotted or hyphenated names also index their joined form (`Next.js` → `next`, `js`, `nextjs`). ## Build time — `leadtype/search/node` @@ -46,9 +48,14 @@ searchDocs(index: DocsSearchIndex, query: string, options?: { content?: DocsSearchContentStore; // enables excerpts synonyms?: Record; limit?: number; // default 8 + maxResultsPerPage?: number; // default 2, applied before limit }): DocsSearchResult[] ``` +`maxResultsPerPage` keeps one long page from filling every result. Set it to `1` +for a list of pages. Filter or merge results after the call only with a larger +`limit`, or the list can come back shorter than you asked for. + `DocsSearchResult` carries `title`, `urlPath`, `urlWithHash`, `headingPath`, `score`, and (with `content`) `excerpt`. The runtime is edge-safe — no Node APIs. @@ -134,6 +141,7 @@ Objects) in production. Defaults live in `docsSearchDefaults`: | `maxContextChars` | 12000 | | `maxSources` | 6 | | `searchLimit` | 8 | +| `maxResultsPerPage` | 2 | | `maxBodyBytes` | 16 KB | ## When to add embeddings diff --git a/docs/search/add-search.mdx b/docs/search/add-search.mdx index d0dbecef..fa95e7c3 100644 --- a/docs/search/add-search.mdx +++ b/docs/search/add-search.mdx @@ -49,7 +49,7 @@ For server routes or any code that already has the JSON, query directly with import type { DocsSearchContentStore } from "leadtype/search"; const contentJson: DocsSearchContentStore = { - version: 3, + version: 4, generatedAt: "2026-06-15T10:30:00.000Z", chunks: [], codeChunks: [], @@ -61,7 +61,7 @@ export default contentJson; import type { DocsSearchIndex } from "leadtype/search"; const indexJson: DocsSearchIndex = { - version: 3, + version: 4, generatedAt: "2026-06-15T10:30:00.000Z", documents: [], chunks: [], diff --git a/docs/search/ai-answers.mdx b/docs/search/ai-answers.mdx index dbb61a89..5d2a0775 100644 --- a/docs/search/ai-answers.mdx +++ b/docs/search/ai-answers.mdx @@ -134,14 +134,14 @@ import type { } from "leadtype/search"; export const docsSearchContent: DocsSearchContentStore = { - version: 3, + version: 4, generatedAt: "2026-06-15T10:30:00.000Z", chunks: [], codeChunks: [], }; export const docsSearchIndex: DocsSearchIndex = { - version: 3, + version: 4, generatedAt: "2026-06-15T10:30:00.000Z", documents: [], chunks: [], diff --git a/packages/leadtype/src/mcp/artifacts.ts b/packages/leadtype/src/mcp/artifacts.ts index 054c122c..d2b195cb 100644 --- a/packages/leadtype/src/mcp/artifacts.ts +++ b/packages/leadtype/src/mcp/artifacts.ts @@ -15,7 +15,10 @@ import type { DocsSearchContentStore, DocsSearchIndex, } from "../search/index.js"; -import { isDocsSearchContentStoreCompatible } from "../search/search.js"; +import { + docsSearchIndexVersionError, + isDocsSearchContentStoreCompatible, +} from "../search/search.js"; /** * Subdirectory (under the artifacts base) that `generate` writes docs artifacts @@ -89,6 +92,13 @@ export async function loadDocsArtifacts( if (!index) { throw missingArtifactError(baseDir, SEARCH_INDEX_FILE); } + const versionError = docsSearchIndexVersionError( + index, + path.join(docsDir, SEARCH_INDEX_FILE) + ); + if (versionError) { + throw versionError; + } const rawManifest = await readOptionalJson( path.join(docsDir, MANIFEST_FILE) diff --git a/packages/leadtype/src/mcp/mcp.test.ts b/packages/leadtype/src/mcp/mcp.test.ts index cefae2d5..ffcdde3e 100644 --- a/packages/leadtype/src/mcp/mcp.test.ts +++ b/packages/leadtype/src/mcp/mcp.test.ts @@ -307,6 +307,35 @@ describe("loadDocsArtifacts (from disk)", () => { expect(textOf(result)).toBe(QUICKSTART_MARKDOWN); }); + it("rejects a search index from an older leadtype", async () => { + const staleDir = await mkdtemp(join(tmpdir(), "leadtype-mcp-stale-")); + try { + const docsDir = join(staleDir, "docs"); + await mkdir(docsDir, { recursive: true }); + const index = createDocsSearchIndex(docs); + await Promise.all([ + writeFile( + join(docsDir, "search-index.json"), + JSON.stringify({ + ...index, + version: 3, + content: index.content && { ...index.content, version: 3 }, + }) + ), + writeFile( + join(docsDir, "agent-readability.json"), + JSON.stringify(buildManifest()) + ), + ]); + + await expect(loadDocsArtifacts({ artifacts: staleDir })).rejects.toThrow( + "uses an unsupported version (3)" + ); + } finally { + await rm(staleDir, { recursive: true, force: true }); + } + }); + it("ignores legacy split content and keeps index-only search available", async () => { const legacyDir = await mkdtemp(join(tmpdir(), "leadtype-mcp-legacy-")); try { diff --git a/packages/leadtype/src/next/client.test.ts b/packages/leadtype/src/next/client.test.ts index 9b0a2ed0..3de4d4be 100644 --- a/packages/leadtype/src/next/client.test.ts +++ b/packages/leadtype/src/next/client.test.ts @@ -105,6 +105,30 @@ describe("createSearchClient", () => { expect(results[0]?.title).toBe("Install"); }); + it("rejects a search index from an older leadtype", async () => { + const bundle = buildSearchBundle(); + const staleIndex = { ...bundle.index, version: 3 }; + const staleContent = { ...bundle.content, version: 3 }; + const scope = nextScope(); + const fetchImpl = ((input: RequestInfo | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url.endsWith(scope.indexUrl)) { + return Promise.resolve(Response.json(staleIndex)); + } + return Promise.resolve(Response.json(staleContent)); + }) as typeof fetch; + + const client = createSearchClient("docs", { + indexUrl: scope.indexUrl, + contentUrl: scope.contentUrl, + fetch: fetchImpl, + }); + + await expect(client.search("install")).rejects.toThrow( + "uses an unsupported version (3)" + ); + }); + it("treats a missing content file as a soft failure", async () => { const bundle = buildSearchBundle(); const scope = nextScope(); diff --git a/packages/leadtype/src/nlweb/ask.ts b/packages/leadtype/src/nlweb/ask.ts index a1113b67..275e7b5a 100644 --- a/packages/leadtype/src/nlweb/ask.ts +++ b/packages/leadtype/src/nlweb/ask.ts @@ -9,7 +9,7 @@ import { searchDocs, } from "../search/index.js"; import { - countDocsSearchTerms, + countDocsSearchChunkTerms, DOCS_SEARCH_INDEX_VERSION, tokenizeDocsSearchText, } from "../search/search.js"; @@ -593,13 +593,20 @@ function assertNlwebArtifacts(artifacts: DocsArtifacts): void { ) { return false; } + const counts = countDocsSearchChunkTerms({ + title: document[1], + headingPath: chunk[3], + text: contentText, + codeText, + urlPath: document[3], + }); addExpectedVisiblePostings( expectedVisiblePostings, chunkIndex, - countDocsSearchTerms(document[1]), - countDocsSearchTerms(chunk[3].join(" ")), - countDocsSearchTerms([document[2], contentText].join(" ")), - countDocsSearchTerms(codeText) + counts.title, + counts.heading, + counts.body, + counts.code ); chunkIds.add(chunkId); totalChunkLength += chunk[4]; @@ -688,20 +695,6 @@ function toNlwebResult( }; } -/** Search hits are heading-level; keep the best hit per page. */ -function dedupeByPage(results: DocsSearchResult[]): DocsSearchResult[] { - const seen = new Set(); - const deduped: DocsSearchResult[] = []; - for (const result of results) { - if (seen.has(result.urlPath)) { - continue; - } - seen.add(result.urlPath); - deduped.push(result); - } - return deduped; -} - function sseEvent(event: string, data: unknown): string { return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; } @@ -838,12 +831,11 @@ export function createAskHandler( const site = resolveSite(config, artifacts); const productName = artifacts.manifest.product.name; - const hits = dedupeByPage( - searchDocs(artifacts.index, parsed.query, { - limit, - content: artifacts.content, - }) - ); + const hits = searchDocs(artifacts.index, parsed.query, { + limit, + maxResultsPerPage: 1, + content: artifacts.content, + }); const results = hits.map((hit) => toNlwebResult(hit, site, productName)); failureCode = NLWEB_ERROR_CODES.internalError; diff --git a/packages/leadtype/src/search/client.ts b/packages/leadtype/src/search/client.ts index 88f32953..73127cbf 100644 --- a/packages/leadtype/src/search/client.ts +++ b/packages/leadtype/src/search/client.ts @@ -5,6 +5,7 @@ import type { DocsSearchResult, } from "./search"; import { + docsSearchIndexVersionError, isDocsSearchContentStoreCompatible, listDocsSearchDocuments, searchDocs, @@ -100,6 +101,10 @@ async function loadArtifacts( () => undefined ), ]); + const versionError = docsSearchIndexVersionError(index, indexUrl); + if (versionError) { + throw versionError; + } const compatibleContent = isDocsSearchContentStoreCompatible( index, content diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 42b1cde7..efc02438 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -17,6 +17,7 @@ import { slugifyDocsHeading, validateDocsQuery, } from "./index"; +import { DOCS_SEARCH_INDEX_VERSION, tokenizeDocsSearchText } from "./search"; const CHUNK_ANCHOR_INDEX = 2; @@ -93,13 +94,29 @@ const cafe = "café"; }, ]; +/** Unrelated pages, so query terms have realistic document frequencies */ +const fillerDocs: DocsSearchDocument[] = [ + "Scripts", + "Callbacks", + "Hooks", + "Theming", + "Iframes", +].map((title) => ({ + id: title, + title, + urlPath: `/docs/${title}`, + absoluteUrl: `https://leadtype.dev/docs/${title}`, + relativePath: title, + content: `## About ${title}\n\n${title} control what loads before a visitor chooses.`, +})); + describe("createDocsSearchIndex and searchDocs", () => { it("stores compact metadata separately from answer content", () => { const index = createDocsSearchIndex(docs, { generatedAt: "2026-01-01T00:00:00.000Z", }); - expect(index.version).toBe(3); + expect(index.version).toBe(DOCS_SEARCH_INDEX_VERSION); expect(index.documents[0]).toEqual([ "quickstart", "Quickstart", @@ -110,7 +127,7 @@ describe("createDocsSearchIndex and searchDocs", () => { ]); expect(index.chunks[0]).toHaveLength(6); expect(index.chunks[0]).not.toHaveProperty("text"); - expect(index.content?.version).toBe(3); + expect(index.content?.version).toBe(DOCS_SEARCH_INDEX_VERSION); expect(index.content?.chunks[0]).toContain("Install the package"); expect(index.content?.codeChunks).toHaveLength(index.chunks.length); expect( @@ -309,6 +326,254 @@ describe("createDocsSearchIndex and searchDocs", () => { expect(searchDocs(index, "unreleased")).toEqual([]); }); + it("splits camel-case and dotted names into searchable words", () => { + expect(tokenizeDocsSearchText("ConsentBanner in Next.js")).toEqual([ + "consentbanner", + "consent", + "banner", + "next", + "js", + "nextjs", + ]); + }); + + it("ranks the page titled for the query above a release that repeats it", () => { + const index = createDocsSearchIndex( + [ + { + id: "release", + title: "v1.3.0: Translations package, SSR consent, CLI telemetry", + description: + "Adds the translations package, backend translations and translations for the CLI.", + urlPath: "/changelog/1.3.0", + absoluteUrl: "https://leadtype.dev/changelog/1.3.0", + relativePath: "changelog/1.3.0", + content: [ + "## New translations package\n\nTranslations ship as their own package.", + "## Backend\n\nThe backend serves translations.", + "## CLI\n\nThe CLI writes translations.", + ].join("\n\n"), + }, + { + id: "translations", + title: "Translations", + description: "Change the copy and add languages.", + urlPath: "/docs/customization/translations", + absoluteUrl: "https://leadtype.dev/docs/customization/translations", + relativePath: "customization/translations", + content: + "## Change the copy\n\nOverride any string the banner and dialog show, then add a language your visitors read. Strings fall back to English when a key is missing.", + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "translations")[0]?.documentId).toBe( + "translations" + ); + }); + + it("counts a camel-case title word once when matching the title", () => { + const index = createDocsSearchIndex( + [ + { + id: "typescript", + title: "TypeScript", + urlPath: "/docs/typescript", + absoluteUrl: "https://leadtype.dev/docs/typescript", + relativePath: "typescript", + content: "## Types\n\nThe package ships its own types.", + }, + { + id: "typescript-setup", + title: "Typescript setup", + urlPath: "/docs/typescript-setup", + absoluteUrl: "https://leadtype.dev/docs/typescript-setup", + relativePath: "typescript-setup", + content: + "## Configure\n\nSet strict in tsconfig for typescript projects.", + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "typescript")[0]?.documentId).toBe("typescript"); + }); + + it("matches a phrase written as separate words across a dotted name", () => { + const index = createDocsSearchIndex( + [ + { + id: "phrase", + title: "Routing", + urlPath: "/docs/routing", + absoluteUrl: "https://leadtype.dev/docs/routing", + relativePath: "routing", + content: + "## Routers\n\nThe Next.js app router renders on the server.", + }, + { + id: "scattered", + title: "Routing guide", + urlPath: "/docs/routing-guide", + absoluteUrl: "https://leadtype.dev/docs/routing-guide", + relativePath: "routing-guide", + content: + "## Routers\n\nNext js routes the app. Next js renders the app.", + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "next js app")[0]?.documentId).toBe("phrase"); + }); + + it("keeps a phrase match that only the phrase boost lifts into the results", () => { + // 60 pages that mention both words apart rank above the phrase match on + // term scores alone, enough for the content boost to set its bound. + const scattered = Array.from( + { length: 60 }, + (_, page): DocsSearchDocument => ({ + id: `guide-${page}`, + title: `Guide ${page}`, + urlPath: `/docs/guide-${page}`, + absoluteUrl: `https://leadtype.dev/docs/guide-${page}`, + relativePath: `guide-${page}`, + content: `## Setup\n\nThe banner ${page} opens first, then a visitor reads the consent text.`, + }) + ); + const index = createDocsSearchIndex( + [ + ...scattered, + { + id: "phrase", + title: "Reference", + urlPath: "/docs/reference", + absoluteUrl: "https://leadtype.dev/docs/reference", + relativePath: "reference", + content: + "## Setup\n\nShow the consent banner once and the dialog lists every choice for each visitor who returns.", + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const top = (limit: number) => + searchDocs(index, "consent banner", { limit }).map( + (result) => result.documentId + ); + + expect(top(5)[0]).toBe("phrase"); + expect(top(5)).toEqual(top(1000).slice(0, 5)); + }); + + it("matches words that only appear in the page URL", () => { + const quickstart = (framework: string): DocsSearchDocument => ({ + id: framework, + title: "Quickstart", + urlPath: `/docs/frameworks/${framework}/quickstart`, + absoluteUrl: `https://leadtype.dev/docs/frameworks/${framework}/quickstart`, + relativePath: `frameworks/${framework}/quickstart`, + content: "## Install the package\n\nAdd the provider to your app.", + }); + const index = createDocsSearchIndex( + [quickstart("react"), quickstart("vue")], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "vue quickstart")[0]?.documentId).toBe("vue"); + }); + + it("centers the excerpt on a dotted name matched through its joined form", () => { + const intro = + "The consent runtime keeps a record of every choice a visitor makes. ".repeat( + 5 + ); + const index = createDocsSearchIndex( + [ + { + id: "frameworks", + title: "Frameworks", + urlPath: "/docs/frameworks", + absoluteUrl: "https://leadtype.dev/docs/frameworks", + relativePath: "frameworks", + content: `## Support\n\n${intro}Next.js renders the banner on the server.`, + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "nextjs")[0]?.excerpt).toContain("Next.js"); + }); + + it("finds a page whose only match is its URL", () => { + const index = createDocsSearchIndex( + [ + { + id: "vue-quickstart", + title: "Quickstart", + urlPath: "/docs/frameworks/vue/quickstart", + absoluteUrl: "https://leadtype.dev/docs/frameworks/vue/quickstart", + relativePath: "frameworks/vue/quickstart", + content: "## Install the package\n\nAdd the plugin to your app.", + }, + { + id: "comparison", + title: "Framework support", + urlPath: "/docs/frameworks", + absoluteUrl: "https://leadtype.dev/docs/frameworks", + relativePath: "frameworks", + content: "## Supported\n\nThe library works with Vue and React.", + }, + ...fillerDocs, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect( + searchDocs(index, "vue").map((result) => result.documentId) + ).toContain("vue-quickstart"); + }); + + it("caps how many sections one page contributes before the limit", () => { + const sections = Array.from( + { length: 6 }, + (_, section) => + `## Analytics setup ${section}\n\nConfigure analytics step ${section}.` + ).join("\n\n"); + const index = createDocsSearchIndex( + [ + { + id: "long", + title: "Analytics", + urlPath: "/docs/analytics", + absoluteUrl: "https://leadtype.dev/docs/analytics", + relativePath: "analytics", + content: sections, + }, + { + id: "other", + title: "Tag managers", + urlPath: "/docs/tag-managers", + absoluteUrl: "https://leadtype.dev/docs/tag-managers", + relativePath: "tag-managers", + content: "## Load analytics\n\nA tag manager can load analytics.", + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + const pages = (maxResultsPerPage?: number) => + searchDocs(index, "analytics", { limit: 3, maxResultsPerPage }).map( + (result) => result.documentId + ); + expect(pages()).toEqual(["long", "long", "other"]); + expect(pages(1)).toEqual(["long", "other"]); + }); + it("falls back to prefix and typo-tolerant matches", () => { const index = createDocsSearchIndex(docs, { generatedAt: "2026-01-01T00:00:00.000Z", diff --git a/packages/leadtype/src/search/search.ts b/packages/leadtype/src/search/search.ts index 9cb17323..238ce2ed 100644 --- a/packages/leadtype/src/search/search.ts +++ b/packages/leadtype/src/search/search.ts @@ -14,12 +14,13 @@ import { const DEFAULT_MAX_CHUNK_CHARS = 1200; const DEFAULT_OVERLAP_CHARS = 160; const DEFAULT_SEARCH_LIMIT = 8; +const DEFAULT_MAX_RESULTS_PER_PAGE = 2; const DEFAULT_MAX_QUERY_CHARS = 400; const DEFAULT_ASK_MAX_QUERY_CHARS = 600; const DEFAULT_MAX_BODY_BYTES = 16 * 1024; const DEFAULT_MAX_SOURCES = 6; const DEFAULT_MAX_CONTEXT_CHARS = 12_000; -export const DOCS_SEARCH_INDEX_VERSION = 3; +export const DOCS_SEARCH_INDEX_VERSION = 4; const TITLE_WEIGHT = 4; const HEADING_WEIGHT = 2; const BODY_WEIGHT = 1; @@ -33,6 +34,10 @@ const PREFIX_TERM_WEIGHT = 0.55; const TYPO_TERM_WEIGHT = 0.45; const PHRASE_MATCH_BOOST = 1.4; const PROXIMITY_MATCH_BOOST = 0.8; +const TITLE_MATCH_BOOST = 1; +const TITLE_PRECISION_WEIGHT = 2; +const PATH_MATCH_BOOST = 0.5; +const HEADING_MATCH_BOOST = 0.5; const MAX_PREFIX_EXPANSIONS = 24; const MAX_TYPO_EXPANSIONS = 16; const PROXIMITY_WINDOW = 8; @@ -42,7 +47,19 @@ const WHITESPACE_PATTERN = /\s+/g; // Tokenizer behavior is serialized into chunk lengths and postings. Changes // to word matching, normalization, stopwords, or token filtering must bump // DOCS_SEARCH_INDEX_VERSION so older artifacts fail with a version mismatch. -const WORD_CHARACTER_PATTERN = /[\p{L}\p{N}]+/gu; +// A word plus any `.`, `-` or `_` joined continuation, so `Next.js`, +// `tanstack-start` and `snake_case` reach the tokenizer as one compound. +const COMPOUND_WORD_PATTERN = /[\p{L}\p{N}]+(?:[._-][\p{L}\p{N}]+)*/gu; +const COMPOUND_SEPARATOR_PATTERN = /[._-]/; +// `ConsentBanner` → `Consent|Banner`, `IABConsent` → `IAB|Consent`. +const CAMEL_CASE_BOUNDARY_PATTERN = + /(?<=\p{Ll})(?=\p{Lu})|(?<=\p{Lu})(?=\p{Lu}\p{Ll})/u; +// Same boundaries without lookbehind: a cheap test that skips the split for +// the plain words that make up most text. +const HAS_CAMEL_CASE_BOUNDARY_PATTERN = /\p{Ll}\p{Lu}|\p{Lu}\p{Lu}\p{Ll}/u; +const DIGITS_ONLY_PATTERN = /^\p{N}+$/u; +const WORD_CHARACTER_PATTERN = /^[\p{L}\p{N}]$/u; +const MAX_JOINED_COMPOUND_PARTS = 3; const DIACRITIC_PATTERN = /[\u0300-\u036f]/g; const SHARED_ROUTE_SEGMENTS = new Set(["shared", "_shared"]); @@ -218,6 +235,11 @@ export type CreateDocsSearchIndexOptions = { export type SearchDocsOptions = ContentStoreOptions & { limit?: number; + /** + * Most sections returned from one page, applied before `limit`. Defaults + * to 2 so a long page whose title matches cannot fill every result. + */ + maxResultsPerPage?: number; synonyms?: Record; }; @@ -296,13 +318,6 @@ export class DocsSearchRequestError extends Error { } } -type MutableTermCounts = { - title: Map; - heading: Map; - body: Map; - code: Map; -}; - type MutableChunk = { id: string; documentIndex: number; @@ -340,17 +355,92 @@ function withHash(url: string, anchor: string): string { return anchor ? `${url}#${anchor}` : url; } +function pushSearchToken(tokens: string[], raw: string): void { + const token = raw.toLowerCase(); + if (token.length > 1 && !STOPWORDS.has(token)) { + tokens.push(token); + } +} + +/** + * Tokens for one word, its camel-case parts, and the joined form of short + * dotted or hyphenated compounds. `ConsentBanner` yields `consentbanner`, + * `consent`, `banner`; `Next.js` yields `next`, `js`, `nextjs`. Without + * `derived`, only the words as written: `consentbanner`; `next`, `js`. + */ +function pushCompoundTokens( + tokens: string[], + compound: string, + derived = true +): void { + const parts = COMPOUND_SEPARATOR_PATTERN.test(compound) + ? compound.split(COMPOUND_SEPARATOR_PATTERN) + : [compound]; + for (const part of parts) { + pushSearchToken(tokens, part); + if (derived && HAS_CAMEL_CASE_BOUNDARY_PATTERN.test(part)) { + for (const word of part.split(CAMEL_CASE_BOUNDARY_PATTERN)) { + pushSearchToken(tokens, word); + } + } + } + if ( + derived && + parts.length > 1 && + parts.length <= MAX_JOINED_COMPOUND_PARTS && + !parts.every((part) => DIGITS_ONLY_PATTERN.test(part)) + ) { + pushSearchToken(tokens, parts.join("")); + } +} + +// Case survives until each token is pushed so camel-case boundaries are +// still visible. +function foldForTokenizing(input: string): string { + return input.normalize("NFKD").replace(DIACRITIC_PATTERN, ""); +} + export function tokenizeDocsSearchText(input: string): string[] { const tokens: string[] = []; - for (const match of normalizeText(input).matchAll(WORD_CHARACTER_PATTERN)) { - const token = match[0]; - if (token.length > 1 && !STOPWORDS.has(token)) { - tokens.push(token); - } + for (const match of foldForTokenizing(input).matchAll( + COMPOUND_WORD_PATTERN + )) { + pushCompoundTokens(tokens, match[0]); + } + return tokens; +} + +/** + * Tokens for the words as written, without camel-case parts or joined + * compounds. Phrase matching also compares these, because the joined + * `nextjs` in `next js nextjs app` would otherwise break the phrase + * `next js app`. + */ +function tokenizeWrittenWords(input: string): string[] { + const tokens: string[] = []; + for (const match of foldForTokenizing(input).matchAll( + COMPOUND_WORD_PATTERN + )) { + pushCompoundTokens(tokens, match[0], false); } return tokens; } +/** Tokens grouped by the word they came from; stopword-only words are dropped. */ +function tokenizeWordGroups(input: string): string[][] { + const words: string[][] = []; + for (const match of foldForTokenizing(input).matchAll( + COMPOUND_WORD_PATTERN + )) { + const tokens: string[] = []; + pushCompoundTokens(tokens, match[0]); + if (tokens.length > 0) { + words.push(tokens); + } + } + return words; +} + export function countDocsSearchTokens(input: string): number { return tokenizeDocsSearchText(input).length; } @@ -404,16 +494,22 @@ function synonymTokensFor( ); } -function collectWeightedSearchTerms( +/** + * Index terms each distinct query token expands to, with the weight of the + * strongest way it matched (exact, stem, synonym, prefix, typo). + */ +function expandQueryTokens( index: DocsSearchIndex, queryTokens: string[], synonyms?: Record -): WeightedSearchTerm[] { - const weightedTerms = new Map(); +): Map> { + const expansions = new Map>(); const indexTerms = Object.keys(index.terms); const indexTermSet = new Set(indexTerms); - for (const token of queryTokens) { + for (const token of new Set(queryTokens)) { + const weightedTerms = new Map(); + expansions.set(token, weightedTerms); if (indexTermSet.has(token)) { addWeightedTerm(weightedTerms, token, EXACT_TERM_WEIGHT); } @@ -464,9 +560,80 @@ function collectWeightedSearchTerms( } } + return expansions; +} + +function mergeQueryExpansions( + expansions: Map> +): WeightedSearchTerm[] { + const weightedTerms = new Map(); + for (const terms of expansions.values()) { + for (const [term, weight] of terms) { + addWeightedTerm(weightedTerms, term, weight); + } + } return Array.from(weightedTerms, ([term, weight]) => ({ term, weight })); } +/** + * Share of the query a short field answers, from 0 to 1: each query token + * contributes the weight of its best expansion that appears in the field. + */ +function queryCoverage( + expansions: Map>, + fieldTokens: Set +): number { + if (expansions.size === 0 || fieldTokens.size === 0) { + return 0; + } + let covered = 0; + for (const [token, terms] of expansions) { + let best = fieldTokens.has(token) ? EXACT_TERM_WEIGHT : 0; + for (const [term, weight] of terms) { + if (weight > best && fieldTokens.has(term)) { + best = weight; + } + } + covered += best; + } + return covered / expansions.size; +} + +function isQueryMatch( + expansions: Map>, + fieldToken: string +): boolean { + for (const [token, terms] of expansions) { + if (token === fieldToken || terms.has(fieldToken)) { + return true; + } + } + return false; +} + +/** + * Share of a field's words the query accounts for, from 0 to 1. Separates a + * page titled "Translations" from a release titled "Translations Package, + * SSR Consent, CLI Telemetry, and Next.js Restructure". A word counts once + * however many tokens it yields, so `TypeScript` is fully matched by + * `typescript` alone. + */ +function fieldPrecision( + expansions: Map>, + fieldWords: string[][] +): number { + if (fieldWords.length === 0) { + return 0; + } + let matched = 0; + for (const word of fieldWords) { + if (word.some((token) => isQueryMatch(expansions, token))) { + matched += 1; + } + } + return matched / fieldWords.length; +} + export function countDocsSearchTerms(input: string): Map { const counts = new Map(); for (const token of tokenizeDocsSearchText(input)) { @@ -475,6 +642,49 @@ export function countDocsSearchTerms(input: string): Map { return counts; } +export type DocsSearchChunkTermCounts = { + title: Map; + heading: Map; + body: Map; + code: Map; +}; + +/** + * Term counts a chunk's postings store. A URL path word the chunk does not + * already contain counts once as body text, so a page found only through its + * path, such as `vue` in `/docs/frameworks/vue/quickstart`, still becomes a + * candidate, without adding weight to words the page already uses. + */ +export function countDocsSearchChunkTerms(chunk: { + title: string; + headingPath: string[]; + text: string; + codeText: string; + urlPath: string; +}): DocsSearchChunkTermCounts { + const counts = { + title: countDocsSearchTerms(chunk.title), + heading: countDocsSearchTerms(chunk.headingPath.join(" ")), + body: countDocsSearchTerms(chunk.text), + code: countDocsSearchTerms(chunk.codeText), + }; + // Written words only: a joined `typescriptsetup` from `typescript-setup` + // would become a prefix match for every `typescript` query. + for (const token of tokenizeWrittenWords(chunk.urlPath)) { + if ( + !( + counts.title.has(token) || + counts.heading.has(token) || + counts.body.has(token) || + counts.code.has(token) + ) + ) { + counts.body.set(token, 1); + } + } + return counts; +} + function hasUnsupportedControlCharacter(input: string): boolean { for (const character of input) { const codePoint = character.codePointAt(0); @@ -662,22 +872,54 @@ function normalizeTextWithOffsets(input: string): { return { normalized, offsets }; } -function buildExcerpt(text: string, queryTokens: string[]): string { - const { normalized: normalizedText, offsets } = - normalizeTextWithOffsets(text); - let matchIndex = -1; +/** + * The normalized text without `.`, `-` or `_` between two word characters, + * so a joined token such as `nextjs` can be found in `next.js`. Offsets keep + * pointing into the original text. + */ +function joinCompoundsWithOffsets( + normalized: string, + offsets: number[] +): { joined: string; offsets: number[] } { + let joined = ""; + const joinedOffsets: number[] = []; + for (let index = 0; index < normalized.length; index += 1) { + const character = normalized.charAt(index); + const isJoin = + COMPOUND_SEPARATOR_PATTERN.test(character) && + WORD_CHARACTER_PATTERN.test(normalized.charAt(index - 1)) && + WORD_CHARACTER_PATTERN.test(normalized.charAt(index + 1)); + if (!isJoin) { + joined += character; + joinedOffsets.push(offsets[index] ?? index); + } + } + return { joined, offsets: joinedOffsets }; +} + +function findOriginalMatchIndex(text: string, queryTokens: string[]): number { + const { normalized, offsets } = normalizeTextWithOffsets(text); + let compounds: { joined: string; offsets: number[] } | undefined; for (const token of queryTokens) { - matchIndex = normalizedText.indexOf(token); + const matchIndex = normalized.indexOf(token); if (matchIndex >= 0) { - break; + return offsets[matchIndex] ?? matchIndex; + } + compounds ??= joinCompoundsWithOffsets(normalized, offsets); + const joinedIndex = compounds.joined.indexOf(token); + if (joinedIndex >= 0) { + return compounds.offsets[joinedIndex] ?? joinedIndex; } } + return -1; +} - if (matchIndex < 0) { +function buildExcerpt(text: string, queryTokens: string[]): string { + const originalMatchIndex = findOriginalMatchIndex(text, queryTokens); + if (originalMatchIndex < 0) { return text.slice(0, 220).trim(); } - const originalMatchIndex = offsets[matchIndex] ?? matchIndex; const start = Math.max(0, originalMatchIndex - 80); const end = Math.min(text.length, originalMatchIndex + 160); const prefix = start > 0 ? "..." : ""; @@ -685,10 +927,6 @@ function buildExcerpt(text: string, queryTokens: string[]): string { return `${prefix}${text.slice(start, end).trim()}${suffix}`; } -function normalizeForPhraseMatch(input: string): string { - return tokenizeDocsSearchText(input).join(" "); -} - function hasOrderedProximityMatch( textTokens: string[], queryTokens: string[] @@ -731,27 +969,183 @@ function hasOrderedProximityMatch( return false; } -function scoreContentMatchBoost(text: string, queryTokens: string[]): number { +type QueryPhrase = { + tokens: string[]; + writtenWords: string; +}; + +function scoreContentMatchBoost(text: string, query: QueryPhrase): number { + const queryTokens = query.tokens; if (queryTokens.length < 2) { return 0; } const normalizedQuery = queryTokens.join(" "); - const normalizedText = normalizeForPhraseMatch(text); - if (normalizedText.includes(normalizedQuery)) { + const textTokens = tokenizeDocsSearchText(text); + if ( + textTokens.join(" ").includes(normalizedQuery) || + (query.writtenWords.includes(" ") && + tokenizeWrittenWords(text).join(" ").includes(query.writtenWords)) + ) { return PHRASE_MATCH_BOOST; } - const textTokens = tokenizeDocsSearchText(text); return hasOrderedProximityMatch(textTokens, queryTokens) ? PROXIMITY_MATCH_BOOST : 0; } +type PageMatchTokens = { + title: Set; + titleWords: string[][]; + titleAndPath: Set; +}; + +/** + * Score multiplier for chunks whose page title, URL path or own heading + * answers the query. BM25 alone saturates a title match into one more term + * occurrence, so a page titled "Consent categories" loses to a changelog + * that repeats both words. The path carries context titles often leave out, + * such as the framework in `/docs/frameworks/react/quickstart`. + */ +function pageMatchMultiplier( + chunk: DocsSearchChunk, + expansions: Map>, + cache: Map +): number { + let page = cache.get(chunk.documentId); + if (!page) { + const titleWords = tokenizeWordGroups(chunk.title); + const title = new Set(titleWords.flat()); + page = { + title, + titleWords, + titleAndPath: new Set([ + ...title, + ...tokenizeDocsSearchText(chunk.urlPath), + ]), + }; + cache.set(chunk.documentId, page); + } + const heading = new Set( + tokenizeDocsSearchText(chunk.headingPath.at(-1) ?? "") + ); + return ( + 1 + + TITLE_MATCH_BOOST * + queryCoverage(expansions, page.title) * + (1 + + TITLE_PRECISION_WEIGHT * fieldPrecision(expansions, page.titleWords)) + + PATH_MATCH_BOOST * queryCoverage(expansions, page.titleAndPath) + + HEADING_MATCH_BOOST * queryCoverage(expansions, heading) + ); +} + +type ScoredChunk = { + chunk: DocsSearchChunk; + termScore: number; + multiplier: number; + score: number; +}; + +const MAX_PAGE_MATCH_MULTIPLIER = + 1 + + TITLE_MATCH_BOOST * (1 + TITLE_PRECISION_WEIGHT) + + PATH_MATCH_BOOST + + HEADING_MATCH_BOOST; +const FIRST_CONTENT_BOOST_CHECKPOINT = 32; + +function compareScoredChunks(left: ScoredChunk, right: ScoredChunk): number { + if (right.score !== left.score) { + return right.score - left.score; + } + return left.chunk.absoluteUrl.localeCompare(right.chunk.absoluteUrl); +} + +/** Best chunks in score order, at most `maxResultsPerPage` from each page. */ +function selectTopChunks( + sorted: ScoredChunk[], + limit: number, + maxResultsPerPage: number +): ScoredChunk[] { + const resultsPerPage = new Map(); + const selected: ScoredChunk[] = []; + for (const scored of sorted) { + if (selected.length >= limit) { + break; + } + const pageCount = resultsPerPage.get(scored.chunk.documentId) ?? 0; + if (pageCount < maxResultsPerPage) { + resultsPerPage.set(scored.chunk.documentId, pageCount + 1); + selected.push(scored); + } + } + return selected; +} + +/** + * Adds the phrase and proximity boost in place. Matching tokenizes the whole + * chunk text, the costliest step of a query, so chunks are boosted in score + * order and the rest are skipped once even the largest boost could not lift + * them into the results. Adding chunks never lowers the score of the last + * selected result, so a chunk below that bound cannot be selected. The bound + * is refreshed at doubling checkpoints, so refreshing costs O(n log n) even + * when it never prunes. + */ +function applyContentMatchBoosts( + scoredChunks: ScoredChunk[], + query: QueryPhrase, + limit: number, + maxResultsPerPage: number +): void { + if (query.tokens.length < 2) { + return; + } + scoredChunks.sort(compareScoredChunks); + const maxBoost = PHRASE_MATCH_BOOST * MAX_PAGE_MATCH_MULTIPLIER; + let lastSelectedScore = Number.NEGATIVE_INFINITY; + let checkpoint = FIRST_CONTENT_BOOST_CHECKPOINT; + for (const [position, scored] of scoredChunks.entries()) { + if (scored.score + maxBoost < lastSelectedScore) { + return; + } + scored.score = + (scored.termScore + scoreContentMatchBoost(scored.chunk.text, query)) * + scored.multiplier; + if (position + 1 === checkpoint) { + checkpoint *= 2; + const boosted = scoredChunks + .slice(0, position + 1) + .sort(compareScoredChunks); + const selected = selectTopChunks(boosted, limit, maxResultsPerPage); + if (selected.length >= limit) { + lastSelectedScore = selected.at(-1)?.score ?? lastSelectedScore; + } + } + } +} + function requestError(message: string, status: number): never { throw new DocsSearchRequestError(message, status); } +/** + * Error for an index this leadtype cannot search, or `undefined` when it can. + * A version bump means the tokenizer changed, so an older index would load + * and quietly miss matches the current rules expect. + */ +export function docsSearchIndexVersionError( + index: { version?: unknown }, + source: string +): Error | undefined { + if (index.version === DOCS_SEARCH_INDEX_VERSION) { + return; + } + return new Error( + `leadtype: ${source} uses an unsupported version (${String(index.version)}); this leadtype reads search index version ${DOCS_SEARCH_INDEX_VERSION}. Run \`leadtype generate\` with this leadtype version and deploy the complete docs output.` + ); +} + export function isDocsSearchContentStoreCompatible( index: DocsSearchIndex, content: unknown @@ -922,7 +1316,7 @@ export function createDocsSearchIndex( ); const documents: DocsSearchIndex["documents"] = []; const mutableChunks: MutableChunk[] = []; - const chunkTermCounts = new Map(); + const chunkTermCounts = new Map(); const docs = runTransformersSync( options.transformers, @@ -961,6 +1355,10 @@ export function createDocsSearchIndex( } documents.push(entry); + // Only the page's first chunk carries its description. Repeating it in + // every chunk let a long description outrank the sections that answer + // the query. + let chunkDescription = description; for (const block of collectSectionBlocks(doc.content)) { const bodyParts = splitWithOverlap( block.text, @@ -978,7 +1376,7 @@ export function createDocsSearchIndex( const codeText = codeParts[partIndex] ?? ""; const chunkText = createChunkText( doc.title, - description, + chunkDescription, block.headingPath, [text, codeText].filter(Boolean).join("\n\n") ); @@ -1020,6 +1418,7 @@ export function createDocsSearchIndex( | undefined ); + chunkDescription = ""; const chunkIndex = mutableChunks.length; const chunkId = `chunk-${chunkIndex}`; const length = countDocsSearchTokens(transformedChunk.text); @@ -1033,14 +1432,16 @@ export function createDocsSearchIndex( codeText: transformedChunk.codeText, length, }); - chunkTermCounts.set(chunkIndex, { - title: countDocsSearchTerms(doc.title), - heading: countDocsSearchTerms(transformedChunk.headingPath.join(" ")), - body: countDocsSearchTerms( - [description, transformedChunk.text].join(" ") - ), - code: countDocsSearchTerms(transformedChunk.codeText), - }); + chunkTermCounts.set( + chunkIndex, + countDocsSearchChunkTerms({ + title: doc.title, + headingPath: transformedChunk.headingPath, + text: transformedChunk.text, + codeText: transformedChunk.codeText, + urlPath: doc.urlPath, + }) + ); } } } @@ -1105,11 +1506,8 @@ export function searchDocs( return []; } - const weightedTerms = collectWeightedSearchTerms( - index, - queryTokens, - options.synonyms - ); + const expansions = expandQueryTokens(index, queryTokens, options.synonyms); + const weightedTerms = mergeQueryExpansions(expansions); if (weightedTerms.length === 0) { return []; } @@ -1158,30 +1556,41 @@ export function searchDocs( } const limit = options.limit ?? DEFAULT_SEARCH_LIMIT; - const scoredChunks: Array<{ chunk: DocsSearchChunk; score: number }> = []; + const maxResultsPerPage = + options.maxResultsPerPage ?? DEFAULT_MAX_RESULTS_PER_PAGE; + const scoredChunks: ScoredChunk[] = []; + const pageTokens = new Map(); for (const [chunkId, score] of scores) { const chunk = readDocsContentChunk(index, chunkId, options.content); if (!chunk) { continue; } + const multiplier = pageMatchMultiplier(chunk, expansions, pageTokens); scoredChunks.push({ chunk, - score: score + scoreContentMatchBoost(chunk.text, queryTokens), + termScore: score, + multiplier, + score: score * multiplier, }); } - scoredChunks.sort((left, right) => { - if (right.score !== left.score) { - return right.score - left.score; - } - return left.chunk.absoluteUrl.localeCompare(right.chunk.absoluteUrl); - }); + applyContentMatchBoosts( + scoredChunks, + { + tokens: queryTokens, + writtenWords: tokenizeWrittenWords(query).join(" "), + }, + limit, + maxResultsPerPage + ); + scoredChunks.sort(compareScoredChunks); // Excerpts are expensive string scans, so build them only for results // that survive the limit. const excerptTokens = Array.from( new Set([...queryTokens, ...weightedTerms.map(({ term }) => term)]) ); - return scoredChunks.slice(0, limit).map(({ chunk, score }) => { + const topChunks = selectTopChunks(scoredChunks, limit, maxResultsPerPage); + return topChunks.map(({ chunk, score }) => { const excerptText = chunk.text || [chunk.title, chunk.description, ...chunk.headingPath].join(" "); @@ -1489,6 +1898,7 @@ export const docsSearchDefaults = { maxChunkChars: DEFAULT_MAX_CHUNK_CHARS, maxContextChars: DEFAULT_MAX_CONTEXT_CHARS, maxQueryChars: DEFAULT_MAX_QUERY_CHARS, + maxResultsPerPage: DEFAULT_MAX_RESULTS_PER_PAGE, maxSources: DEFAULT_MAX_SOURCES, overlapChars: DEFAULT_OVERLAP_CHARS, searchLimit: DEFAULT_SEARCH_LIMIT,