diff --git a/.github/scripts/check-worker-page-cost.mjs b/.github/scripts/check-worker-page-cost.mjs new file mode 100644 index 00000000..a3c99d33 --- /dev/null +++ b/.github/scripts/check-worker-page-cost.mjs @@ -0,0 +1,214 @@ +#!/usr/bin/env node +/** + * The Worker's cost per docs page, and how many more pages the budget holds + * (#395). It runs in `Worker bundle fits the size budget`, after the size + * itself is measured. + * + * #390's eleven ja pages and #388's PR 3 showed how a content batch finds out + * it is too big: the budget step goes red after the batch is written. A + * translated page costs its compiled module, its search text and a little + * routing data, and nothing said how much up front. So this prints, from the + * bundle just packaged: + * + * - the KiB each locale's page modules and search text take, per page; + * - what one more translated page costs: the most expensive locale's + * module and search text per page, plus `OVERHEAD_KIB`; + * - how many more such pages fit under the budget; + * - what one more fully translated locale (one page per English page) would + * cost, and whether it fits. + * + * `OVERHEAD_KIB` is what a page adds outside its module and search text: + * frontmatter, the route manifests, the prerender entry. Measured on #395: + * filling ja from 43 to 98 pages added 4,791 KiB of upload, of which 4,019 + * were page modules and 238 search text, so 534 KiB over 55 pages, about 10 + * KiB each. The projection is within about 2% of that measured growth. + * + * It reads the esbuild metafile OpenNext writes next to the bundled handler + * (`handler.mjs.meta.json`). Turbopack names a page's chunk after its file, + * `content_docs___mdx_.js`, and an English page's chunk + * has no locale part, so `bytesInOutput` per chunk is the bytes each page puts + * in the upload. A chunk under `content_docs_` that names no `_mdx_` file is + * shared between pages and is counted as such, not as a page. A locale's + * search text is the chunk of `.route-data/search-.ts`. + * + * The projection is a mean, not a promise: a page with long code samples costs + * more. It never fails the step on a number; the budget check above does that. + * It fails when it cannot read what it projects from (`unreadable`, + * `no-pages`), because a projection that silently prints nothing is the + * state #395 exists to end. + * + * Usage: `check-worker-page-cost.mjs --meta --size + * --budget `, from anywhere. `--self-test` runs every rule against fake + * metafiles and needs no build. + */ +import { readFileSync } from 'node:fs'; +import { basename, dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const KIB = 1024; +const OVERHEAD_KIB = 10; // per page, outside its module and search text (header) +const RULES = ['unreadable', 'no-pages']; + +/** `languages` and `defaultLanguage` out of `i18n.ts`. Throws rather than guessing. */ +function readI18n(text) { + const languages = /languages:\s*\[([^\]]+)\]/.exec(text)?.[1]?.split(',').map((s) => s.trim().replace(/['"]/g, '')).filter(Boolean) ?? []; + const defaultLanguage = /defaultLanguage:\s*['"]([^'"]+)['"]/.exec(text)?.[1]; + if (!languages.length || !languages.includes(defaultLanguage)) { + throw new Error('could not read languages[] and defaultLanguage out of apps/docs/lib/i18n.ts'); + } + return { languages, defaultLanguage }; +} + +/** Per-locale page bytes out of a parsed metafile. */ +function pageBytes(meta, { languages, defaultLanguage }) { + const locales = new Map(languages.map((l) => [l, { pages: 0, bytes: 0, search: 0 }])); + let shared = 0; + for (const output of Object.values(meta?.outputs ?? {})) { + for (const [path, input] of Object.entries(output.inputs ?? {})) { + const name = basename(path); + const search = languages.find((l) => name.includes(`route-data_search-${l}_ts`)); + if (search) locales.get(search).search += input.bytesInOutput; + if (!name.startsWith('content_docs_')) continue; + if (!name.includes('_mdx_')) { + shared += input.bytesInOutput; + continue; + } + const locale = languages.find((l) => l !== defaultLanguage && name.includes(`_${l}_mdx_`)) ?? defaultLanguage; + const row = locales.get(locale); + row.pages += 1; + row.bytes += input.bytesInOutput; + } + } + return { locales, shared }; +} + +/** The summary, as Markdown lines, plus findings. */ +function project({ meta, size, budget, i18n }) { + const findings = []; + if (!meta || !Number.isFinite(size) || !Number.isFinite(budget)) { + findings.push({ rule: 'unreadable', detail: 'no metafile, size or budget to project from' }); + return { findings, lines: [] }; + } + const { locales, shared } = pageBytes(meta, i18n); + const english = locales.get(i18n.defaultLanguage); + const translated = [...locales].filter(([l, r]) => l !== i18n.defaultLanguage && r.pages > 0); + if (!english.pages) { + findings.push({ rule: 'no-pages', detail: 'the metafile names no `content_docs_*_mdx_*` chunk, so the page modules were not found' }); + return { findings, lines: [] }; + } + // A translated page is what the next batch adds; English stands in when no locale has one yet. + // The most expensive locale sets the cost, so the projection errs towards less room. + const cost = (r) => (r.bytes + r.search) / r.pages / KIB; + const [costliest, worst] = (translated.length ? translated : [[i18n.defaultLanguage, english]]).reduce((a, b) => (cost(b[1]) > cost(a[1]) ? b : a)); + const perPage = cost(worst) + OVERHEAD_KIB; + const headroom = budget - size; + const left = Math.max(0, Math.floor(headroom / perPage)); + const locale = english.pages * perPage; + const lines = [ + '#### Per-page cost', + '', + '| locale | page modules | module KiB | search KiB | KiB per page |', + '|:--|--:|--:|--:|--:|', + ...[...locales] + .filter(([, r]) => r.pages) + .map(([l, r]) => `| ${l} | ${r.pages} | ${(r.bytes / KIB).toFixed(2)} | ${(r.search / KIB).toFixed(2)} | ${cost(r).toFixed(2)} |`), + `| shared | — | ${(shared / KIB).toFixed(2)} | — | — |`, + '', + `A translated page costs about **${perPage.toFixed(2)} KiB**: ${costliest}'s ${cost(worst).toFixed(2)} KiB per page, the most of any locale, plus ${OVERHEAD_KIB} KiB of routing data.`, + `The budget holds **${left}** more such pages (${headroom.toFixed(2)} KiB of headroom).`, + `One more fully translated locale (${english.pages} pages) would add about ${locale.toFixed(0)} KiB: ${ + locale <= headroom ? `it fits, leaving ${(headroom - locale).toFixed(0)} KiB` : `it does NOT fit, by ${(locale - headroom).toFixed(0)} KiB` + }.`, + ]; + return { findings, lines }; +} + +function arg(name) { + const i = process.argv.indexOf(name); + return i < 0 ? undefined : process.argv[i + 1]; +} + +function gate() { + const i18n = readI18n(readFileSync(join(ROOT, 'apps/docs/lib/i18n.ts'), 'utf8')); + let meta; + try { + meta = JSON.parse(readFileSync(arg('--meta') ?? '', 'utf8')); + } catch { + meta = undefined; + } + const { findings, lines } = project({ meta, size: Number(arg('--size')), budget: Number(arg('--budget')), i18n }); + if (lines.length) console.log(lines.join('\n')); + for (const f of findings) console.error(` (${f.rule}) ${f.detail}`); + if (findings.length) { + console.error(`\n✗ Worker page cost: NOT MEASURED. The rules are in this script's header.`); + return 1; + } + return 0; +} + +/* Self-test: fake metafiles, each case checked for its findings and, where it projects, its numbers. */ +const I18N = { languages: ['en', 'ja', 'de'], defaultLanguage: 'en' }; +const chunk = (name, kib) => [`.next/server/chunks/ssr/${name}`, { bytesInOutput: kib * KIB }]; +const metafile = (...chunks) => ({ outputs: { 'handler.mjs': { inputs: Object.fromEntries([chunk('next_dist_x.js', 900), ...chunks]) } } }); +const CASES = [ + [ + 'ja pages set the per-page cost; en and shared are listed, not averaged', + { meta: metafile(chunk('content_docs_a_mdx_1.js', 50), chunk('content_docs_b_mdx_2.js', 70), chunk('content_docs_a_ja_mdx_3.js', 100), chunk('content_docs_x._.js', 30)), size: 1000, budget: 1450, i18n: I18N }, + [], + ['**110.00 KiB**', 'ja\'s 100.00', '**4** more', '2 pages) would add about 220 KiB: it fits, leaving 230 KiB', '| shared | — | 30.00 |'], + ], + [ + 'no translation yet: English sets the cost', + { meta: metafile(chunk('content_docs_a_mdx_1.js', 40), chunk('content_docs_b_mdx_2.js', 60)), size: 1000, budget: 1100, i18n: I18N }, + [], + ['**60.00 KiB**', '**1** more', 'about 120 KiB: it does NOT fit, by 20 KiB'], + ], + [ + 'over the budget: no page left, and a locale does not fit', + { meta: metafile(chunk('content_docs_a_mdx_1.js', 40), chunk('content_docs_a_de_mdx_9.js', 80)), size: 1200, budget: 1100, i18n: I18N }, + [], + ['**0** more', 'it does NOT fit, by 190 KiB'], + ], + [ + 'search text counts, and the costliest locale sets the cost', + { meta: metafile(chunk('content_docs_a_mdx_1.js', 40), chunk('content_docs_a_ja_mdx_3.js', 60), chunk('content_docs_b_ja_mdx_4.js', 60), chunk('apps_docs__route-data_search-ja_ts_0xv.js', 20), chunk('content_docs_a_de_mdx_9.js', 50)), size: 1000, budget: 1400, i18n: I18N }, + [], + ['| ja | 2 | 120.00 | 20.00 | 70.00 |', 'about **80.00 KiB**: ja\'s 70.00', '**5** more'], + ], + ['no metafile', { meta: undefined, size: 1, budget: 2, i18n: I18N }, ['unreadable'], []], + ['no size', { meta: metafile(chunk('content_docs_a_mdx_1.js', 40)), size: NaN, budget: 2, i18n: I18N }, ['unreadable'], []], + ['the chunks were renamed', { meta: metafile(chunk('pages_a.js', 40)), size: 1, budget: 2, i18n: I18N }, ['no-pages'], []], +]; + +function selfTest() { + let failed = 0; + const fired = new Set(); + for (const [name, input, wantRules, wantText] of CASES) { + const { findings, lines } = project(input); + const got = findings.map((f) => f.rule).sort(); + got.forEach((r) => fired.add(r)); + const text = lines.join('\n'); + const missing = wantText.filter((t) => !text.includes(t)); + const pass = JSON.stringify(got) === JSON.stringify([...wantRules].sort()) && !missing.length; + if (!pass) failed += 1; + console.log(`${pass ? '✓' : '✗'} ${name}: ${got.length ? got.join(', ') : 'no findings'}${missing.length ? ` — missing ${JSON.stringify(missing)}` : ''}`); + } + const silent = RULES.filter((r) => !fired.has(r)); + if (silent.length) { + failed += 1; + console.log(`✗ rules no fixture makes fire: ${silent.join(', ')}`); + } + let unreadable = false; + try { + readI18n('defineI18n({})'); + } catch { + unreadable = true; + } + if (!unreadable) failed += 1; + console.log(`${unreadable ? '✓' : '✗'} an unreadable i18n.ts is rejected`); + console.log(failed ? `\n✗ Worker page cost self-test: ${failed} failure(s)` : `\n✓ Worker page cost self-test: ${CASES.length} cases, all ${RULES.length} rules can fire`); + return failed ? 1 : 0; +} + +process.exitCode = process.argv.includes('--self-test') ? selfTest() : gate(); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ed37e43..6754206e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -409,6 +409,7 @@ jobs: exit 1 fi + VERDICT=0 awk -v size="$SIZE_KIB" -v budget="$WORKER_BUDGET_KIB" -v limit="$WORKER_LIMIT_KIB" ' BEGIN { over = (size > budget) @@ -426,7 +427,18 @@ jobs: printf "one of them. Reduce the bundle, or raise the budget deliberately and say why.\n" } exit (over ? 1 : 0) - }' | tee -a "$GITHUB_STEP_SUMMARY" + }' | tee -a "$GITHUB_STEP_SUMMARY" || VERDICT=$? + + # #395: what the next content batch has room for, printed whether or + # not this one fits. The KiB a translated page costs, read from the + # bundle just packaged, and how many more pages and locales that + # leaves under the budget. It goes red only when it cannot read the + # bundle; the verdict above is the budget gate. + echo | tee -a "$GITHUB_STEP_SUMMARY" + node "$GITHUB_WORKSPACE/.github/scripts/check-worker-page-cost.mjs" \ + --meta .open-next/server-functions/default/apps/docs/handler.mjs.meta.json \ + --size "$SIZE_KIB" --budget "$WORKER_BUDGET_KIB" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$VERDICT" # `include-hidden-files` is load-bearing, not tidiness: the compiled # OpenNext config the deploy reads lives at `.open-next/.build/`, and diff --git a/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx b/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx index b23fbc38..cf977c19 100644 --- a/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx +++ b/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx @@ -17,6 +17,7 @@ import type { InferPageType } from 'fumadocs-core/source'; import type { TOCItemType } from 'fumadocs-core/toc'; import { EditionBadges } from '@/components/editions'; import { EDITIONS_PAGE_SLUGS } from '@/lib/editions'; +import { loadPageBody } from '@/lib/page-body'; /** * The page type the loader actually produces, frontmatter schema included. @@ -267,7 +268,11 @@ export default async function Page(props: { // Under `dynamicParams = false` this await runs at BUILD time, in Node, on // every one of the 1139 prerendered paths — the Worker serves the prerendered // result out of the static-assets incremental cache and does not re-render. - const loaded = await page.data.load(); + // + // Loaded through `lib/page-body.ts`, not `page.data.load()`: `source` is + // built without the compiled bodies so the route handlers do not bundle a + // second copy of every page (#395). + const loaded = await loadPageBody(page); const MDX = loaded.body; // Resolved once and handed to both controls (through `copySource` and @@ -309,11 +314,11 @@ export default async function Page(props: { // translation it is this locale's text, which the `.mdx` surface does not // serve (English-only, see `markdownUrl`), so it is rendered into the page // instead: `getLLMText` builds it exactly as `/llms.mdx/docs/...` builds the - // English body, from the module `page.data.load()` above already loaded. + // English body, from the module `loadPageBody` above already loaded. const copySource: MarkdownSource = contentLang === i18n.defaultLanguage ? { url: pageMarkdownUrl } - : { text: await getLLMText(page) }; + : { text: await getLLMText(page, loaded._exports._markdown as string) }; // "Open in ChatGPT / Claude" follows the same rule (#305): the assistant is // sent to the page on screen. The English Markdown when that is the English diff --git a/apps/docs/app/api/search/route.ts b/apps/docs/app/api/search/route.ts index b41e3342..a762706a 100644 --- a/apps/docs/app/api/search/route.ts +++ b/apps/docs/app/api/search/route.ts @@ -1,6 +1,7 @@ import { source } from '@/lib/source'; import { i18n } from '@/lib/i18n'; -import { searchData } from '@/.search'; +import { unpack } from '@/lib/packed'; +import { searchData } from '@/.route-data/search'; import { createSearchAPI, type AdvancedIndex, @@ -123,9 +124,9 @@ type StructuredData = AdvancedIndex['structuredData']; /** * One locale's `structuredData` — each page's headings and paragraphs — keyed - * by file path under `content/docs`, from `.search/` (#393). + * by file path under `content/docs`, from `.route-data/` (#393). * - * `scripts/gen-search-data.mjs` writes it before every build, with the same + * `scripts/gen-route-data.mjs` writes it before every build, with the same * remark chain fumadocs-mdx compiles the pages with. Reading it from there * instead of from `page.data.load()` is the point: `load()` imports the page's * compiled module, the module registry keeps it for the life of the isolate, @@ -134,11 +135,7 @@ type StructuredData = AdvancedIndex['structuredData']; * dropped once the index has the text. */ async function texts(locale: Locale): Promise> { - // Gzipped JSON in base64, because the module is Worker upload too. - const packed = (await searchData[locale]()).default; - const bytes = Uint8Array.from(atob(packed), (c) => c.charCodeAt(0)); - const json = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip')); - return JSON.parse(await new Response(json).text()); + return unpack((await searchData[locale]()).default); } /** @@ -156,10 +153,10 @@ async function indexes(locale: Locale): Promise { const file = page.data.info.path; let structuredData = own[file] ?? fallback?.[file]; if (!structuredData) { - // `next dev` serves pages added since it wrote `.search/`; a build - // writes it first, so there a missing page is a stale `.search/`. + // `next dev` serves pages added since it wrote `.route-data/`; a build + // writes it first, so there a missing page is a stale `.route-data/`. if (process.env.NODE_ENV !== 'development') { - throw new Error(`/api/search: no search data for ${file}; run scripts/gen-search-data.mjs`); + throw new Error(`/api/search: no search data for ${file}; run scripts/gen-route-data.mjs`); } structuredData = (await page.data.load()).structuredData; } diff --git a/apps/docs/lib/packed.ts b/apps/docs/lib/packed.ts new file mode 100644 index 00000000..8d448fed --- /dev/null +++ b/apps/docs/lib/packed.ts @@ -0,0 +1,12 @@ +/** + * Reads a value `scripts/gen-route-data.mjs` packed: JSON, gzipped, in base64. + * + * The generated modules are Worker upload as well as text the route handlers + * read, so they are stored compressed (#393). `atob`, `Blob` and + * `DecompressionStream` are in both Node and the Workers runtime. + */ +export async function unpack(packed: string): Promise { + const bytes = Uint8Array.from(atob(packed), (c) => c.charCodeAt(0)); + const json = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip')); + return JSON.parse(await new Response(json).text()); +} diff --git a/apps/docs/lib/page-body.ts b/apps/docs/lib/page-body.ts new file mode 100644 index 00000000..301e9a9c --- /dev/null +++ b/apps/docs/lib/page-body.ts @@ -0,0 +1,22 @@ +import { docs } from 'fumadocs-mdx:collections/server'; +import type { InferPageType } from 'fumadocs-core/source'; +import type { source } from '@/lib/source'; + +/** + * Each page's entry in fumadocs-mdx's full collection, by file path under + * `content/docs`. + * + * This is the one module that imports the full collection, with its lazy + * import of every compiled page (#395). Import it from the page route only: + * anything a route handler imports puts every compiled page in the route + * handlers' module graph, a second copy of them in the Worker upload, which is + * what `lib/source.ts` is built to avoid. + */ +const entries = new Map(docs.docs.map((entry) => [entry.info.path, entry])); + +/** `page`'s compiled body, toc and exports, as `page.data.load()` returns them on the full collection. */ +export async function loadPageBody(page: InferPageType) { + const entry = entries.get(page.data.info.path); + if (!entry) throw new Error(`${page.data.info.path}: not in the fumadocs-mdx collection`); + return entry.load(); +} diff --git a/apps/docs/lib/source.ts b/apps/docs/lib/source.ts index 5feca672..b478cef4 100644 --- a/apps/docs/lib/source.ts +++ b/apps/docs/lib/source.ts @@ -1,8 +1,22 @@ -import { docs } from 'fumadocs-mdx:collections/server'; +import { docs } from '@/.route-data/collection'; import { type InferPageType, loader } from 'fumadocs-core/source'; import { lucideIconsPlugin } from 'fumadocs-core/source/lucide-icons'; import { i18n } from '@/lib/i18n'; +/** + * Every page's tree position, URL and frontmatter, WITHOUT its compiled body + * (#395). + * + * The collection is fumadocs-mdx's own, rewritten by + * `scripts/gen-route-data.mjs` so that a page's body is not a lazy import of + * its compiled module. Six route handlers import this file, and a lazy import + * is still bundled: with the full collection here the route handlers' module + * graph carried a second copy of every page, about 20 MB of the Worker upload. + * `page.data.load()` therefore does not return a renderable body on this + * source; the page route loads bodies through `lib/page-body.ts`, which is the + * one module that imports the full collection. `getText('processed')` works + * for English pages, which is what the `llms` routes read. + */ export const source = loader({ baseUrl: '/docs', i18n, @@ -79,8 +93,14 @@ function stripMdxComments(markdown: string): string { return markdown.replace(FENCE_OR_MDX_COMMENT, (match, fence?: string) => (fence ? match : '')); } -export async function getLLMText(page: InferPageType) { - const processed = stripMdxComments(await page.data.getText('processed')); +/** + * `page`'s Markdown as the `llms` surfaces serve it. `processed` is the page's + * processed Markdown when the caller already has it (the page route, from the + * module it rendered); otherwise it is read through `source`, which has it for + * English pages only. + */ +export async function getLLMText(page: InferPageType, processed?: string) { + processed = stripMdxComments(processed ?? (await page.data.getText('processed'))); return `# ${page.data.title} diff --git a/apps/docs/package.json b/apps/docs/package.json index 7d875e3c..1c23e75a 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -5,12 +5,12 @@ "description": "ObjectOS Documentation Site", "license": "Apache-2.0", "scripts": { - "dev": "fumadocs-mdx && node scripts/gen-search-data.mjs && next dev", + "dev": "fumadocs-mdx && node scripts/gen-route-data.mjs --dev && next dev", "gen:zh-hant": "node scripts/gen-zh-hant.mjs", - "build": "fumadocs-mdx && node scripts/gen-search-data.mjs && next build", + "build": "fumadocs-mdx && node scripts/gen-route-data.mjs && next build", "start": "next start", "lint": "next lint", - "type-check": "next typegen && fumadocs-mdx && node scripts/gen-search-data.mjs && tsc --noEmit && node scripts/check-frontmatter.mjs", + "type-check": "next typegen && fumadocs-mdx && node scripts/gen-route-data.mjs && tsc --noEmit && node scripts/check-frontmatter.mjs", "postinstall": "fumadocs-mdx", "preview": "opennextjs-cloudflare build && opennextjs-cloudflare preview", "deploy": "opennextjs-cloudflare build && opennextjs-cloudflare deploy", diff --git a/apps/docs/scripts/gen-route-data.mjs b/apps/docs/scripts/gen-route-data.mjs new file mode 100644 index 00000000..8f69feaa --- /dev/null +++ b/apps/docs/scripts/gen-route-data.mjs @@ -0,0 +1,203 @@ +#!/usr/bin/env node +/** + * Writes what the route handlers read instead of the compiled pages: + * `apps/docs/.route-data/` (#393, #395). + * + * ## Search text (#393) + * + * `/api/search` used to read each page's `structuredData` (its headings and + * paragraphs) through `page.data.load()`, which imports the page's compiled + * MDX module. That is the whole rendered page, highlighted code included, and + * a module once imported stays in the module registry for the life of the + * isolate. Building one locale's index therefore left every page module of + * that locale resident after the index was gone. Measured on the built route + * at `09784e7`, with only one index kept: 58 MB after English, 86 MB after all + * eight locales, about 94 MB with ja fully translated, against a Worker + * isolate's 128 MB. + * + * So the same `structuredData` is computed here and written as one module per + * locale, `search-.ts`, which `search.ts` maps to a lazy import. The + * route imports the locale it is asked for, unzips and parses it. + * + * ## The route handlers' collection (#395) + * + * Every page's compiled module was in the Worker twice. Turbopack compiles the + * page routes and the route handlers as two separate module graphs, and six + * route handlers (search, og, sitemap, `llms.txt`, `llms-full.txt`, + * `llms.mdx`) import `lib/source.ts`, whose collection holds a lazy + * `import()` of every page. A lazy import is still bundled, so the handlers' + * graph carried a second copy of all of them: 19,847 KiB of a 61,529 KiB + * upload with #388's PR 3 applied, which is what put that PR over the budget. + * Only the page route renders a page body, and only at build time. + * + * So `collection.ts` is the collection fumadocs-mdx writes to + * `.source/server.ts`, with every page's lazy body import replaced by a + * function that reads this directory instead. The frontmatter and `meta.json` + * imports are kept as they are, so the page tree, URLs and frontmatter are the + * ones `fumadocs-mdx` built. `lib/source.ts` builds its loader on it; the page + * route loads bodies from the full collection itself. + * + * What the handlers read from a body is the processed Markdown + * (`getText('processed')`, for the `llms` routes), and only for English pages: + * `/llms.mdx/docs/...` and `/llms-full.txt` serve English. That is + * `markdown.ts`. A translation's Markdown is read by the page route, from the + * module it already loaded. + * + * With `--dev`, `collection.ts` re-exports `.source/server.ts` instead: + * `next dev` regenerates that file when a page is added, and a copy taken at + * startup would not see the new page. + * + * ## How the text is made + * + * It is the same text, not a lookalike. The processor is fumadocs-mdx's own + * remark chain, built with the public `applyMdxPreset` from the compiled + * `source.config` (`.source/source.config.mjs`, written by the `fumadocs-mdx` + * CLI), plus `remarkInclude`, which `buildMDX` puts first, and `remarkLLMs`, + * which its postprocess step runs last for `includeProcessedMarkdown`. + * `remarkStructure` runs in the remark phase too, so the rehype plugins, which + * run after both and only shape the rendered HTML, are left out: that is + * `rehypeCode`, and it is most of the cost. + * + * Every string is JSON, gzipped, in base64 (`lib/packed.ts` unpacks it), + * because it is Worker upload too: as plain JSON the English, zh-Hans and + * zh-Hant search modules were about 1 MB each. + * + * A file's locale is its suffix, as `source.config.ts` reads it: `foo.ja.mdx` + * is `ja`, `foo.mdx` is the default language. A locale whose page falls back + * to English reads that page from the default language's module, so English + * text is written once. + * + * The output directory carries its own `.gitignore`, because it is a build + * product. Run from `apps/docs`, after `fumadocs-mdx`, as the `build`, + * `type-check` and `dev` scripts do. + */ +import { applyMdxPreset, remarkInclude } from 'fumadocs-mdx/config'; +import { frontmatter } from 'fumadocs-core/content/md/frontmatter'; +import { remarkLLMs } from 'fumadocs-core/mdx-plugins/remark-llms'; +import { createRequire } from 'node:module'; +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { gzipSync } from 'node:zlib'; + +const APP = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const CONTENT = resolve(APP, '../../content/docs'); +const CONFIG = join(APP, '.source/source.config.mjs'); +const COLLECTION = join(APP, '.source/server.ts'); +const OUT = join(APP, '.route-data'); +const DEV = process.argv.includes('--dev'); + +/** `languages` and `defaultLanguage` out of `lib/i18n.ts`, read as text. Throws rather than guessing. */ +function readI18n() { + const text = readFileSync(join(APP, 'lib/i18n.ts'), 'utf8'); + const languages = /languages:\s*\[([^\]]+)\]/.exec(text)?.[1]?.split(',').map((s) => s.trim().replace(/['"]/g, '')).filter(Boolean) ?? []; + const defaultLanguage = /defaultLanguage:\s*['"]([^'"]+)['"]/.exec(text)?.[1]; + if (!languages.length || !languages.includes(defaultLanguage)) { + throw new Error('could not read languages[] and defaultLanguage out of apps/docs/lib/i18n.ts'); + } + return { languages, defaultLanguage }; +} + +function walk(dir) { + return readdirSync(dir, { withFileTypes: true }) + .flatMap((e) => (e.isDirectory() ? walk(join(dir, e.name)) : e.name.endsWith('.mdx') ? [join(dir, e.name)] : [])) + .sort(); +} + +/** + * `.source/server.ts` with each page's `() => import("…?collection=docs")` + * replaced by `() => body("")`. Throws unless every page's import was + * replaced and no other `import(` is left, so a change in what fumadocs-mdx + * writes fails the build here instead of putting the bodies back. + */ +function lightCollection(source, pages) { + const BODY = /("[^"]+\.mdx"): \(\) => import\("[^"]+\?collection=docs"\)/g; + const replaced = source.match(BODY)?.length ?? 0; + const light = source.replace(BODY, (_, key) => `${key}: () => body(${key})`); + const left = light.match(/\bimport\(\s*["'][^"']*\?collection=/g)?.length ?? 0; + if (replaced !== pages || left) { + throw new Error( + `.source/server.ts: replaced ${replaced} page body import(s) of ${pages} pages and ${left} are left. ` + + 'fumadocs-mdx changed what it writes; update lightCollection() in scripts/gen-route-data.mjs.', + ); + } + return `${light} +import { unpack } from '../lib/packed'; + +/** + * What a route handler reads from a page body (scripts/gen-route-data.mjs): the + * processed Markdown of an English page. The page route renders bodies from the + * full collection, so nothing else is asked for here. + */ +let markdown: Promise> | undefined; +async function body(file: string) { + markdown ??= import('./markdown').then((m) => unpack>(m.default)); + const text = (await markdown)[file]; + if (text === undefined) { + throw new Error(\`\${file}: the route handlers' collection has no body for it; only English pages' processed Markdown is generated (scripts/gen-route-data.mjs)\`); + } + return { _markdown: text }; +} +`; +} + +if (!existsSync(CONFIG) || !existsSync(COLLECTION)) { + console.error(`✗ route data: ${relative(process.cwd(), CONFIG)} or ${relative(process.cwd(), COLLECTION)} does not exist. Run \`fumadocs-mdx\` first.`); + process.exit(1); +} + +const { languages, defaultLanguage } = readI18n(); +// @mdx-js/mdx is fumadocs-mdx's dependency, not this app's, so it is resolved from there: the copy buildMDX uses. +const { createProcessor } = await import( + pathToFileURL(createRequire(import.meta.resolve('fumadocs-mdx')).resolve('@mdx-js/mdx')).href +); +const config = (await import(pathToFileURL(CONFIG).href)).default; +const mdx = await applyMdxPreset({ ...config.mdxOptions, rehypeCodeOptions: false })('bundler'); +const processor = createProcessor({ + outputFormat: 'program', + ...mdx, + remarkPlugins: [remarkInclude, ...(mdx.remarkPlugins ?? []), [remarkLLMs, { _data: true }]], + rehypePlugins: [], + format: 'mdx', +}); + +const search = new Map(languages.map((l) => [l, {}])); +const markdown = {}; +const files = walk(CONTENT); +for (const path of files) { + const file = relative(CONTENT, path).split('\\').join('/'); + const locale = languages.find((l) => l !== defaultLanguage && file.endsWith(`.${l}.mdx`)) ?? defaultLanguage; + const vfile = await processor.process({ + path, + value: frontmatter(readFileSync(path, 'utf8')).content, + data: { frontmatter: {}, _getProcessor: () => processor }, + }); + if (!vfile.data.structuredData) throw new Error(`${file}: remarkStructure wrote no structuredData`); + if (typeof vfile.data.markdown !== 'string') throw new Error(`${file}: remarkLLMs wrote no Markdown`); + search.get(locale)[file] = vfile.data.structuredData; + if (locale === defaultLanguage) markdown[file] = vfile.data.markdown; +} + +const banner = '// Generated by scripts/gen-route-data.mjs before every build. Do not edit.\n'; +const pack = (value) => gzipSync(JSON.stringify(value), { level: 9 }).toString('base64'); +const write = (name, text) => writeFileSync(join(OUT, name), `${banner}${text}`); + +rmSync(OUT, { recursive: true, force: true }); +mkdirSync(OUT, { recursive: true }); +writeFileSync(join(OUT, '.gitignore'), '*\n'); +for (const [locale, pages] of search) { + // A string, not an object literal: the route parses it when it builds the + // index, and the parsed copy is garbage once the index holds the text. + write(`search-${locale}.ts`, `export default ${JSON.stringify(pack(pages))};\n`); +} +const loaders = languages.map((l) => ` ${JSON.stringify(l)}: () => import(${JSON.stringify(`./search-${l}`)}),`).join('\n'); +write('search.ts', `/** Each locale's \`structuredData\`, keyed by file path under content/docs, packed (lib/packed.ts). */\nexport const searchData = {\n${loaders}\n} as const;\n`); +write('markdown.ts', `/** Each English page's processed Markdown, keyed by file path under content/docs, packed (lib/packed.ts). */\nexport default ${JSON.stringify(pack(markdown))};\n`); +write( + 'collection.ts', + DEV + ? "/** `next dev`: the full collection, which fumadocs-mdx keeps current as pages are added. */\nexport { docs } from '../.source/server';\n" + : lightCollection(readFileSync(COLLECTION, 'utf8'), files.length), +); +const sizes = [...search].map(([l, pages]) => `${l} ${Object.keys(pages).length}`).join(', '); +console.log(`✓ route data: wrote .route-data/${DEV ? ' for next dev' : ''} (pages per locale: ${sizes}; English Markdown: ${Object.keys(markdown).length})`); diff --git a/apps/docs/scripts/gen-search-data.mjs b/apps/docs/scripts/gen-search-data.mjs deleted file mode 100644 index 59c20e38..00000000 --- a/apps/docs/scripts/gen-search-data.mjs +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env node -/** - * Writes the text `/api/search` indexes, one module per locale (#393). - * - * The route used to read each page's `structuredData` (its headings and - * paragraphs) through `page.data.load()`, which imports the page's compiled - * MDX module. That is the whole rendered page, highlighted code included, and - * a module once imported stays in the module registry for the life of the - * isolate. Building one locale's index therefore left every page module of - * that locale resident after the index was gone. Measured on the built route - * at `09784e7`, with only one index kept: 58 MB after English, 86 MB after all - * eight locales, about 94 MB with ja fully translated, against a Worker - * isolate's 128 MB. - * - * So the same `structuredData` is computed here, before `next build`, and - * written as one string per locale to `.search/.ts`: the JSON, - * gzipped, in base64. The route imports the locale it is asked for, unzips and - * parses it. What stays resident is that string, not the page modules. - * - * Gzipped because the string is also Worker upload, which has a budget the - * `Worker bundle fits the size budget` CI step holds: as plain JSON the - * English, zh-Hans and zh-Hant modules were about 1 MB each, 3.5 MB of the - * 6 MB headroom main had at `09784e7`; gzipped and in base64, about a third. - * - * It is the same `structuredData`, not a lookalike. The processor is - * fumadocs-mdx's own remark chain, built with the public `applyMdxPreset` from - * the compiled `source.config` (`.source/source.config.mjs`, written by the - * `fumadocs-mdx` CLI), plus `remarkInclude`, which `buildMDX` puts first. - * `remarkStructure` runs in the remark phase, so the rehype plugins, which run - * after it and only shape the rendered HTML, are left out: that is - * `rehypeCode`, and it is most of the cost. - * - * A file's locale is its suffix, as `source.config.ts` reads it: `foo.ja.mdx` - * is `ja`, `foo.mdx` is the default language. A locale whose page falls back - * to English reads that page from the default language's module, so English - * text is written once. - * - * The output directory carries its own `.gitignore`, because it is a build - * product. Run from `apps/docs`, after `fumadocs-mdx`, as the `build`, - * `type-check` and `dev` scripts do. - */ -import { applyMdxPreset, remarkInclude } from 'fumadocs-mdx/config'; -import { frontmatter } from 'fumadocs-core/content/md/frontmatter'; -import { createRequire } from 'node:module'; -import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; -import { dirname, join, relative, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { gzipSync } from 'node:zlib'; - -const APP = resolve(dirname(fileURLToPath(import.meta.url)), '..'); -const CONTENT = resolve(APP, '../../content/docs'); -const CONFIG = join(APP, '.source/source.config.mjs'); -const OUT = join(APP, '.search'); - -/** `languages` and `defaultLanguage` out of `lib/i18n.ts`, read as text. Throws rather than guessing. */ -function readI18n() { - const text = readFileSync(join(APP, 'lib/i18n.ts'), 'utf8'); - const languages = /languages:\s*\[([^\]]+)\]/.exec(text)?.[1]?.split(',').map((s) => s.trim().replace(/['"]/g, '')).filter(Boolean) ?? []; - const defaultLanguage = /defaultLanguage:\s*['"]([^'"]+)['"]/.exec(text)?.[1]; - if (!languages.length || !languages.includes(defaultLanguage)) { - throw new Error('could not read languages[] and defaultLanguage out of apps/docs/lib/i18n.ts'); - } - return { languages, defaultLanguage }; -} - -function walk(dir) { - return readdirSync(dir, { withFileTypes: true }) - .flatMap((e) => (e.isDirectory() ? walk(join(dir, e.name)) : e.name.endsWith('.mdx') ? [join(dir, e.name)] : [])) - .sort(); -} - -if (!existsSync(CONFIG)) { - console.error(`✗ search data: ${relative(process.cwd(), CONFIG)} does not exist. Run \`fumadocs-mdx\` first.`); - process.exit(1); -} - -const { languages, defaultLanguage } = readI18n(); -// @mdx-js/mdx is fumadocs-mdx's dependency, not this app's, so it is resolved from there: the copy buildMDX uses. -const { createProcessor } = await import( - pathToFileURL(createRequire(import.meta.resolve('fumadocs-mdx')).resolve('@mdx-js/mdx')).href -); -const config = (await import(pathToFileURL(CONFIG).href)).default; -const mdx = await applyMdxPreset({ ...config.mdxOptions, rehypeCodeOptions: false })('bundler'); -const processor = createProcessor({ - outputFormat: 'program', - ...mdx, - remarkPlugins: [remarkInclude, ...(mdx.remarkPlugins ?? [])], - rehypePlugins: [], - format: 'mdx', -}); - -const byLocale = new Map(languages.map((l) => [l, {}])); -for (const path of walk(CONTENT)) { - const file = relative(CONTENT, path).split('\\').join('/'); - const locale = languages.find((l) => l !== defaultLanguage && file.endsWith(`.${l}.mdx`)) ?? defaultLanguage; - const vfile = await processor.process({ - path, - value: frontmatter(readFileSync(path, 'utf8')).content, - data: { frontmatter: {}, _getProcessor: () => processor }, - }); - if (!vfile.data.structuredData) throw new Error(`${file}: remarkStructure wrote no structuredData`); - byLocale.get(locale)[file] = vfile.data.structuredData; -} - -rmSync(OUT, { recursive: true, force: true }); -mkdirSync(OUT, { recursive: true }); -writeFileSync(join(OUT, '.gitignore'), '*\n'); -const banner = '// Generated by scripts/gen-search-data.mjs before every build. Do not edit.\n'; -for (const [locale, pages] of byLocale) { - // A string, not an object literal: the route parses it when it builds the - // index, and the parsed copy is garbage once the index holds the text. - const packed = gzipSync(JSON.stringify(pages), { level: 9 }).toString('base64'); - writeFileSync(join(OUT, `${locale}.ts`), `${banner}export default ${JSON.stringify(packed)};\n`); -} -const loaders = languages.map((l) => ` ${JSON.stringify(l)}: () => import(${JSON.stringify(`./${l}`)}),`).join('\n'); -writeFileSync( - join(OUT, 'index.ts'), - `${banner}/** Each locale's \`structuredData\`, keyed by file path under content/docs, as gzipped JSON in base64. */\nexport const searchData = {\n${loaders}\n} as const;\n`, -); -const sizes = [...byLocale].map(([l, pages]) => `${l} ${Object.keys(pages).length}`).join(', '); -console.log(`✓ search data: wrote .search/ (pages per locale: ${sizes})`); diff --git a/tools/ci-scripts/run-self-tests.mjs b/tools/ci-scripts/run-self-tests.mjs index fb1f15f6..ab995a5b 100644 --- a/tools/ci-scripts/run-self-tests.mjs +++ b/tools/ci-scripts/run-self-tests.mjs @@ -121,6 +121,13 @@ * Its fixtures are fake handlers and fake heap readings. They prove that a * failed request, a reading above the budget, and a heap still growing on the * second pass each go red. + * + * `check-worker-page-cost.mjs` is listed for its `--self-test` only (#395). Its + * gate mode reads the packaged Worker's metafile, so it runs inside `ci.yml`'s + * `Worker bundle fits the size budget` step. Its fixtures are fake metafiles: + * they prove the per-page cost and the pages left are computed from the + * costliest locale, and that a missing metafile or renamed page chunks fail + * rather than print nothing. */ import { readFileSync, readdirSync, existsSync } from 'node:fs'; @@ -145,6 +152,7 @@ const SELF_TESTED = [ 'check-positioning.mjs', 'check-search-locales.mjs', 'check-search-heap.mjs', + 'check-worker-page-cost.mjs', 'check-doc-samples.mjs', 'bump-upstream-pin.mjs', ];