fix(worker): stop bundling every docs page twice (#395)

.github/scripts/check-worker-page-cost.mjs

@@ -0,0 +1,214 @@
+#!/usr/bin/env node
+/**
+ * The Worker's cost per docs page, and how many more pages the budget holds
+ * (#395). It runs in `Worker bundle fits the size budget`, after the size
+ * itself is measured.
+ *
+ * #390's eleven ja pages and #388's PR 3 showed how a content batch finds out
+ * it is too big: the budget step goes red after the batch is written. A
+ * translated page costs its compiled module, its search text and a little
+ * routing data, and nothing said how much up front. So this prints, from the
+ * bundle just packaged:
+ *
+ *   - the KiB each locale's page modules and search text take, per page;
+ *   - what one more translated page costs: the most expensive locale's
+ *     module and search text per page, plus `OVERHEAD_KIB`;
+ *   - how many more such pages fit under the budget;
+ *   - what one more fully translated locale (one page per English page) would
+ *     cost, and whether it fits.
+ *
+ * `OVERHEAD_KIB` is what a page adds outside its module and search text:
+ * frontmatter, the route manifests, the prerender entry. Measured on #395:
+ * filling ja from 43 to 98 pages added 4,791 KiB of upload, of which 4,019
+ * were page modules and 238 search text, so 534 KiB over 55 pages, about 10
+ * KiB each. The projection is within about 2% of that measured growth.
+ *
+ * It reads the esbuild metafile OpenNext writes next to the bundled handler
+ * (`handler.mjs.meta.json`). Turbopack names a page's chunk after its file,
+ * `content_docs_<path>_<locale>_mdx_<hash>.js`, and an English page's chunk
+ * has no locale part, so `bytesInOutput` per chunk is the bytes each page puts
+ * in the upload. A chunk under `content_docs_` that names no `_mdx_` file is
+ * shared between pages and is counted as such, not as a page. A locale's
+ * search text is the chunk of `.route-data/search-<locale>.ts`.
+ *
+ * The projection is a mean, not a promise: a page with long code samples costs
+ * more. It never fails the step on a number; the budget check above does that.
+ * It fails when it cannot read what it projects from (`unreadable`,
+ * `no-pages`), because a projection that silently prints nothing is the
+ * state #395 exists to end.
+ *
+ * Usage: `check-worker-page-cost.mjs --meta <handler.mjs.meta.json> --size <KiB>
+ * --budget <KiB>`, from anywhere. `--self-test` runs every rule against fake
+ * metafiles and needs no build.
+ */
+import { readFileSync } from 'node:fs';
+import { basename, dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+const KIB = 1024;
+const OVERHEAD_KIB = 10; // per page, outside its module and search text (header)
+const RULES = ['unreadable', 'no-pages'];
+
+/** `languages` and `defaultLanguage` out of `i18n.ts`. Throws rather than guessing. */
+function readI18n(text) {
+  const languages = /languages:\s*\[([^\]]+)\]/.exec(text)?.[1]?.split(',').map((s) => s.trim().replace(/['"]/g, '')).filter(Boolean) ?? [];
+  const defaultLanguage = /defaultLanguage:\s*['"]([^'"]+)['"]/.exec(text)?.[1];
+  if (!languages.length || !languages.includes(defaultLanguage)) {
+    throw new Error('could not read languages[] and defaultLanguage out of apps/docs/lib/i18n.ts');
+  }
+  return { languages, defaultLanguage };
+}
+
+/** Per-locale page bytes out of a parsed metafile. */
+function pageBytes(meta, { languages, defaultLanguage }) {
+  const locales = new Map(languages.map((l) => [l, { pages: 0, bytes: 0, search: 0 }]));
+  let shared = 0;
+  for (const output of Object.values(meta?.outputs ?? {})) {
+    for (const [path, input] of Object.entries(output.inputs ?? {})) {
+      const name = basename(path);
+      const search = languages.find((l) => name.includes(`route-data_search-${l}_ts`));
+      if (search) locales.get(search).search += input.bytesInOutput;
+      if (!name.startsWith('content_docs_')) continue;
+      if (!name.includes('_mdx_')) {
+        shared += input.bytesInOutput;
+        continue;
+      }
+      const locale = languages.find((l) => l !== defaultLanguage && name.includes(`_${l}_mdx_`)) ?? defaultLanguage;
+      const row = locales.get(locale);
+      row.pages += 1;
+      row.bytes += input.bytesInOutput;
+    }
+  }
+  return { locales, shared };
+}
+
+/** The summary, as Markdown lines, plus findings. */
+function project({ meta, size, budget, i18n }) {
+  const findings = [];
+  if (!meta || !Number.isFinite(size) || !Number.isFinite(budget)) {
+    findings.push({ rule: 'unreadable', detail: 'no metafile, size or budget to project from' });
+    return { findings, lines: [] };
+  }
+  const { locales, shared } = pageBytes(meta, i18n);
+  const english = locales.get(i18n.defaultLanguage);
+  const translated = [...locales].filter(([l, r]) => l !== i18n.defaultLanguage && r.pages > 0);
+  if (!english.pages) {
+    findings.push({ rule: 'no-pages', detail: 'the metafile names no `content_docs_*_mdx_*` chunk, so the page modules were not found' });
+    return { findings, lines: [] };
+  }
+  // A translated page is what the next batch adds; English stands in when no locale has one yet.
+  // The most expensive locale sets the cost, so the projection errs towards less room.
+  const cost = (r) => (r.bytes + r.search) / r.pages / KIB;
+  const [costliest, worst] = (translated.length ? translated : [[i18n.defaultLanguage, english]]).reduce((a, b) => (cost(b[1]) > cost(a[1]) ? b : a));
+  const perPage = cost(worst) + OVERHEAD_KIB;
+  const headroom = budget - size;
+  const left = Math.max(0, Math.floor(headroom / perPage));
+  const locale = english.pages * perPage;
+  const lines = [
+    '#### Per-page cost',
+    '',
+    '| locale | page modules | module KiB | search KiB | KiB per page |',
+    '|:--|--:|--:|--:|--:|',
+    ...[...locales]
+      .filter(([, r]) => r.pages)
+      .map(([l, r]) => `| ${l} | ${r.pages} | ${(r.bytes / KIB).toFixed(2)} | ${(r.search / KIB).toFixed(2)} | ${cost(r).toFixed(2)} |`),
+    `| shared | — | ${(shared / KIB).toFixed(2)} | — | — |`,
+    '',
+    `A translated page costs about **${perPage.toFixed(2)} KiB**: ${costliest}'s ${cost(worst).toFixed(2)} KiB per page, the most of any locale, plus ${OVERHEAD_KIB} KiB of routing data.`,
+    `The budget holds **${left}** more such pages (${headroom.toFixed(2)} KiB of headroom).`,
+    `One more fully translated locale (${english.pages} pages) would add about ${locale.toFixed(0)} KiB: ${
+      locale <= headroom ? `it fits, leaving ${(headroom - locale).toFixed(0)} KiB` : `it does NOT fit, by ${(locale - headroom).toFixed(0)} KiB`
+    }.`,
+  ];
+  return { findings, lines };
+}
+
+function arg(name) {
+  const i = process.argv.indexOf(name);
+  return i < 0 ? undefined : process.argv[i + 1];
+}
+
+function gate() {
+  const i18n = readI18n(readFileSync(join(ROOT, 'apps/docs/lib/i18n.ts'), 'utf8'));
+  let meta;
+  try {
+    meta = JSON.parse(readFileSync(arg('--meta') ?? '', 'utf8'));
+  } catch {
+    meta = undefined;
+  }
+  const { findings, lines } = project({ meta, size: Number(arg('--size')), budget: Number(arg('--budget')), i18n });
+  if (lines.length) console.log(lines.join('\n'));
+  for (const f of findings) console.error(`  (${f.rule}) ${f.detail}`);
+  if (findings.length) {
+    console.error(`\n✗ Worker page cost: NOT MEASURED. The rules are in this script's header.`);
+    return 1;
+  }
+  return 0;
+}
+
+/* Self-test: fake metafiles, each case checked for its findings and, where it projects, its numbers. */
+const I18N = { languages: ['en', 'ja', 'de'], defaultLanguage: 'en' };
+const chunk = (name, kib) => [`.next/server/chunks/ssr/${name}`, { bytesInOutput: kib * KIB }];
+const metafile = (...chunks) => ({ outputs: { 'handler.mjs': { inputs: Object.fromEntries([chunk('next_dist_x.js', 900), ...chunks]) } } });
+const CASES = [
+  [
+    'ja pages set the per-page cost; en and shared are listed, not averaged',
+    { meta: metafile(chunk('content_docs_a_mdx_1.js', 50), chunk('content_docs_b_mdx_2.js', 70), chunk('content_docs_a_ja_mdx_3.js', 100), chunk('content_docs_x._.js', 30)), size: 1000, budget: 1450, i18n: I18N },
+    [],
+    ['**110.00 KiB**', 'ja\'s 100.00', '**4** more', '2 pages) would add about 220 KiB: it fits, leaving 230 KiB', '| shared | — | 30.00 |'],
+  ],
+  [
+    'no translation yet: English sets the cost',
+    { meta: metafile(chunk('content_docs_a_mdx_1.js', 40), chunk('content_docs_b_mdx_2.js', 60)), size: 1000, budget: 1100, i18n: I18N },
+    [],
+    ['**60.00 KiB**', '**1** more', 'about 120 KiB: it does NOT fit, by 20 KiB'],
+  ],
+  [
+    'over the budget: no page left, and a locale does not fit',
+    { meta: metafile(chunk('content_docs_a_mdx_1.js', 40), chunk('content_docs_a_de_mdx_9.js', 80)), size: 1200, budget: 1100, i18n: I18N },
+    [],
+    ['**0** more', 'it does NOT fit, by 190 KiB'],
+  ],
+  [
+    'search text counts, and the costliest locale sets the cost',
+    { meta: metafile(chunk('content_docs_a_mdx_1.js', 40), chunk('content_docs_a_ja_mdx_3.js', 60), chunk('content_docs_b_ja_mdx_4.js', 60), chunk('apps_docs__route-data_search-ja_ts_0xv.js', 20), chunk('content_docs_a_de_mdx_9.js', 50)), size: 1000, budget: 1400, i18n: I18N },
+    [],
+    ['| ja | 2 | 120.00 | 20.00 | 70.00 |', 'about **80.00 KiB**: ja\'s 70.00', '**5** more'],
+  ],
+  ['no metafile', { meta: undefined, size: 1, budget: 2, i18n: I18N }, ['unreadable'], []],
+  ['no size', { meta: metafile(chunk('content_docs_a_mdx_1.js', 40)), size: NaN, budget: 2, i18n: I18N }, ['unreadable'], []],
+  ['the chunks were renamed', { meta: metafile(chunk('pages_a.js', 40)), size: 1, budget: 2, i18n: I18N }, ['no-pages'], []],
+];
+
+function selfTest() {
+  let failed = 0;
+  const fired = new Set();
+  for (const [name, input, wantRules, wantText] of CASES) {
+    const { findings, lines } = project(input);
+    const got = findings.map((f) => f.rule).sort();
+    got.forEach((r) => fired.add(r));
+    const text = lines.join('\n');
+    const missing = wantText.filter((t) => !text.includes(t));
+    const pass = JSON.stringify(got) === JSON.stringify([...wantRules].sort()) && !missing.length;
+    if (!pass) failed += 1;
+    console.log(`${pass ? '✓' : '✗'} ${name}: ${got.length ? got.join(', ') : 'no findings'}${missing.length ? ` — missing ${JSON.stringify(missing)}` : ''}`);
+  }
+  const silent = RULES.filter((r) => !fired.has(r));
+  if (silent.length) {
+    failed += 1;
+    console.log(`✗ rules no fixture makes fire: ${silent.join(', ')}`);
+  }
+  let unreadable = false;
+  try {
+    readI18n('defineI18n({})');
+  } catch {
+    unreadable = true;
+  }
+  if (!unreadable) failed += 1;
+  console.log(`${unreadable ? '✓' : '✗'} an unreadable i18n.ts is rejected`);
+  console.log(failed ? `\n✗ Worker page cost self-test: ${failed} failure(s)` : `\n✓ Worker page cost self-test: ${CASES.length} cases, all ${RULES.length} rules can fire`);
+  return failed ? 1 : 0;
+}
+
+process.exitCode = process.argv.includes('--self-test') ? selfTest() : gate();

.github/workflows/ci.yml

@@ -409,6 +409,7 @@ jobs:
             exit 1
           fi
 
+          VERDICT=0
           awk -v size="$SIZE_KIB" -v budget="$WORKER_BUDGET_KIB" -v limit="$WORKER_LIMIT_KIB" '
             BEGIN {
               over = (size > budget)
@@ -426,7 +427,18 @@ jobs:
                 printf "one of them. Reduce the bundle, or raise the budget deliberately and say why.\n"
               }
               exit (over ? 1 : 0)
-            }' | tee -a "$GITHUB_STEP_SUMMARY"
+            }' | tee -a "$GITHUB_STEP_SUMMARY" || VERDICT=$?
+
+          # #395: what the next content batch has room for, printed whether or
+          # not this one fits. The KiB a translated page costs, read from the
+          # bundle just packaged, and how many more pages and locales that
+          # leaves under the budget. It goes red only when it cannot read the
+          # bundle; the verdict above is the budget gate.
+          echo | tee -a "$GITHUB_STEP_SUMMARY"
+          node "$GITHUB_WORKSPACE/.github/scripts/check-worker-page-cost.mjs" \
+            --meta .open-next/server-functions/default/apps/docs/handler.mjs.meta.json \
+            --size "$SIZE_KIB" --budget "$WORKER_BUDGET_KIB" | tee -a "$GITHUB_STEP_SUMMARY"
+          exit "$VERDICT"
 
       # `include-hidden-files` is load-bearing, not tidiness: the compiled
       # OpenNext config the deploy reads lives at `.open-next/.build/`, and

apps/docs/app/[lang]/docs/[[...slug]]/page.tsx

@@ -17,6 +17,7 @@ import type { InferPageType } from 'fumadocs-core/source';
 import type { TOCItemType } from 'fumadocs-core/toc';
 import { EditionBadges } from '@/components/editions';
 import { EDITIONS_PAGE_SLUGS } from '@/lib/editions';
+import { loadPageBody } from '@/lib/page-body';
 
 /**
  * The page type the loader actually produces, frontmatter schema included.
@@ -267,7 +268,11 @@ export default async function Page(props: {
   // Under `dynamicParams = false` this await runs at BUILD time, in Node, on
   // every one of the 1139 prerendered paths — the Worker serves the prerendered
   // result out of the static-assets incremental cache and does not re-render.
-  const loaded = await page.data.load();
+  //
+  // Loaded through `lib/page-body.ts`, not `page.data.load()`: `source` is
+  // built without the compiled bodies so the route handlers do not bundle a
+  // second copy of every page (#395).
+  const loaded = await loadPageBody(page);
   const MDX = loaded.body;
 
   // Resolved once and handed to both controls (through `copySource` and
@@ -309,11 +314,11 @@ export default async function Page(props: {
   // translation it is this locale's text, which the `.mdx` surface does not
   // serve (English-only, see `markdownUrl`), so it is rendered into the page
   // instead: `getLLMText` builds it exactly as `/llms.mdx/docs/...` builds the
-  // English body, from the module `page.data.load()` above already loaded.
+  // English body, from the module `loadPageBody` above already loaded.
   const copySource: MarkdownSource =
     contentLang === i18n.defaultLanguage
       ? { url: pageMarkdownUrl }
-      : { text: await getLLMText(page) };
+      : { text: await getLLMText(page, loaded._exports._markdown as string) };
 
   // "Open in ChatGPT / Claude" follows the same rule (#305): the assistant is
   // sent to the page on screen. The English Markdown when that is the English

apps/docs/app/api/search/route.ts

@@ -1,6 +1,7 @@
 import { source } from '@/lib/source';
 import { i18n } from '@/lib/i18n';
-import { searchData } from '@/.search';
+import { unpack } from '@/lib/packed';
+import { searchData } from '@/.route-data/search';
 import {
   createSearchAPI,
   type AdvancedIndex,
@@ -123,9 +124,9 @@ type StructuredData = AdvancedIndex['structuredData'];
 
 /**
  * One locale's `structuredData` — each page's headings and paragraphs — keyed
- * by file path under `content/docs`, from `.search/` (#393).
+ * by file path under `content/docs`, from `.route-data/` (#393).
  *
- * `scripts/gen-search-data.mjs` writes it before every build, with the same
+ * `scripts/gen-route-data.mjs` writes it before every build, with the same
  * remark chain fumadocs-mdx compiles the pages with. Reading it from there
  * instead of from `page.data.load()` is the point: `load()` imports the page's
  * compiled module, the module registry keeps it for the life of the isolate,
@@ -134,11 +135,7 @@ type StructuredData = AdvancedIndex['structuredData'];
  * dropped once the index has the text.
  */
 async function texts(locale: Locale): Promise<Record<string, StructuredData>> {
-  // Gzipped JSON in base64, because the module is Worker upload too.
-  const packed = (await searchData[locale]()).default;
-  const bytes = Uint8Array.from(atob(packed), (c) => c.charCodeAt(0));
-  const json = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'));
-  return JSON.parse(await new Response(json).text());
+  return unpack((await searchData[locale]()).default);
 }
 
 /**
@@ -156,10 +153,10 @@ async function indexes(locale: Locale): Promise<AdvancedIndex[]> {
       const file = page.data.info.path;
       let structuredData = own[file] ?? fallback?.[file];
       if (!structuredData) {
-        // `next dev` serves pages added since it wrote `.search/`; a build
-        // writes it first, so there a missing page is a stale `.search/`.
+        // `next dev` serves pages added since it wrote `.route-data/`; a build
+        // writes it first, so there a missing page is a stale `.route-data/`.
         if (process.env.NODE_ENV !== 'development') {
-          throw new Error(`/api/search: no search data for ${file}; run scripts/gen-search-data.mjs`);
+          throw new Error(`/api/search: no search data for ${file}; run scripts/gen-route-data.mjs`);
         }
         structuredData = (await page.data.load()).structuredData;
       }

apps/docs/lib/packed.ts

@@ -0,0 +1,12 @@
+/**
+ * Reads a value `scripts/gen-route-data.mjs` packed: JSON, gzipped, in base64.
+ *
+ * The generated modules are Worker upload as well as text the route handlers
+ * read, so they are stored compressed (#393). `atob`, `Blob` and
+ * `DecompressionStream` are in both Node and the Workers runtime.
+ */
+export async function unpack<T>(packed: string): Promise<T> {
+  const bytes = Uint8Array.from(atob(packed), (c) => c.charCodeAt(0));
+  const json = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('gzip'));
+  return JSON.parse(await new Response(json).text());
+}

apps/docs/lib/page-body.ts

@@ -0,0 +1,22 @@
+import { docs } from 'fumadocs-mdx:collections/server';
+import type { InferPageType } from 'fumadocs-core/source';
+import type { source } from '@/lib/source';
+
+/**
+ * Each page's entry in fumadocs-mdx's full collection, by file path under
+ * `content/docs`.
+ *
+ * This is the one module that imports the full collection, with its lazy
+ * import of every compiled page (#395). Import it from the page route only:
+ * anything a route handler imports puts every compiled page in the route
+ * handlers' module graph, a second copy of them in the Worker upload, which is
+ * what `lib/source.ts` is built to avoid.
+ */
+const entries = new Map(docs.docs.map((entry) => [entry.info.path, entry]));
+
+/** `page`'s compiled body, toc and exports, as `page.data.load()` returns them on the full collection. */
+export async function loadPageBody(page: InferPageType<typeof source>) {
+  const entry = entries.get(page.data.info.path);
+  if (!entry) throw new Error(`${page.data.info.path}: not in the fumadocs-mdx collection`);
+  return entry.load();
+}