diff --git a/.github/workflows/llms-txt.yml b/.github/workflows/llms-txt.yml new file mode 100644 index 00000000..a3bc5d27 --- /dev/null +++ b/.github/workflows/llms-txt.yml @@ -0,0 +1,43 @@ +# Builds the site exactly like production (npm run build = generate-llms + docusaurus build) +# and verifies that the LLM/MCP text files (llms.txt, llms-full.txt, /docs/**.txt) +# are present and in sync with the HTML pages. +# +# Production deploys are done by Cloudflare Pages from `master`; this workflow does not +# deploy anything. It only guards against regressions before they reach master. +name: llms.txt parity + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + +jobs: + build-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + # Placeholders only. The Algolia theme refuses to build with empty values. + # Real search keys live in the Cloudflare Pages environment, never in this repo. + ALGOLIA_APP_ID: placeholder + ALGOLIA_API_KEY: placeholder + ALGOLIA_INDEX_NAME: placeholder + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # showLastUpdateTime reads git history + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - run: npm ci + + - name: Build site (same command as production) + run: npm run build + + - name: Verify LLM text files match the built pages + run: npm run verify-llms diff --git a/README.md b/README.md index 733f219f..7329e40d 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,40 @@ yarn serve --- +### 🚀 Deployment + +The live site is built and published by **Cloudflare Pages** (project `capmonstercloud-docs-v2`) directly from this GitHub repository. Every push to `master` triggers a production build. There is no deploy step in this repo — merging to `master` **is** the deploy. + +The Cloudflare build runs `npm run build`, which is: + +1. `node scripts/generate-llms.mjs` — generates the plain-text documentation for AI agents (see below); +2. `docusaurus build` — builds the site into `build/`. + +Search (Algolia) credentials are provided as environment variables in the Cloudflare Pages project settings. **Never commit them to this repository** — it is public. + +The `Dockerfile` is a legacy artifact from a previous hosting setup and is not used by the current deployment. + +### 🤖 Text version for AI agents (llms.txt / MCP) + +Alongside the HTML pages the build publishes plain-text copies of the documentation, used by the [CapMonster Cloud MCP server](https://github.com/CapMonsterCloud/capmonster-mcp-captcha-solver) and other AI agents: + +- `https://docs.capmonster.cloud/llms.txt` — index of all pages with short descriptions; +- `https://docs.capmonster.cloud/llms-full.txt` — the whole documentation in one file; +- `https://docs.capmonster.cloud/docs/.txt` — one file per page, e.g. `/docs/captchas/recaptcha-v3-task/` → `/docs/captchas/recaptcha-v3-task.txt`. + +They are generated from the English MDX sources on every build by `scripts/generate-llms.mjs` (output is git-ignored), so they are always in sync with the deployed pages. Content that lives in React components rather than MDX (``, ``, ``) is rendered to text by that script — if you add a new content-bearing component, teach the script about it too. + +To check locally: + +```bash +npm run build # generates txt files + builds the site +npm run verify-llms # checks that txt files exist and match the built pages +``` + +The same check runs in GitHub Actions (`.github/workflows/llms-txt.yml`) on every pull request. + +--- + ## 📄 License [MIT](./LICENSE) © [CapMonster Cloud](https://capmonster.cloud/en/?utm_source=github&utm_medium=referral&utm_campaign=docs_repo_read) diff --git a/package.json b/package.json index 43a70e39..926c47c6 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start --host 0.0.0.0", "generate-llms": "node scripts/generate-llms.mjs", + "verify-llms": "node scripts/verify-llms.mjs", "build": "node scripts/generate-llms.mjs && docusaurus build", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", diff --git a/scripts/generate-llms.mjs b/scripts/generate-llms.mjs index f9346ef6..b47b1618 100644 --- a/scripts/generate-llms.mjs +++ b/scripts/generate-llms.mjs @@ -62,6 +62,51 @@ const EXCLUDE_PATTERNS = [ /node_modules/, ]; +// Public prices endpoint used by src/hooks/useFetchPrices.ts. Fetched once +// at build time so can be rendered as text. If the request +// fails, the price line is simply omitted - the build never fails on it. +const PRICES_URL = 'https://dash.capmonster.cloud/api/prices?all=true'; +const PRICES_TIMEOUT_MS = 10_000; + +// Text for React components that carry content but have no MDX body. +// Keep in sync with: +// src/components/McpNotice/index.js (mcpTitle / mcpText / mcpLink) +// src/locales/.json (moreBlogInfo, hundred* strings) +const COMPONENT_TEXT = { + en: { + mcpTitle: 'CapMonster Cloud MCP', + mcpText: 'CapMonster Cloud MCP can also be used to detect and solve CAPTCHA.', + mcpLink: 'Learn more about MCP', + moreBlogInfo: 'More on the topic in our blog', + fullPriceText: 'View full price', + pricesPageUrl: 'https://capmonster.cloud/en/prices/', + price: 'Price', + successRate: 'Success rate', + resultType: { + token: '1000 tokens', + image: '1000 images', + dynamic: '1000 dynamic images', + answers: '1000 answers', + }, + }, + ru: { + mcpTitle: 'CapMonster Cloud MCP', + mcpText: 'Для распознавания и решения CAPTCHA также можно использовать CapMonster Cloud MCP.', + mcpLink: 'Подробнее о работе с MCP', + moreBlogInfo: 'Ещё больше по теме в нашем блоге', + fullPriceText: 'Посмотреть полный прайс', + pricesPageUrl: 'https://capmonster.cloud/ru/prices/', + price: 'Цена', + successRate: 'Успешность', + resultType: { + token: '1000 токенов', + image: '1000 картинок', + dynamic: '1000 дин. картинок', + answers: '1000 ответов', + }, + }, +}; + // --- CLI -------------------------------------------------------------- const { values } = parseArgs({ @@ -69,6 +114,7 @@ const { values } = parseArgs({ locale: { type: 'string', default: 'en' }, out: { type: 'string', default: 'static' }, 'site-url': { type: 'string', default: SITE_URL }, + 'no-prices': { type: 'boolean', default: false }, verbose: { type: 'boolean', default: false }, help: { type: 'boolean', short: 'h', default: false }, }, @@ -82,6 +128,7 @@ Options: --locale= Locale to dump (default: en) --out= Output directory (default: static) --site-url= Override site URL + --no-prices Do not fetch prices for (offline / deterministic output) --verbose Print each processed file --help Show this help `); @@ -130,8 +177,14 @@ function parseFrontmatter(raw) { // --- MDX -> MARKDOWN -------------------------------------------------- -/** Strip MDX-specific syntax, keep readable markdown for an LLM. */ -function mdxToMarkdown(content) { +/** + * Strip MDX-specific syntax, keep readable markdown for an LLM. + * + * `ctx` carries what is needed to render content-bearing React components + * as text: { text: COMPONENT_TEXT[locale], prices: Map|null, + * mcpUrl: absolute page URL of the MCP doc (fixDocLinks turns it into .txt) }. + */ +function mdxToMarkdown(content, ctx) { // --- Protect code blocks from being touched by other regexes ------- const stash = []; const STASH = (s) => { @@ -154,13 +207,62 @@ function mdxToMarkdown(content) { (_, attrs) => { const title = (attrs.match(/\btitle=["']([^"']+)["']/) || [])[1] || ''; const type = (attrs.match(/\btype=["']([^"']+)["']/) || [])[1] || ''; - const req = /\brequired\b/.test(attrs); if (!title) return ''; - const meta = [type, req ? 'required' : ''].filter(Boolean).join(', '); + // Mirror src/theme/ParamItem: a string value is printed verbatim + // ("required (if Task is not filled)"), the bare boolean prints + // "required", anything else prints "optional" - unless `response` is set, + // in which case no marker is shown at all. + const reqStr = (attrs.match(/\brequired=["']([^"']*)["']/) || [])[1]; + const reqBool = /\brequired(?:=\{true\})?(?=\s|\/|$)/.test(attrs); + const isResponse = /\bresponse\b/.test(attrs); + let marker = ''; + if (!isResponse) { + if (reqStr !== undefined && reqStr.trim()) marker = reqStr.trim(); + else if (reqBool) marker = 'required'; + else marker = 'optional'; + } + const meta = [type, marker].filter(Boolean).join(', '); return `**${title}**${meta ? ` (${meta})` : ''}:`; } ); + // 2b. -> tip admonition (mirrors src/components/McpNotice). + // Rendered as markdown here so step 3 turns it into a blockquote. + result = result.replace(/]*\/>/g, () => { + const t = ctx.text; + return `\n:::tip ${t.mcpTitle}\n${t.mcpText} [${t.mcpLink}](${ctx.mcpUrl}).\n:::\n`; + }); + + // 2c. -> "X - Price: $N / 1000 tokens. Success rate: N%." + // The label is title ?? item.Name, like the component. Omitted when prices + // could not be fetched (or --no-prices). + result = result.replace(/]*?)\/>/g, (_, attrs) => { + if (!ctx.prices) return ''; + const id = (attrs.match(/\bcaptchaId=["']([^"']+)["']/) || [])[1]; + const item = id && ctx.prices.get(id); + if (!item) return ''; + const t = ctx.text; + const label = (attrs.match(/\btitle=["']([^"']+)["']/) || [])[1] || item.Name || id; + const unit = t.resultType[item.ResultType] || t.resultType.token; + const parts = [`**${t.price}:** $${item.Price} / ${unit}`]; + if (typeof item.SuccessRate === 'number') parts.push(`**${t.successRate}:** ${item.SuccessRate}%`); + return `\n${label} - ${parts.join('. ')}.\n`; + }); + // adds a "View full price" link above the blocks. Skipped + // together with the price lines when prices are unavailable. + result = result.replace( + /]*>([\s\S]*?)<\/PriceBlockWrap>/g, + (_, inner) => (ctx.prices ? `\n${ctx.text.fullPriceText}: ${ctx.text.pricesPageUrl}\n${inner}\n` : `\n${inner}\n`) + ); + + // 2d. -> "More on the topic in our blog: " + result = result.replace(/]*?)\/>/g, (_, attrs) => { + const url = (attrs.match(/\burl=["']([^"']+)["']/) || [])[1]; + if (!url) return ''; + const title = (attrs.match(/\btitle=["']([^"']+)["']/) || [])[1]; + return `\n${ctx.text.moreBlogInfo}: ${title ? `[${title}](${url})` : url}\n`; + }); + // 3. Docusaurus admonitions -> blockquotes // Use [ \t]+ (not \s+) so a bare newline after :::type is not treated as a title. // Allow optional leading whitespace before the closing ::: (some files indent it). @@ -244,7 +346,9 @@ function mdxToMarkdown(content) { // 10. Strip all images - LLMs cannot see them and paths are meaningless in plain text result = result.replace(/!\[[^\]]*\]\([^)]*\)\s*/g, ''); - // 11. Collapse excess blank lines + // 11. Strip trailing whitespace (indented JSX children leave whitespace-only + // lines behind), then collapse excess blank lines. + result = result.replace(/[^\S\n]+$/gm, ''); result = result.replace(/\n{3,}/g, '\n\n'); // --- Restore stashed code blocks ---------------------------------- @@ -258,10 +362,23 @@ function extractH1(content) { return m ? m[1].trim() : null; } +// Lines produced by the component renderers above must never become a page +// description in llms.txt - skip them in firstParagraph(). +const COMPONENT_LINE_RE = (() => { + const all = Object.values(COMPONENT_TEXT); + const alt = (pick) => all.map((t) => pick(t).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'); + return new RegExp( + `^(?:.*\\*\\*(?:${alt((t) => t.price)}):\\*\\*.*|(?:${alt((t) => t.moreBlogInfo)}|${alt((t) => t.fullPriceText)}):.*)$`, + 'gm' + ); +})(); + function firstParagraph(content) { const cleaned = content + .replace(COMPONENT_LINE_RE, '') // price / blog-link lines from components + .replace(/^\s*\d+\.\s+.*$/gm, '') // numbered lists .replace(/```[\s\S]*?```/g, '') // fenced code blocks - .replace(/`[^`\n]+`/g, '') // inline code + .replace(/`([^`\n]+)`/g, '$1') // inline code -> keep its text .replace(/^#{1,6}\s+.+$/gm, '') // headings .replace(/^>\s+.*$/gm, '') // blockquotes .replace(/^\s*[-*+]\s+.*$/gm, '') // lists @@ -289,53 +406,60 @@ function humanize(s) { * * Uses the source file path (not the published URL) for correct relative resolution. */ -function fixDocLinks(content, siteUrl, urlPrefix, sourceFileRel) { - // Treat the source file as a "virtual file" at its published URL path (no trailing slash). - // e.g. "api/methods/create-task.mdx" -> base = "https://.../docs/api/methods/create-task" - // This makes URL resolution behave like the file system (sibling = same directory). - const fileNoExt = sourceFileRel.replace(/\.[^/.]+$/, ''); - const base = `${siteUrl}${urlPrefix}/${fileNoExt}`; - - const NON_DOC_EXT = /\.(js|ts|json|css|png|jpg|gif|svg|zip|pdf|crx|mdx)$/i; - - // 1. Absolute capmonster doc links (trailing slash -> .txt) - content = content.replace( - /\]\((https:\/\/docs\.capmonster\.cloud\/docs\/[^)#\s]*?)\/?(\#[^)]*?)?\)/g, - (_, href, anchor = '') => `](${href.replace(/\/+$/, '')}.txt${anchor})` - ); - - // 2. Relative links -> two sub-passes: - // a) links with .mdx extension (any relative path) - // b) dot-relative links without extension (e.g. ./../captchas/no-captcha-task) +function fixDocLinks(content, { siteUrl, urlPrefix, sourceFileRel, pageUrl, knownUrls }) { const docsBase = `${siteUrl}${urlPrefix}`; - const resolveRel = (match, rel, anchor = '') => { + // Source file as a "virtual file" at its URL path (no trailing slash), so that + // links to .mdx files resolve like the file system (sibling = same directory). + const fileBase = `${docsBase}/${sourceFileRel.replace(/\.[^/.]+$/, '')}`; + const SKIP_EXT = /\.(js|ts|json|css|png|jpe?g|gif|svg|zip|pdf|crx|txt)$/i; + const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + + // Canonical page URL -> ".txt" only when we generate that page. Category + // (generated-index) pages have no .txt, so they keep their page URL. + const toTarget = (absNoSlash, anchor) => { + const page = `${absNoSlash}/`; + return knownUrls.has(page) ? `${absNoSlash}.txt${anchor}` : `${page}${anchor}`; + }; + + // Resolve `rel` against `base`, clamp into this locale's docs prefix and return + // the absolute URL without trailing slash (or null if it leaves the site). + const resolve = (rel, base, isSourceFile) => { try { - let resolved = new URL(rel, base).href - .replace(/\.mdx$/, '') - .replace(/\/+$/, ''); - if (!resolved.startsWith(siteUrl)) return match; - // Clamp: if relative traversal escaped the /docs prefix, re-anchor it. - if (!resolved.startsWith(docsBase)) { - const tail = resolved.slice(siteUrl.length); // e.g. "/captchas/foo" - resolved = `${docsBase}${tail}`; - } - return `](${resolved}.txt${anchor})`; + let abs = new URL(rel, base).href.split('?')[0].replace(/\.mdx?$/i, '').replace(/\/+$/, ''); + if (!abs.startsWith(siteUrl)) return null; + if (!abs.startsWith(docsBase)) abs = `${docsBase}${abs.slice(siteUrl.length)}`; // escaped the prefix + let relPath = abs.slice(docsBase.length).replace(/^\/+/, ''); + if (isSourceFile) relPath = collapseCategoryIndex(relPath); // foo/foo.mdx -> /foo/ + return relPath ? `${docsBase}/${relPath}` : docsBase; } catch { - return match; + return null; } }; - content = content.replace(/\]\(([^)#\s][^)]*?\.mdx)(\#[^)]*?)?\)/g, resolveRel); - - // dot-relative paths with no extension (must start with . to avoid matching bare words) + // 1. Absolute links into this locale's docs (https://docs.capmonster.cloud/docs/...) content = content.replace( - /\]\((\.\.?\/[^)#\s]*?[^.)\s])(\#[^)]*?)?\)/g, - (match, rel, anchor = '') => { - if (/\.(js|ts|json|css|png|jpg|gif|svg|zip|pdf|crx|txt)$/i.test(rel)) return match; - return resolveRel(match, rel, anchor); + new RegExp(`\\]\\((${esc(docsBase)}(?:/[^)#\\s]*)?)(#[^)]*)?\\)`, 'g'), + (m, href, anchor = '') => { + if (SKIP_EXT.test(href)) return m; + const r = resolve(href, `${docsBase}/`, false); + return r ? `](${toTarget(r, anchor)})` : m; } ); + // 2. Relative links to source files (.md/.mdx): relative to the source file. + content = content.replace(/\]\(([^)#\s][^)]*?\.mdx?)(#[^)]*)?\)/gi, (m, rel, anchor = '') => { + const r = resolve(rel, fileBase, true); + return r ? `](${toTarget(r, anchor)})` : m; + }); + + // 3. Dot-relative links without extension (e.g. ../recaptcha-v3-task/): the browser + // resolves these against the page URL, which ends with a slash - so do we. + content = content.replace(/\]\((\.\.?\/[^)#\s]*?[^.)\s])(#[^)]*)?\)/g, (m, rel, anchor = '') => { + if (SKIP_EXT.test(rel)) return m; + const r = resolve(rel, pageUrl, false); + return r ? `](${toTarget(r, anchor)})` : m; + }); + return content; } @@ -378,12 +502,27 @@ function buildUrl(filePath, rootDir, frontmatter) { return `${siteUrl}${cfg.urlPrefix}/${slug}/`; } - // "foo/index" -> "foo" - const cleaned = rel.replace(/\/index$/i, '').replace(/^index$/i, ''); + // Docusaurus "category index" convention: foo/index.md, foo/README.md and + // foo/foo.md all become the URL of the folder itself (/foo/), not /foo/foo/. + // Without this the .txt for e.g. mcp/mcp.mdx would live at /docs/mcp/mcp.txt + // while the page is served at /docs/mcp/. + const cleaned = collapseCategoryIndex(rel); const tail = cleaned ? `/${cleaned}/` : '/'; return `${siteUrl}${cfg.urlPrefix}${tail}`; } +/** "foo/index" | "foo/README" | "foo/foo" -> "foo"; "index" -> "" */ +function collapseCategoryIndex(relNoExt) { + const parts = relNoExt.split('/'); + const last = parts[parts.length - 1]; + const parent = parts.length > 1 ? parts[parts.length - 2] : null; + // Docusaurus compares folder and file names case-insensitively. + if (/^(index|readme)$/i.test(last) || (parent !== null && last.toLowerCase() === parent.toLowerCase())) { + parts.pop(); + } + return parts.join('/'); +} + function getCategory(filePath, rootDir) { const rel = path.relative(rootDir, filePath).replace(/\\/g, '/'); const top = rel.split('/')[0].replace(/\.(md|mdx)$/i, ''); @@ -393,6 +532,29 @@ function getCategory(filePath, rootDir) { : { key: top, label: humanize(top), order: 50 }; } +// --- PRICES ----------------------------------------------------------- + +/** + * Fetch the public price list once (same endpoint the site uses at runtime). + * Returns Map or null on any failure - never throws, so a + * hiccup on the prices API cannot break the documentation build. + */ +async function fetchPrices() { + try { + const res = await fetch(PRICES_URL, { signal: AbortSignal.timeout(PRICES_TIMEOUT_MS) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + const items = Array.isArray(data?.PricesV2) ? data.PricesV2 : []; + if (items.length === 0) throw new Error('empty PricesV2'); + const map = new Map(items.filter((i) => i && i.Id).map((i) => [i.Id, i])); + console.log(`Fetched prices for ${map.size} captcha types`); + return map; + } catch (e) { + console.log(`WARN: prices unavailable (${e.message}); will be omitted from txt`); + return null; + } +} + // --- MAIN ------------------------------------------------------------- async function main() { @@ -410,10 +572,18 @@ async function main() { const files = await walkDocs(rootDir); console.log(`Found ${files.length} MDX/MD files`); + const ctx = { + text: COMPONENT_TEXT[values.locale] || COMPONENT_TEXT.en, + prices: values['no-prices'] ? null : await fetchPrices(), + // page URL on purpose: fixDocLinks() rewrites it to the .txt URL like any other doc link + mcpUrl: `${siteUrl}${cfg.urlPrefix}/mcp/`, + }; + const pages = []; for (const file of files) { try { - const raw = (await fs.readFile(file, 'utf-8')).replace(/^\uFEFF/, ''); + // Strip BOM and normalise CRLF so output is identical on Windows and Linux builds. + const raw = (await fs.readFile(file, 'utf-8')).replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n'); const { data: frontmatter, content } = parseFrontmatter(raw); if (frontmatter.draft === true) { @@ -421,7 +591,7 @@ async function main() { continue; } - const markdown = mdxToMarkdown(content); + const markdown = mdxToMarkdown(content, ctx); const baseName = path.basename(file, path.extname(file)); const title = frontmatter.title || @@ -437,10 +607,11 @@ async function main() { : 999; pages.push({ - file: path.relative(rootDir, file), + file: path.relative(rootDir, file).replace(/\\/g, '/'), url, title, description, category: cat, position, - content: fixDocLinks(markdown, siteUrl, cfg.urlPrefix, path.relative(rootDir, file).replace(/\\/g, '/')), + markdown, + content: null, // filled below, once every page URL is known }); log(` ok ${path.relative(rootDir, file)}`); @@ -454,6 +625,20 @@ async function main() { process.exit(1); } + // Second pass: rewrite internal links. Links to pages we generate point to + // their .txt twin; links to anything else (category pages) stay as page URLs. + const knownUrls = new Set(pages.map((p) => p.url)); + for (const p of pages) { + p.content = fixDocLinks(p.markdown, { + siteUrl, + urlPrefix: cfg.urlPrefix, + sourceFileRel: p.file, + pageUrl: p.url, + knownUrls, + }); + delete p.markdown; + } + // Sort: category order -> sidebar_position -> title pages.sort( (a, b) => diff --git a/scripts/verify-llms.mjs b/scripts/verify-llms.mjs new file mode 100644 index 00000000..1fbdc686 --- /dev/null +++ b/scripts/verify-llms.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node +/** + * scripts/verify-llms.mjs + * + * Post-build check that the LLM text files (llms.txt, llms-full.txt and the + * per-page /docs/**.txt files) are present in the built site and in sync with + * the HTML pages. Run after `npm run build`: + * + * node scripts/verify-llms.mjs # checks ./build for locale en + * node scripts/verify-llms.mjs --build=dist # custom build dir + * + * Exits 1 with a list of problems when something is off. Zero dependencies. + * + * Checks: + * 1. llms.txt and llms-full.txt exist in the build output and are not tiny. + * 2. Every .txt link in llms.txt points to a file that exists in the build. + * 3. Every per-page .txt has a "URL:" header whose page exists in the build + * (catches URL-mapping bugs such as /docs/mcp/mcp/ vs /docs/mcp/). + * 4. The number of per-page .txt files equals the number of publishable + * source docs (non-draft .md/.mdx outside "_" folders). + * 5. No per-page .txt still contains raw MDX: import lines, unstripped JSX + * components, admonition markers or MDX comments. + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; + +const SITE_URL = 'https://docs.capmonster.cloud'; +const LOCALES = { + en: { docsDir: 'i18n/en/docusaurus-plugin-content-docs/current', urlPrefix: '/docs' }, + ru: { docsDir: 'docs', urlPrefix: '/ru/docs' }, +}; + +const { values } = parseArgs({ + options: { + build: { type: 'string', default: 'build' }, + locale: { type: 'string', default: 'en' }, + }, +}); + +const cfg = LOCALES[values.locale]; +if (!cfg) { + console.error(`Unknown locale: ${values.locale}`); + process.exit(1); +} + +const buildDir = path.resolve(values.build); +const problems = []; +const warnings = []; +const fail = (msg) => problems.push(msg); +const warn = (msg) => warnings.push(msg); +const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +// Markdown links into this locale's docs: ](https://docs.capmonster.cloud/docs/...) +const DOC_LINK_RE = new RegExp(`\\]\\((${escapeRe(SITE_URL)}(${escapeRe(cfg.urlPrefix)}(?:/[^)#?\\s]*)?))(?:[#?][^)]*)?\\)`, 'g'); +// A rendered line, e.g. "$0.9 / 1000 tokens" +const PRICE_LINE_RE = /\$\d+(?:\.\d+)? \/ 1000 /; + +/** Check that a doc link target exists in the build: foo.txt as a file, /foo/ as foo/index.html. */ +async function checkDocLink(urlPath, where) { + // Non-page assets under the docs prefix (a PDF, an image) are not pages; skip them. + if (/\.[a-z0-9]+$/i.test(urlPath) && !urlPath.endsWith('.txt')) return; + if (urlPath.endsWith('.txt')) { + if (!(await exists(path.join(buildDir, urlPath)))) fail(`${where}: links to ${urlPath} but that file is not in the build`); + return; + } + const html = path.join(buildDir, urlPath.replace(/\/+$/, ''), 'index.html'); + if (!(await exists(html))) fail(`${where}: links to page ${urlPath} which is not in the build`); +} + +async function exists(p) { + try { await fs.access(p); return true; } catch { return false; } +} + +async function walk(dir, pred) { + const out = []; + for (const e of await fs.readdir(dir, { withFileTypes: true })) { + const full = path.join(dir, e.name); + if (e.isDirectory()) out.push(...await walk(full, pred)); + else if (pred(full)) out.push(full); + } + return out; +} + +async function countSourceDocs(rootDir) { + const files = await walk(rootDir, (f) => /\.(md|mdx)$/i.test(f)); + let n = 0; + for (const f of files) { + const rel = path.relative(rootDir, f).replace(/\\/g, '/'); + if (/(^|\/)_/.test(rel)) continue; + // Strip BOM like the generator does - 17 English sources carry one. + const raw = (await fs.readFile(f, 'utf-8')).replace(/^\uFEFF/, ''); + if (/^---\r?\n[\s\S]*?^draft:\s*true\s*$[\s\S]*?\r?\n---/m.test(raw)) continue; + n++; + } + return n; +} + +async function main() { + if (!(await exists(buildDir))) { + console.error(`Build directory not found: ${buildDir}. Run "npm run build" first.`); + process.exit(1); + } + + // 1. index files + const llmsPath = path.join(buildDir, 'llms.txt'); + const fullPath = path.join(buildDir, 'llms-full.txt'); + for (const [p, min] of [[llmsPath, 2_000], [fullPath, 50_000]]) { + if (!(await exists(p))) { fail(`missing ${path.relative(buildDir, p)}`); continue; } + const size = (await fs.stat(p)).size; + if (size < min) fail(`${path.relative(buildDir, p)} is suspiciously small (${size} B < ${min} B)`); + } + + // 2. every link in llms.txt resolves to a built file + let linked = 0; + if (await exists(llmsPath)) { + const llms = await fs.readFile(llmsPath, 'utf-8'); + for (const m of llms.matchAll(DOC_LINK_RE)) { + linked++; + if (!m[2].endsWith('.txt')) fail(`llms.txt links to a page URL instead of a .txt file: ${m[2]}`); + await checkDocLink(m[2], 'llms.txt'); + } + if (linked === 0) fail('llms.txt contains no .txt links'); + } + + // 3. each per-page txt maps to an existing HTML page + const pagesDir = path.join(buildDir, cfg.urlPrefix.replace(/^\//, '')); + const txtFiles = (await exists(pagesDir)) ? await walk(pagesDir, (f) => f.endsWith('.txt')) : []; + if (txtFiles.length === 0) fail(`no per-page .txt files under ${path.relative(buildDir, pagesDir)}`); + let bodyLinks = 0; + let pagesWithPrice = 0; + + for (const f of txtFiles) { + const rel = path.relative(buildDir, f).replace(/\\/g, '/'); + const txt = await fs.readFile(f, 'utf-8'); + + const urlLine = txt.match(/^URL: (\S+)\s*$/m); + if (!urlLine) { fail(`${rel}: no "URL:" header`); continue; } + const url = urlLine[1]; + if (!url.startsWith(SITE_URL)) { fail(`${rel}: URL does not start with ${SITE_URL}: ${url}`); continue; } + const urlPath = url.slice(SITE_URL.length).replace(/\/+$/, ''); + const html = path.join(buildDir, urlPath, 'index.html'); + if (!(await exists(html))) fail(`${rel}: header URL ${url} has no page in the build (expected ${urlPath}/index.html)`); + + // the txt should sit next to the page: /docs/foo/ -> /docs/foo.txt + const expectedTxt = `${urlPath}.txt`.replace(/^\//, ''); + if (expectedTxt !== rel) fail(`${rel}: expected file name ${expectedTxt} for URL ${url}`); + + // 2b. in-body links into the docs must resolve (both .txt twins and page URLs) + for (const m of txt.matchAll(DOC_LINK_RE)) { + bodyLinks++; + await checkDocLink(m[2], rel); + } + + if (PRICE_LINE_RE.test(txt)) pagesWithPrice++; + + // 5. raw MDX leftovers + const body = txt.replace(/```[\s\S]*?```/g, '').replace(/`[^`\n]+`/g, ''); + const leftovers = [ + [/^import\s+.+\s+from\s+['"]/m, 'import statement'], + [/<[A-Z][A-Za-z.]*(\s[^>]*)?\/?>/, 'JSX component tag'], + [/^[ \t]*:::/m, 'admonition marker'], + [/\{\/\*[\s\S]*?\*\/\}/, 'MDX comment'], + ]; + for (const [re, what] of leftovers) { + const m = body.match(re); + if (m) fail(`${rel}: raw MDX leftover (${what}): ${m[0].slice(0, 80).replace(/\s+/g, ' ')}`); + } + } + + // 4. count parity with source docs + const srcCount = await countSourceDocs(path.resolve(cfg.docsDir)); + if (srcCount !== txtFiles.length) + fail(`page count mismatch: ${srcCount} publishable source docs vs ${txtFiles.length} per-page .txt files`); + + // Prices are baked in at build time from a live endpoint. A hiccup there + // must not fail the build, but it should not go unnoticed either. + if (txtFiles.length && pagesWithPrice === 0) + warn('no per-page .txt contains a price line - the prices API was probably unavailable during the build'); + + // --- report + console.log( + `Checked ${txtFiles.length} per-page .txt files, ${linked} llms.txt links, ${bodyLinks} in-body links, ` + + `${srcCount} source docs, ${pagesWithPrice} pages with prices` + ); + for (const w of warnings) console.warn(`WARN: ${w}`); + if (problems.length) { + console.error(`\n${problems.length} problem(s):`); + for (const p of problems) console.error(` - ${p}`); + process.exit(1); + } + console.log('OK: LLM text files are present and in sync with the built pages'); +} + +main().catch((e) => { + console.error('Fatal error:', e); + process.exit(1); +});