-
Notifications
You must be signed in to change notification settings - Fork 49
Enhance LLM export with llms.txt index and copyToLLM styling #336
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,6 +8,8 @@ const SNIPPET_RE = /^<<<\s*(.+)$/gm; | |
| interface Options { | ||
| srcDir: string; | ||
| outDir?: string; | ||
| title?: string; | ||
| description?: string; | ||
| } | ||
|
|
||
| export function llmMarkdownPlugin(options: Options): Plugin { | ||
|
|
@@ -45,26 +47,75 @@ export function llmMarkdownPlugin(options: Options): Plugin { | |
| }, | ||
| async closeBundle() { | ||
| if (!outDir) return; | ||
| await generateLlmMarkdown(options.srcDir, path.join(outDir, "llms")); | ||
| await generateLlmMarkdown(options.srcDir, outDir, { | ||
| title: options.title ?? "", | ||
| description: options.description ?? "", | ||
| }); | ||
| }, | ||
| }; | ||
| } | ||
|
|
||
| export async function generateLlmMarkdown( | ||
| srcDir: string, | ||
| outDir: string, | ||
| distDir: string, | ||
| meta: { title: string; description: string } = { title: "", description: "" }, | ||
| ): Promise<void> { | ||
| const files = await collectMarkdownFiles(srcDir); | ||
| await fs.rm(outDir, { recursive: true, force: true }); | ||
| // ponytail: 按路径排序,编号目录即正确顺序;要严格 sidebar 顺序再读 themeConfig | ||
| const files = (await collectMarkdownFiles(srcDir)).sort(); | ||
| const llmsDir = path.join(distDir, "llms"); | ||
| await fs.rm(llmsDir, { recursive: true, force: true }); | ||
|
|
||
| await Promise.all( | ||
| const pages = await Promise.all( | ||
| files.map(async (file) => { | ||
| const relativePath = normalizePath(path.relative(srcDir, file)); | ||
| const output = path.join(outDir, relativePath); | ||
| const markdown = await renderLlmMarkdown(file, srcDir); | ||
| const output = path.join(llmsDir, relativePath); | ||
| await fs.mkdir(path.dirname(output), { recursive: true }); | ||
| await fs.writeFile(output, await renderLlmMarkdown(file, srcDir), "utf8"); | ||
| await fs.writeFile(output, markdown, "utf8"); | ||
| return { relativePath, markdown }; | ||
| }), | ||
| ); | ||
|
|
||
| await writeLlmsIndex(distDir, pages, meta); | ||
| } | ||
|
|
||
| async function writeLlmsIndex( | ||
| distDir: string, | ||
| pages: { relativePath: string; markdown: string }[], | ||
| meta: { title: string; description: string }, | ||
| ): Promise<void> { | ||
| const header = `# ${meta.title}\n${meta.description ? `\n> ${meta.description}\n` : ""}`; | ||
|
|
||
| // llms.txt:llmstxt.org 标准索引,逐页链接指向 .md 原文 | ||
| const links = pages | ||
| .map((page) => { | ||
| const title = firstHeading(page.markdown) ?? page.relativePath; | ||
| return `- [${title}](${ORIGIN}/llms/${page.relativePath})`; | ||
| }) | ||
| .join("\n"); | ||
| await fs.writeFile( | ||
| path.join(distDir, "llms.txt"), | ||
| `${header}\n## 文档\n\n${links}\n`, | ||
| "utf8", | ||
| ); | ||
|
|
||
| // llms-full.txt:全站正文拼接成单文件 | ||
| const body = pages.map((page) => page.markdown.trim()).join("\n\n---\n\n"); | ||
| await fs.writeFile( | ||
| path.join(distDir, "llms-full.txt"), | ||
| `${header}\n${body}\n`, | ||
| "utf8", | ||
| ); | ||
| } | ||
|
|
||
| function firstHeading(markdown: string): string | null { | ||
| // 去掉 VitePress 自定义锚点 {#id},标题更干净 | ||
| return ( | ||
| markdown | ||
| .match(/^#\s+(.+)$/m)?.[1] | ||
| .replace(/\s*\{#[\w-]+\}\s*$/, "") | ||
| .trim() ?? null | ||
| ); | ||
| } | ||
|
Comment on lines
+111
to
119
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. VitePress headings often contain custom anchors like function firstHeading(markdown: string): string | null {
const match = markdown.match(/^#\\s+(.+)$/m);
if (!match) return null;
return match[1].replace(/\\{#[\\w-]+\\\}\\s*$/, "").trim();
} |
||
|
|
||
| export async function renderLlmMarkdown( | ||
|
|
@@ -76,6 +127,7 @@ export async function renderLlmMarkdown( | |
| let markdown = await fs.readFile(file, "utf8"); | ||
|
|
||
| markdown = stripFrontmatter(markdown); | ||
| markdown = unwrapLlmOnly(markdown); | ||
| markdown = await expandSnippets(markdown, srcDir); | ||
| markdown = flattenVitePressContainers(markdown); | ||
| markdown = absolutizeLinks(markdown, relativePath, origin); | ||
|
|
@@ -239,6 +291,11 @@ function stripFrontmatter(markdown: string): string { | |
| return markdown.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, ""); | ||
| } | ||
|
|
||
| // 去掉 <llm-only> 标记,保留其中内容(网页端由 CSS 隐藏,这里让正文进入 LLM 输出) | ||
| function unwrapLlmOnly(markdown: string): string { | ||
| return markdown.replace(/<\/?llm-only(?:\s+[^>]*)?>/gi, ""); | ||
| } | ||
|
Comment on lines
+295
to
+297
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If the function unwrapLlmOnly(markdown: string): string {
return markdown.replace(/<\\/?llm-only(?:\\s+[^>]*)?>/gi, "");
} |
||
|
|
||
| function languageFromFile(filePath: string): string { | ||
| const basename = path.basename(filePath); | ||
| if (basename === "build.zig.zon") return "zig"; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| /* 只给 LLM 看:<llm-only> 的内容网页隐藏,仅进入 /llms、llms.txt、llms-full.txt */ | ||
| llm-only { | ||
| display: none; | ||
| } | ||
|
|
||
| .copy_llm { | ||
| display: flex; | ||
| align-items: center; | ||
| gap: 18px; | ||
| margin: 12px 0 24px; | ||
| } | ||
|
|
||
| .copy_llm-wrapper { | ||
| position: relative; | ||
| display: inline-block; | ||
| } | ||
|
|
||
| .copy_llm-trigger { | ||
| display: inline-flex; | ||
| align-items: center; | ||
| gap: 8px; | ||
| padding: 0; | ||
| border: 0; | ||
| background: transparent; | ||
| color: var(--vp-c-text-2); | ||
| font-size: 14px; | ||
| font-weight: 500; | ||
| line-height: 24px; | ||
| cursor: pointer; | ||
| transition: color 0.25s; | ||
| } | ||
|
|
||
| .copy_llm-trigger:hover { | ||
| color: var(--vp-c-text-1); | ||
| } | ||
|
|
||
| .copy_llm-trigger:focus-visible { | ||
| outline: 2px solid var(--vp-c-brand-1); | ||
| outline-offset: 2px; | ||
| border-radius: 4px; | ||
| } | ||
|
|
||
| .copy_llm-menu { | ||
| position: absolute; | ||
| left: 0; | ||
| top: calc(100% + 8px); | ||
| z-index: 10; | ||
| min-width: 210px; | ||
| padding: 6px; | ||
| border: 1px solid var(--vp-c-divider); | ||
| border-radius: 8px; | ||
| background: var(--vp-c-bg-elv); | ||
| box-shadow: var(--vp-shadow-3); | ||
| } | ||
|
|
||
| .copy_llm-item { | ||
| display: block; | ||
| width: 100%; | ||
| padding: 8px 10px; | ||
| border: 0; | ||
| border-radius: 6px; | ||
| background: transparent; | ||
| color: var(--vp-c-text-1); | ||
| text-align: left; | ||
| cursor: pointer; | ||
| font-size: 13px; | ||
| line-height: 18px; | ||
| transition: background-color 0.25s; | ||
| } | ||
|
|
||
| .copy_llm-item:hover, | ||
| .copy_llm-item:focus-visible { | ||
| background: var(--vp-c-default-soft); | ||
| outline: none; | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Using relative paths in
llms.txtis more robust and portable than hardcodingORIGIN. It allows the index to work correctly on local previews, staging environments, or if the site is hosted under a different domain or base path. LLM clients will resolve these relative links against the URL of thellms.txtfile itself.