diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index fb91960c2c6..b45d7005993 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -1,10 +1,12 @@ // @ts-ignore -import { apiAnchor } from "@tsed/vitepress-theme/markdown/api-anchor/api-anchor.js"; +import {apiAnchor} from "@tsed/vitepress-theme/markdown/api-anchor/api-anchor.js"; import llmstxt from "vitepress-plugin-llms"; -import { defineConfig } from "vitepress"; -import pkg from "../../package.json"; -import referenceSidebar from "../public/reference-sidebar.json"; -import team from "../team.json"; +import {defineConfig} from "vitepress"; +import pkg from "../../package.json" with {type: "json"}; +import referenceSidebar from "../public/reference-sidebar.json" with {type: "json"}; +import team from "../team.json" with {type: "json"}; +import {buildLlmContentsPlugin} from "./plugins/buildLlmContents.js"; +import {apiLlmLinks} from "./plugins/apiLllmLinks.js"; const sort = (items: {text: string; link: string}[]) => items.sort((a, b) => a.text.localeCompare(b.text)); @@ -405,9 +407,46 @@ const Releases = [ export default defineConfig({ vite: { plugins: [ + buildLlmContentsPlugin({ + sections: [ + { + source: "guide", + destination: "public/ai/guides", + label: "Guides" + }, + { + source: "introduction", + destination: "public/ai/introduction", + label: "Introduction" + }, + { + source: "api", + destination: "public/ai/api", + label: "API references" + } + ], + sidebar: { + groups: [ + { + text: "Core", + pattern: /core|@tsed\/di|hooks|schema$|\/exceptions$|engines|json-mapper|open-spec/ + }, + { + text: "Platform", + pattern: /platform/ + }, + { + text: "ORM", + pattern: /adapters|ioredis|mikro-orm|mongoose|objection|prisma/ + } + ], + thirdPartyGroup: "Third parties" + } + }), + apiLlmLinks, llmstxt({ ignoreFilesPerOutput: { - llmsFullTxt: ["api/**"] + llmsTxt: ["api/**"] } }) ] diff --git a/docs/.vitepress/plugins/apiLllmLinks.ts b/docs/.vitepress/plugins/apiLllmLinks.ts new file mode 100644 index 00000000000..2af68742d55 --- /dev/null +++ b/docs/.vitepress/plugins/apiLllmLinks.ts @@ -0,0 +1,13 @@ +import {getApiReferenceLinks} from "./utils/sidebar.js"; + +export const apiLlmLinks = { + enforce: "pre" as const, + name: "tsed-api-llm-links", + transform(content: string, id: string) { + if (!id.endsWith("/api.md")) { + return null; + } + + return content.replace("", getApiReferenceLinks()); + } +}; diff --git a/docs/.vitepress/plugins/buildLlmContents.ts b/docs/.vitepress/plugins/buildLlmContents.ts new file mode 100644 index 00000000000..c0b6b6a0634 --- /dev/null +++ b/docs/.vitepress/plugins/buildLlmContents.ts @@ -0,0 +1,74 @@ +import {join} from "node:path"; + +import {intro, log, outro, spinner} from "@clack/prompts"; + +import {copyFiles} from "./utils/copy-files.js"; +import {buildReferenceSidebar, type ApiSidebarOptions} from "./utils/sidebar.js"; + +export interface LlmContentSection { + destination: string; + label: string; + source: string; +} + +export interface BuildLlmContentsOptions { + docsRoot?: string; + sections?: LlmContentSection[]; + sidebar?: ApiSidebarOptions; +} + +const DEFAULT_DOCS_ROOT = join(import.meta.dirname, "..", ".."); +/** + * Each entry describes a docs directory to copy into /public/ai. + * Markdown is normalized (remark), snippet directives are inlined, + * and @@Symbol@@ tokens are rewritten to /ai/api links. + */ +const DEFAULT_DOC_SECTIONS: LlmContentSection[] = []; +let buildTask: Promise | undefined; + +export async function buildLlmContents({ + docsRoot = DEFAULT_DOCS_ROOT, + sections = DEFAULT_DOC_SECTIONS, + sidebar +}: BuildLlmContentsOptions = {}) { + intro("Building LLM references"); + + try { + for (const section of sections) { + const completed = await copyFiles({ + cwd: docsRoot, + src: section.source, + dest: section.destination, + label: section.label + }); + + if (!completed) { + log.warn(`${section.label} copy skipped`); + } + } + + const sidebarStep = spinner(); + sidebarStep.start("Generating API sidebar"); + await buildReferenceSidebar(docsRoot, sidebar); + sidebarStep.stop("Sidebar generated"); + + outro("LLM references ready"); + } catch (error) { + log.error(error instanceof Error ? error.message : String(error)); + outro("LLM references build failed"); + throw error; + } +} + +// Orchestration only; implementation lives in ./llm/* + +export function buildLlmContentsPlugin(options: BuildLlmContentsOptions = {}) { + return { + enforce: "pre" as const, + name: "tsed-build-llm-contents", + async configResolved() { + buildTask ??= buildLlmContents(options); + await buildTask; + } + }; +} diff --git a/docs/.vitepress/plugins/utils/copy-files.ts b/docs/.vitepress/plugins/utils/copy-files.ts new file mode 100644 index 00000000000..c94ef56ae41 --- /dev/null +++ b/docs/.vitepress/plugins/utils/copy-files.ts @@ -0,0 +1,84 @@ +import {dirname, join} from "node:path"; + +import {log, progress} from "@clack/prompts"; +import fsExtra from "fs-extra"; +import {globby} from "globby"; + +import {transformMarkdown} from "./markdown.js"; + +const {copy, ensureDir, pathExists, readFile, remove, writeFile} = fsExtra; + +export interface CopyFilesOptions { + cwd: string; + dest: string; + label: string; + src: string; +} + +interface CopyFileOptions { + cwd: string; + destinationRoot: string; + progressLogger: Pick, "advance" | "stop">; + relativePath: string; + sourceRoot: string; +} + +export async function copyFiles({cwd, src, dest, label}: CopyFilesOptions) { + const sourceDir = join(cwd, src); + const destinationDir = join(cwd, dest); + + if (!(await pathExists(sourceDir))) { + log.warn(`[build-llm-contents] Skip ${label.toLowerCase()}: missing ${sourceDir}`); + return false; + } + + await remove(destinationDir).catch(() => undefined); + await ensureDir(destinationDir); + + const files = await globby(["**/*"], {cwd: sourceDir, dot: true, onlyFiles: true}); + + if (files.length === 0) { + log.warn(`[build-llm-contents] No files found to sync for ${label.toLowerCase()}`); + return false; + } + + const copyProgress = progress({style: "heavy", max: files.length, size: 40}); + + copyProgress.start(`Syncing ${label}`); + + try { + for (const relativePath of files) { + await copyFile({ + cwd, + sourceRoot: sourceDir, + destinationRoot: destinationDir, + relativePath, + progressLogger: copyProgress + }); + } + } catch (error) { + copyProgress.stop(`${label} failed`); + throw error; + } + + copyProgress.stop(`${label} updated`); + return true; +} + +async function copyFile({cwd, sourceRoot, destinationRoot, relativePath, progressLogger}: CopyFileOptions) { + const sourcePath = join(sourceRoot, relativePath); + const destinationPath = join(destinationRoot, relativePath); + + await ensureDir(dirname(destinationPath)); + + if (relativePath.endsWith(".md")) { + const fileContent = await readFile(sourcePath, "utf8"); + const cleaned = await transformMarkdown(fileContent, {docsRoot: cwd}); + await writeFile(destinationPath, cleaned); + progressLogger.advance(1, `Formatted ${relativePath}`); + return; + } + + await copy(sourcePath, destinationPath); + progressLogger.advance(1, `Copied ${relativePath}`); +} diff --git a/docs/.vitepress/plugins/utils/markdown.ts b/docs/.vitepress/plugins/utils/markdown.ts new file mode 100644 index 00000000000..26c15188765 --- /dev/null +++ b/docs/.vitepress/plugins/utils/markdown.ts @@ -0,0 +1,212 @@ +import {extname, join} from "node:path"; + +import fsExtra from "fs-extra"; +import remarkParse from "remark-parse"; +import remarkStringify from "remark-stringify"; +import unified from "unified"; + +const {readFile} = fsExtra; +const markdownProcessor = unified().use(remarkParse).use(remarkStringify, {fences: true, bullet: "-"}).use(remarkCleanApiMarkdown); +const INLINE_SNIPPET_RE = /^<<<\s+@\/([^\s]+?)(?:\s+\[(.+?)\])?\s*$/gm; +const SYMBOL_TOKEN_RE = /@@([A-Za-z0-9_.-]+)@@/g; + +interface ApiData { + modules?: Record; +} + +interface ApiModule { + symbols?: ApiSymbol[]; +} + +interface ApiSymbol { + path?: string; + symbolName: string; +} + +interface ExampleBlock { + end: number; + label: string; + original: string; + relativePath: string; + start: number; +} + +interface HeadingInfo { + module: string; + title: string; +} + +const symbolIndexCache = new Map>(); + +export interface TransformMarkdownOptions { + docsRoot?: string; +} + +export async function transformMarkdown(content: string, options: TransformMarkdownOptions = {}) { + const {docsRoot} = options; + let nextContent = content; + + if (docsRoot) { + nextContent = await inlineExampleBlocks(nextContent, docsRoot); + nextContent = await replaceSymbolLinks(nextContent, docsRoot); + } + + const {frontmatter, body} = extractFrontmatter(nextContent); + const processed = await markdownProcessor.process(body.trimStart()); + const cleanedBody = String(processed).trimStart(); + return frontmatter ? `${frontmatter}\n${cleanedBody}` : cleanedBody; +} + +async function inlineExampleBlocks(content: string, docsRoot: string) { + const matches: ExampleBlock[] = []; + let match; + + while ((match = INLINE_SNIPPET_RE.exec(content)) !== null) { + matches.push({ + start: match.index, + end: match.index + match[0].length, + relativePath: match[1], + label: match[2] ?? "", + original: match[0] + }); + } + + if (!matches.length) { + return content; + } + + let result = ""; + let lastIndex = 0; + + for (const entry of matches) { + result += content.slice(lastIndex, entry.start); + result += await loadSnippetBlock(entry, docsRoot); + lastIndex = entry.end; + } + + result += content.slice(lastIndex); + return result; +} + +async function replaceSymbolLinks(content: string, docsRoot: string) { + const index = await loadSymbolIndex(docsRoot); + return content.replace(SYMBOL_TOKEN_RE, (match: string, symbolName: string) => { + const entry = index.get(symbolName); + + if (!entry) { + console.warn(`[build-llm-contents] Unable to resolve symbol ${symbolName}`); + return match; + } + + return `[${symbolName}](/ai${entry.path}.md)`; + }); +} + +async function loadSymbolIndex(docsRoot: string) { + const cachedIndex = symbolIndexCache.get(docsRoot); + if (cachedIndex) { + return cachedIndex; + } + + const apiPath = join(docsRoot, "public/api.json"); + const data = JSON.parse(await readFile(apiPath, "utf8")) as ApiData; + const map = new Map(); + + Object.values(data.modules ?? {}).forEach((module: ApiModule) => { + module.symbols?.forEach((symbol: ApiSymbol) => { + if (symbol.symbolName && symbol.path) { + map.set(symbol.symbolName, symbol); + } + }); + }); + + symbolIndexCache.set(docsRoot, map); + return map; +} + +async function loadSnippetBlock(entry: ExampleBlock, docsRoot: string) { + const absolutePath = join(docsRoot, entry.relativePath); + + try { + const code = await readFile(absolutePath, "utf8"); + const language = getLanguageFromExtension(extname(absolutePath)); + const labelSuffix = entry.label ? ` [${entry.label}]` : ""; + return `\`\`\`${language}${labelSuffix}\n${code.trimEnd()}\n\`\`\``; + } catch (error) { + console.warn( + `[build-llm-contents] Unable to inline snippet ${absolutePath}: ${error instanceof Error ? error.message : String(error)}` + ); + return entry.original; + } +} + +function getLanguageFromExtension(extension: string) { + return extension ? extension.replace(/^\./, "") : ""; +} + +function remarkCleanApiMarkdown() { + return (tree: any) => { + let headingInfo: HeadingInfo | undefined; + + tree.children = tree.children.filter((node: any) => { + if (node.type === "html") { + const trimmed = node.value.trim(); + + if (trimmed.startsWith("