diff --git a/znai-cli/src/main/java/org/testingisdocumenting/znai/cli/ZnaiCliApp.java b/znai-cli/src/main/java/org/testingisdocumenting/znai/cli/ZnaiCliApp.java index 129fa79af..fe4529b14 100644 --- a/znai-cli/src/main/java/org/testingisdocumenting/znai/cli/ZnaiCliApp.java +++ b/znai-cli/src/main/java/org/testingisdocumenting/znai/cli/ZnaiCliApp.java @@ -171,6 +171,7 @@ private WebSite generateDocs(Path sourceRoot) { WebResource.withPath(userDefinedFavicon, HtmlPage.FAVICON_PATH): WebResource.fromResource(HtmlPage.FAVICON_PATH); + String llmUrlPrefix = System.getenv("ZNAI_LLM_URL_PREFIX"); WebSite.Configuration webSiteCfg = WebSite.withRoot(sourceRoot). withId(getDocId()). withDocumentationType(config.getMarkupType()). @@ -185,7 +186,8 @@ private WebSite generateDocs(Path sourceRoot) { withWebResources(favIconResource). withPageModifiedTimeStrategy(pageModifiedTimeStrategy()). withEnabledPreview(config.isPreviewMode()). - withValidateExternalLinks(config.isValidateExternalLinks()); + withValidateExternalLinks(config.isValidateExternalLinks()). + withLlmUrlPrefix(llmUrlPrefix == null ? "" : llmUrlPrefix); return config.isExportMode() ? webSiteCfg.parseOnly(): diff --git a/znai-docs/znai/release-notes/1.81/add-2025-10-20-llm-page-urls.md b/znai-docs/znai/release-notes/1.81/add-2025-10-20-llm-page-urls.md new file mode 100644 index 000000000..b2958d3b5 --- /dev/null +++ b/znai-docs/znai/release-notes/1.81/add-2025-10-20-llm-page-urls.md @@ -0,0 +1 @@ +* Add: `llm.txt` now contains urls to the pages. Set custom prefix using `ZNAI_LLM_URL_PREFIX` env var \ No newline at end of file diff --git a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/LlmContentGenerator.java b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/LlmContentGenerator.java new file mode 100644 index 000000000..ab6416b8d --- /dev/null +++ b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/LlmContentGenerator.java @@ -0,0 +1,76 @@ +/* + * Copyright 2025 znai maintainers + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.testingisdocumenting.znai.website; + +import org.testingisdocumenting.znai.core.DocMeta; +import org.testingisdocumenting.znai.parser.MarkupParserResult; +import org.testingisdocumenting.znai.structure.Page; +import org.testingisdocumenting.znai.structure.TocItem; + +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class LlmContentGenerator { + private final DocMeta docMeta; + private final String llmUrlPrefix; + private final Map pageByTocItem; + private final Map parserResultByTocItem; + + public LlmContentGenerator(DocMeta docMeta, + String llmUrlPrefix, + Map pageByTocItem, + Map parserResultByTocItem) { + this.docMeta = docMeta; + this.llmUrlPrefix = llmUrlPrefix; + this.pageByTocItem = pageByTocItem; + this.parserResultByTocItem = parserResultByTocItem; + } + + public String generateContent() { + StringBuilder llmContent = new StringBuilder(); + llmContent.append("\"").append(docMeta.getTitle()).append("\" full guide:\n\n"); + + pageByTocItem.forEach((tocItem, page) -> { + if (page == null) { + return; + } + + MarkupParserResult parserResult = parserResultByTocItem.get(tocItem); + if (parserResult == null || parserResult.markdown() == null) { + return; + } + + String pageUrl = buildPageUrl(tocItem); + llmContent.append("# ").append(tocItem.getChapterTitle()).append(" :: ").append(tocItem.getPageTitle()).append("\n"); + llmContent.append("answer-link: ").append(pageUrl).append("\n\n"); + llmContent.append(parserResult.markdown()); + llmContent.append("\n\n"); + }); + + return llmContent.toString(); + } + + private String buildPageUrl(TocItem tocItem) { + return Stream.of(llmUrlPrefix, + docMeta.getId(), + tocItem.getDirName(), + tocItem.getFileNameWithoutExtension()) + .filter(part -> !part.isEmpty()) + .collect(Collectors.joining("/")); + } +} diff --git a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/WebSite.java b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/WebSite.java index 0656553aa..2d27d3fd8 100644 --- a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/WebSite.java +++ b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/WebSite.java @@ -101,6 +101,8 @@ public class WebSite implements Log { private final PageModifiedTimeStrategy pageModifiedTimeStrategy; + private final String llmUrlPrefix; + private RegexpBasedPreprocessor regexpBasedPreprocessor; private WebSite(Configuration siteConfig) { @@ -109,6 +111,7 @@ private WebSite(Configuration siteConfig) { docMeta = siteConfig.docMeta; pageModifiedTimeStrategy = siteConfig.pageModifiedTimeStrategy != null ? siteConfig.pageModifiedTimeStrategy : new FileBasedPageModifiedTime(); + llmUrlPrefix = siteConfig.llmUrlPrefix; tocChangeListeners = new HashSet<>(); @@ -683,23 +686,15 @@ private void generateSearchIndex() { private void generateLlmContent() { reportPhase("generating LLM context file"); - StringBuilder llmContent = new StringBuilder(); - llmContent.append("\"").append(docMeta.getTitle()).append("\" full guide:\n\n"); - - forEachPage((tocItem, page) -> { - if (page == null) { - return; - } - - MarkupParserResult parserResult = parserResultByTocItem.get(tocItem); - if (parserResult != null && parserResult.markdown() != null) { - llmContent.append("# ").append(tocItem.getChapterTitle()).append(" -> ").append(tocItem.getPageTitle()).append("\n\n"); - llmContent.append(parserResult.markdown()); - llmContent.append("\n\n"); - } - }); - - deployer.deploy("llm.txt", llmContent.toString()); + LlmContentGenerator generator = new LlmContentGenerator( + docMeta, + llmUrlPrefix, + pageByTocItem, + parserResultByTocItem + ); + + String llmContent = generator.generateContent(); + deployer.deploy("llm.txt", llmContent); } private void buildJsonOfAllPages() { @@ -973,6 +968,7 @@ public static class Configuration { private DocMeta docMeta = new DocMeta(Collections.emptyMap()); private PageModifiedTimeStrategy pageModifiedTimeStrategy; private List additionalLookupPaths; + private String llmUrlPrefix; private Configuration() { webResources = new ArrayList<>(); @@ -1054,6 +1050,11 @@ public Configuration withAdditionalLookupPaths(List additionalLookupPath return this; } + public Configuration withLlmUrlPrefix(String llmUrlPrefix) { + this.llmUrlPrefix = llmUrlPrefix; + return this; + } + public Configuration withMetaFromJsonFile(Path path) { String json = fileTextContent(path); docMeta = new DocMeta(json);