diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingFencePlugin.java b/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingFencePlugin.java index 082d6b997..41b3f7118 100644 --- a/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingFencePlugin.java +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingFencePlugin.java @@ -74,4 +74,9 @@ public List textForSearch() { public PluginParamsDefinition parameters() { return delegate.parameters(); } + + @Override + public String markdownRepresentation() { + return delegate.markdownRepresentation(); + } } diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingIncludePlugin.java b/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingIncludePlugin.java index 7e7b4faf0..edb12ab58 100644 --- a/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingIncludePlugin.java +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingIncludePlugin.java @@ -70,4 +70,9 @@ public PluginResult process(ComponentsRegistry componentsRegistry, ParserHandler public PluginParamsDefinition parameters() { return delegate.parameters(); } + + @Override + public String markdownRepresentation() { + return delegate.markdownRepresentation(); + } } diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingInlineCodePlugin.java b/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingInlineCodePlugin.java index 51e444657..5fa523ba9 100644 --- a/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingInlineCodePlugin.java +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/TrackingInlineCodePlugin.java @@ -69,4 +69,9 @@ public PluginResult process(ComponentsRegistry componentsRegistry, Path markupPa paramsTracker.trackParams(pluginParams); return delegate.process(componentsRegistry, markupPath, pluginParams); } + + @Override + public String markdownRepresentation() { + return delegate.markdownRepresentation(); + } } diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/reveal/ReadMoreFencePlugin.java b/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/reveal/ReadMoreFencePlugin.java index 36313f89e..0edd3c515 100644 --- a/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/reveal/ReadMoreFencePlugin.java +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/extensions/reveal/ReadMoreFencePlugin.java @@ -23,6 +23,7 @@ import org.testingisdocumenting.znai.extensions.PluginParamsDefinition; import org.testingisdocumenting.znai.extensions.PluginResult; import org.testingisdocumenting.znai.extensions.fence.FencePlugin; +import org.testingisdocumenting.znai.markdown.PageMarkdownSection; import org.testingisdocumenting.znai.parser.MarkupParser; import org.testingisdocumenting.znai.parser.MarkupParserResult; import org.testingisdocumenting.znai.search.SearchScore; @@ -31,6 +32,7 @@ import java.nio.file.Path; import java.util.List; +import java.util.stream.Collectors; import java.util.stream.Stream; public class ReadMoreFencePlugin implements FencePlugin { @@ -74,4 +76,15 @@ public Stream auxiliaryFiles(ComponentsRegistry componentsRegistr public List textForSearch() { return List.of(SearchScore.STANDARD.text(parserResult.getAllText())); } + + @Override + public String markdownRepresentation() { + if (parserResult == null || parserResult.markdown() == null) { + return ""; + } + + return parserResult.markdown().sections().stream() + .map(PageMarkdownSection::markdown) + .collect(Collectors.joining("\n\n")); + } } diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/MarkdownGeneratorParserHandler.java b/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/MarkdownGeneratorParserHandler.java index 10ec84145..c83190e30 100644 --- a/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/MarkdownGeneratorParserHandler.java +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/MarkdownGeneratorParserHandler.java @@ -26,11 +26,13 @@ import org.testingisdocumenting.znai.extensions.include.IncludePlugin; import org.testingisdocumenting.znai.extensions.inlinedcode.InlinedCodePlugin; import org.testingisdocumenting.znai.parser.HeadingProps; +import org.testingisdocumenting.znai.parser.PageSectionIdTitle; import org.testingisdocumenting.znai.parser.ParserHandler; import org.testingisdocumenting.znai.parser.docelement.DocElement; import org.testingisdocumenting.znai.parser.table.MarkupTableData; import org.testingisdocumenting.znai.reference.DocReferences; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -40,17 +42,30 @@ enum State { DEFAULT } - private final StringBuilder markdown; - private final int baseHeadingLevel; + private final List sections; private State state = State.DEFAULT; - public MarkdownGeneratorParserHandler(int baseHeadingLevel) { - this.markdown = new StringBuilder(); - this.baseHeadingLevel = baseHeadingLevel; + private String currentSectionId; + private String currentSectionTitle; + private StringBuilder currentMarkdown; + + public MarkdownGeneratorParserHandler() { + this.sections = new ArrayList<>(); + + this.currentSectionId = ""; + this.currentSectionTitle = ""; + this.currentMarkdown = new StringBuilder(); } - public String getMarkdown() { - return markdown.toString().replaceAll("\n\n\n", "\n\n"); + public PageMarkdown getMarkdown() { + return new PageMarkdown(List.copyOf(sections)); + } + + private void finalizeCurrentSection() { + if (!currentMarkdown.isEmpty()) { + String markdown = currentMarkdown.toString().replaceAll("\n\n\n", "\n\n"); + sections.add(new PageMarkdownSection(currentSectionId, currentSectionTitle, markdown)); + } } @Override @@ -59,19 +74,21 @@ public void onSectionStart(String title, HeadingProps headingProps, Heading head return; } - int level = Math.max(1, heading.getLevel() + baseHeadingLevel); - markdown.append("#".repeat(level)).append(" ").append(title).append("\n\n"); + finalizeCurrentSection(); + + currentSectionId = new PageSectionIdTitle(title, headingProps.props()).getId(); + currentSectionTitle = title; + currentMarkdown = new StringBuilder(); } @Override public void onSubHeading(int level, String title, HeadingProps headingProps, Heading heading) { - int adjustedLevel = Math.max(1, level + baseHeadingLevel); - markdown.append("#".repeat(adjustedLevel)).append(" ").append(title).append("\n\n"); + currentMarkdown.append("#".repeat(level)).append(" ").append(title).append("\n\n"); } @Override public void onSimpleText(String value) { - markdown.append(value); + currentMarkdown.append(value); } @Override @@ -81,49 +98,49 @@ public void onParagraphStart() { @Override public void onParagraphEnd() { switch (state) { - case INSIDE_LIST -> markdown.append("\n"); - case DEFAULT -> markdown.append("\n\n"); + case INSIDE_LIST -> currentMarkdown.append("\n"); + case DEFAULT -> currentMarkdown.append("\n\n"); } } @Override public void onInlinedCode(String inlinedCode, DocReferences docReferences) { - markdown.append("`").append(inlinedCode).append("`"); + currentMarkdown.append("`").append(inlinedCode).append("`"); } @Override public void onEmphasisStart() { - markdown.append("*"); + currentMarkdown.append("*"); } @Override public void onEmphasisEnd() { - markdown.append("*"); + currentMarkdown.append("*"); } @Override public void onStrongEmphasisStart() { - markdown.append("**"); + currentMarkdown.append("**"); } @Override public void onStrongEmphasisEnd() { - markdown.append("**"); + currentMarkdown.append("**"); } @Override public void onStrikeThroughStart() { - markdown.append("~~"); + currentMarkdown.append("~~"); } @Override public void onStrikeThroughEnd() { - markdown.append("~~"); + currentMarkdown.append("~~"); } @Override public void onListItemStart() { - markdown.append("- "); + currentMarkdown.append("- "); } @Override @@ -138,7 +155,7 @@ public void onBulletListStart(char bulletMarker, boolean tight) { @Override public void onBulletListEnd() { state = State.DEFAULT; - markdown.append("\n"); + currentMarkdown.append("\n"); } @Override @@ -149,36 +166,36 @@ public void onOrderedListStart(char delimiter, int startNumber) { @Override public void onOrderedListEnd() { state = State.DEFAULT; - markdown.append("\n"); + currentMarkdown.append("\n"); } @Override public void onBlockQuoteStart() { - markdown.append("> "); + currentMarkdown.append("> "); } @Override public void onBlockQuoteEnd() { - markdown.append("\n"); + currentMarkdown.append("\n"); } @Override public void onImage(String title, String destination, String alt) { - markdown.append("![").append(alt != null ? alt : "").append("](").append(destination); + currentMarkdown.append("![").append(alt != null ? alt : "").append("](").append(destination); if (title != null && !title.trim().isEmpty()) { - markdown.append(" \"").append(title).append("\""); + currentMarkdown.append(" \"").append(title).append("\""); } - markdown.append(")\n\n"); + currentMarkdown.append(")\n\n"); } @Override public void onSnippet(PluginParams pluginParams, String lang, String lineNumber, String snippet) { - markdown.append("```").append(lang != null ? lang : "").append("\n"); - markdown.append(snippet); + currentMarkdown.append("```").append(lang != null ? lang : "").append("\n"); + currentMarkdown.append(snippet); if (!snippet.endsWith("\n")) { - markdown.append("\n"); + currentMarkdown.append("\n"); } - markdown.append("```\n\n"); + currentMarkdown.append("```\n\n"); } @Override @@ -190,60 +207,57 @@ public void onTable(MarkupTableData tableData) { return; } - // Header row - markdown.append("| ").append(String.join(" | ", columnTitles)).append(" |\n"); - - // Separator row - markdown.append("|").append(" --- |".repeat(columnTitles.size())).append("\n"); - - // Data rows + currentMarkdown.append("| ").append(String.join(" | ", columnTitles)).append(" |\n"); + currentMarkdown.append("|").append(" --- |".repeat(columnTitles.size())).append("\n"); + for (List row : data) { - markdown.append("| "); + currentMarkdown.append("| "); for (int i = 0; i < columnTitles.size(); i++) { String cell = i < row.size() && row.get(i) != null ? row.get(i).toString() : ""; - markdown.append(cell.replace("|", "\\|")); + currentMarkdown.append(cell.replace("|", "\\|")); if (i < columnTitles.size() - 1) { - markdown.append(" | "); + currentMarkdown.append(" | "); } } - markdown.append(" |\n"); + currentMarkdown.append(" |\n"); } - markdown.append("\n\n"); + currentMarkdown.append("\n\n"); } @Override public void onFootnoteDefinition(ParsedFootnote footnote) { - markdown.append("[^").append(footnote.id().id()).append("]: ").append(footnote.allText()).append("\n\n"); + currentMarkdown.append("[^").append(footnote.id().id()).append("]: ").append(footnote.allText()).append("\n\n"); } @Override public void onSoftLineBreak() { - markdown.append(" "); + currentMarkdown.append(" "); } @Override public void onHardLineBreak() { - markdown.append(" \n"); + currentMarkdown.append(" \n"); } @Override public void onThematicBreak() { - markdown.append("---\n\n"); + currentMarkdown.append("---\n\n"); } @Override public void onHtml(String html, boolean isInlined) { String plainText = Jsoup.parse(html).text(); if (!plainText.isEmpty()) { - markdown.append(plainText); + currentMarkdown.append(plainText); if (!isInlined) { - markdown.append("\n\n"); + currentMarkdown.append("\n\n"); } } } @Override public void onParsingEnd() { + finalizeCurrentSection(); } @Override @@ -252,17 +266,17 @@ public void onSectionEnd() { @Override public void onLinkStart(String url) { - markdown.append("["); + currentMarkdown.append("["); } @Override public void onLinkEnd() { - markdown.append("]()"); + currentMarkdown.append("]()"); } @Override public void onFootnoteReference(FootnoteId footnoteId) { - markdown.append("[^").append(footnoteId.id()).append("]"); + currentMarkdown.append("[^").append(footnoteId.id()).append("]"); } @Override @@ -296,18 +310,18 @@ public void onGlobalAnchorRefEnd() { @Override public void onIncludePlugin(IncludePlugin includePlugin, PluginResult pluginResult) { String markdownRepresentation = includePlugin.markdownRepresentation(); - markdown.append(markdownRepresentation).append("\n\n"); + currentMarkdown.append(markdownRepresentation).append("\n\n"); } @Override public void onFencePlugin(FencePlugin fencePlugin, PluginResult pluginResult) { String markdownRepresentation = fencePlugin.markdownRepresentation(); - markdown.append(markdownRepresentation).append("\n\n"); + currentMarkdown.append(markdownRepresentation).append("\n\n"); } @Override public void onInlinedCodePlugin(InlinedCodePlugin inlinedCodePlugin, PluginResult pluginResult) { String markdownRepresentation = inlinedCodePlugin.markdownRepresentation(); - markdown.append(markdownRepresentation); + currentMarkdown.append(markdownRepresentation); } } \ No newline at end of file diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/PageMarkdown.java b/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/PageMarkdown.java new file mode 100644 index 000000000..9dd0085e4 --- /dev/null +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/PageMarkdown.java @@ -0,0 +1,27 @@ +/* + * Copyright 2025 znai maintainers + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.testingisdocumenting.znai.markdown; + +import java.util.List; + +/** + * Represents a page's markdown content organized by sections. + * Each section corresponds to a first-level heading, with content before + * the first heading stored in a section with empty id and title. + */ +public record PageMarkdown(List sections) { +} diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/PageMarkdownSection.java b/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/PageMarkdownSection.java new file mode 100644 index 000000000..dbb88b1b8 --- /dev/null +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/markdown/PageMarkdownSection.java @@ -0,0 +1,24 @@ +/* + * Copyright 2025 znai maintainers + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.testingisdocumenting.znai.markdown; + +/** + * Represents a section of markdown with an id, title, and content. + * Used to store markdown content separated by first-level headings. + */ +public record PageMarkdownSection(String id, String title, String markdown) { +} diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/parser/MarkupParserResult.java b/znai-core/src/main/java/org/testingisdocumenting/znai/parser/MarkupParserResult.java index 0d0ee8613..0753c9b17 100644 --- a/znai-core/src/main/java/org/testingisdocumenting/znai/parser/MarkupParserResult.java +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/parser/MarkupParserResult.java @@ -18,6 +18,7 @@ package org.testingisdocumenting.znai.parser; import org.testingisdocumenting.znai.core.AuxiliaryFile; +import org.testingisdocumenting.znai.markdown.PageMarkdown; import org.testingisdocumenting.znai.parser.docelement.DocElement; import org.testingisdocumenting.znai.search.PageSearchEntry; import org.testingisdocumenting.znai.structure.PageMeta; @@ -32,7 +33,7 @@ public record MarkupParserResult(DocElement docElement, List searchEntries, List auxiliaryFiles, PageMeta pageMeta, - String markdown) { + PageMarkdown markdown) { /** * Top level page element. use get content to get access to the children * diff --git a/znai-core/src/main/java/org/testingisdocumenting/znai/parser/commonmark/MarkdownParser.java b/znai-core/src/main/java/org/testingisdocumenting/znai/parser/commonmark/MarkdownParser.java index b8a0f26cd..5b9481165 100644 --- a/znai-core/src/main/java/org/testingisdocumenting/znai/parser/commonmark/MarkdownParser.java +++ b/znai-core/src/main/java/org/testingisdocumenting/znai/parser/commonmark/MarkdownParser.java @@ -61,7 +61,7 @@ public MarkupParserResult parse(Path path, String markdown) { SearchCrawlerParserHandler searchCrawler = new SearchCrawlerParserHandler(); DocElementCreationParserHandler elementCreationHandler = new DocElementCreationParserHandler(componentsRegistry, path); - MarkdownGeneratorParserHandler markdownGenerator = new MarkdownGeneratorParserHandler(1); + MarkdownGeneratorParserHandler markdownGenerator = new MarkdownGeneratorParserHandler(); ParserHandlersList parserHandler = new ParserHandlersList(elementCreationHandler, searchCrawler, markdownGenerator); diff --git a/znai-core/src/test/groovy/org/testingisdocumenting/znai/markdown/MarkdownGeneratorParserHandlerTest.groovy b/znai-core/src/test/groovy/org/testingisdocumenting/znai/markdown/MarkdownGeneratorParserHandlerTest.groovy index 8f39964b0..6eec738f4 100644 --- a/znai-core/src/test/groovy/org/testingisdocumenting/znai/markdown/MarkdownGeneratorParserHandlerTest.groovy +++ b/znai-core/src/test/groovy/org/testingisdocumenting/znai/markdown/MarkdownGeneratorParserHandlerTest.groovy @@ -23,7 +23,9 @@ import org.testingisdocumenting.znai.parser.commonmark.MarkdownParser import org.junit.Test import java.nio.file.Paths +import java.util.stream.Collectors +import static org.testingisdocumenting.webtau.Matchers.contain import static org.testingisdocumenting.znai.parser.TestComponentsRegistry.TEST_COMPONENTS_REGISTRY class MarkdownGeneratorParserHandlerTest { @@ -31,72 +33,59 @@ class MarkdownGeneratorParserHandlerTest { @Test void "should generate markdown for basic text"() { - def result = process("Hello world") + def result = pageMarkdownAsText("Hello world") result.should == "Hello world\n\n" } @Test void "should generate markdown with text formatting"() { - def result = process("This is **bold** and *italic* and ~~strikethrough~~") + def result = pageMarkdownAsText("This is **bold** and *italic* and ~~strikethrough~~") result.should == "This is **bold** and *italic* and ~~strikethrough~~\n\n" } - @Test - void "should generate headers with base level adjustment"() { - def result = process("# Header") - result.should == "# Header\n\n" - - result = process("# Header", 2) - result.should == "### Header\n\n" - } - @Test void "should generate inline code"() { - def result = process("Code: `hello()`") + def result = pageMarkdownAsText("Code: `hello()`") result.should == "Code: `hello()`\n\n" } @Test void "should generate code blocks"() { - def result = process("```java\nSystem.out.println();\n```") + def result = pageMarkdownAsText("```java\nSystem.out.println();\n```") result.should == "```java\nSystem.out.println();\n```\n\n" } @Test void "should generate lists"() { - def result = process("- Item one\n- Item two") + def result = pageMarkdownAsText("- Item one\n- Item two") result.should == "- Item one\n- Item two\n\n" } @Test void "should generate block quotes"() { - def result = process("> Quote") + def result = pageMarkdownAsText("> Quote") result.should == "> Quote\n\n" } @Test void "should generate thematic breaks"() { - def result = process("Text\n\n---\n\nMore text") + def result = pageMarkdownAsText("Text\n\n---\n\nMore text") result.should == "Text\n\n---\n\nMore text\n\n" } @Test void "should handle multi-paragraph content"() { - def result = process("First paragraph\n\nSecond paragraph") + def result = pageMarkdownAsText("First paragraph\n\nSecond paragraph") result.should == "First paragraph\n\nSecond paragraph\n\n" } - @Test - void "should handle complex document with proper header level shifting"() { - def result = process("# Main Title\n\nContent here\n\n## Subtitle\n\nMore content with **bold**", 1) - result.should == "## Main Title\n\nContent here\n\n### Subtitle\n\nMore content with **bold**\n\n" - } - @Test void "should preserve markdown formatting in complex content"() { - def result = process("# Documentation\n\nThis has **bold**, *italic*, and `code`.\n\n```python\nprint('hello')\n```\n\n- List item\n- Another item") - result.should == "# Documentation\n" + - "\n" + + def result = pageMarkdown("# Documentation\n\nThis has **bold**, *italic*, and `code`.\n\n```python\nprint('hello')\n```\n\n- List item\n- Another item") + + result.sections().size().should == 1 + result.sections().get(0).title().should == "Documentation" + result.sections().get(0).markdown().should == "This has **bold**, *italic*, and `code`.\n" + "\n" + "```python\n" + @@ -109,27 +98,114 @@ class MarkdownGeneratorParserHandlerTest { @Test void "should handle empty input"() { - def result = process("") + def result = pageMarkdownAsText("") result.should == "" } @Test void "should use plugin markdownRepresentation method for include plugins"() { - def handler = new MarkdownGeneratorParserHandler(0) + def handler = new MarkdownGeneratorParserHandler() def plugin = new DummyIncludePlugin() - + plugin.process(TEST_COMPONENTS_REGISTRY, handler, Paths.get("test.md"), TEST_COMPONENTS_REGISTRY.pluginParamsFactory().create("dummy", "test-param", [:])) - + handler.onIncludePlugin(plugin, PluginResult.empty()) - - def result = handler.getMarkdown() + handler.onParsingEnd() + + def result = joinSections(handler.getMarkdown()) result.should == "**Dummy plugin content**: test-param\n\n" } - private static String process(String markdown, int baseHeadingLevel = 0) { - def handler = new MarkdownGeneratorParserHandler(baseHeadingLevel) + @Test + void "should separate content into sections by first-level headers"() { + def markdown = """# Section One + +Content in section one + +## Subsection + +More content + +# Section Two + +Content in section two""" + + def pageMarkdown =pageMarkdown(markdown) + + pageMarkdown.sections().size().should == 2 + + def section1 = pageMarkdown.sections()[0] + section1.title().should == "Section One" + section1.id().should == "section-one" + + def section2 = pageMarkdown.sections()[1] + section2.title().should == "Section Two" + section2.id().should == "section-two" + } + + @Test + void "should capture content before first header in section with empty title"() { + def markdown = """Content before any header + +More content + +# First Header + +Content after header""" + + def pageMarkdown = pageMarkdown(markdown) + + pageMarkdown.sections().size().should == 2 + + def section1 = pageMarkdown.sections()[0] + section1.title().should == "" + section1.id().should == "" + section1.markdown().trim().should == "Content before any header\n\nMore content" + } + + @Test + void "should include markdown representation from include plugin in section"() { + def markdown = """# Global + +Content before include + +:include-dummy: included-content +""" + + def pageMarkdown = pageMarkdown(markdown) + + pageMarkdown.sections().size().should == 1 + + def section1 = pageMarkdown.sections()[0] + section1.title().should == "Global" + section1.id().should == "global" + + def sectionMd = section1.markdown() + + sectionMd.should contain("Content before include") + sectionMd.should contain("**Dummy plugin content**: included-content") + } + + private static String pageMarkdownAsText(String markdown) { + return joinSections(pageMarkdown(markdown)) + } + + private static PageMarkdown pageMarkdown(String markdown) { + def handler = new MarkdownGeneratorParserHandler() parser.parse(Paths.get("test.md"), handler, markdown) + handler.onParsingEnd() + return handler.getMarkdown() } + + private static String joinSections(PageMarkdown pageMarkdown) { + if (pageMarkdown.sections().isEmpty()) { + return "" + } + + return pageMarkdown.sections().stream() + .map(section -> section.markdown()) + .collect(Collectors.joining()) + } } \ No newline at end of file diff --git a/znai-core/src/test/groovy/org/testingisdocumenting/znai/parser/TestMarkdownParser.groovy b/znai-core/src/test/groovy/org/testingisdocumenting/znai/parser/TestMarkdownParser.groovy index a73bf3cb6..345f5635f 100644 --- a/znai-core/src/test/groovy/org/testingisdocumenting/znai/parser/TestMarkdownParser.groovy +++ b/znai-core/src/test/groovy/org/testingisdocumenting/znai/parser/TestMarkdownParser.groovy @@ -17,6 +17,8 @@ package org.testingisdocumenting.znai.parser import org.testingisdocumenting.znai.core.ComponentsRegistry +import org.testingisdocumenting.znai.markdown.PageMarkdown +import org.testingisdocumenting.znai.markdown.PageMarkdownSection import org.testingisdocumenting.znai.parser.commonmark.MarkdownParser import org.testingisdocumenting.znai.parser.docelement.DocElement import org.testingisdocumenting.znai.search.PageSearchEntry @@ -38,7 +40,8 @@ class TestMarkdownParser extends MarkdownParser { def searchEntry = new PageSearchEntry(new PageSectionIdTitle('dummy page section title', [:]), [SearchScore.STANDARD.text(markdown)]) - return new MarkupParserResult(page, [], [searchEntry], [], new PageMeta(), markdown) + def pageMarkdownSection = new PageMarkdownSection("test-id", "test-title", markdown) + return new MarkupParserResult(page, [], [searchEntry], [], new PageMeta(), new PageMarkdown([pageMarkdownSection])) } @Override diff --git a/znai-core/src/test/groovy/org/testingisdocumenting/znai/parser/TestMarkupParser.groovy b/znai-core/src/test/groovy/org/testingisdocumenting/znai/parser/TestMarkupParser.groovy index e04041ad5..9c0c80f61 100644 --- a/znai-core/src/test/groovy/org/testingisdocumenting/znai/parser/TestMarkupParser.groovy +++ b/znai-core/src/test/groovy/org/testingisdocumenting/znai/parser/TestMarkupParser.groovy @@ -16,6 +16,8 @@ package org.testingisdocumenting.znai.parser +import org.testingisdocumenting.znai.markdown.PageMarkdown +import org.testingisdocumenting.znai.markdown.PageMarkdownSection import org.testingisdocumenting.znai.parser.docelement.DocElement import org.testingisdocumenting.znai.search.PageSearchEntry import org.testingisdocumenting.znai.search.SearchScore @@ -32,7 +34,8 @@ class TestMarkupParser implements MarkupParser { def searchEntry = new PageSearchEntry(new PageSectionIdTitle('dummy page section title', [:]), [SearchScore.STANDARD.text(markup)]) - return new MarkupParserResult(page, [], [searchEntry], [], new PageMeta(), markup) + def pageMarkdownSection = new PageMarkdownSection("test-id", "test-title", markup) + return new MarkupParserResult(page, [], [searchEntry], [], new PageMeta(), new PageMarkdown([pageMarkdownSection])) } @Override diff --git a/znai-docs/znai/release-notes/1.82/add-2025-11-02-add-page-sections-to-llm-txt.md b/znai-docs/znai/release-notes/1.82/add-2025-11-02-add-page-sections-to-llm-txt.md new file mode 100644 index 000000000..fdcfb5ecb --- /dev/null +++ b/znai-docs/znai/release-notes/1.82/add-2025-11-02-add-page-sections-to-llm-txt.md @@ -0,0 +1 @@ +* Add: `llm.txt` now have links for each individual page section \ No newline at end of file diff --git a/znai-docs/znai/release-notes/1.82/add-2025-11-02-search-entries-xml.md b/znai-docs/znai/release-notes/1.82/add-2025-11-02-search-entries-xml.md new file mode 100644 index 000000000..f71d4d14d --- /dev/null +++ b/znai-docs/znai/release-notes/1.82/add-2025-11-02-search-entries-xml.md @@ -0,0 +1 @@ +* Add: `search-entries.xml` now contains page section text as markdown representation \ No newline at end of file diff --git a/znai-sphinx/src/main/java/org/testingisdocumenting/znai/parser/sphinx/SphinxDocTreeParser.java b/znai-sphinx/src/main/java/org/testingisdocumenting/znai/parser/sphinx/SphinxDocTreeParser.java index ed94a1832..9f5f3de2e 100644 --- a/znai-sphinx/src/main/java/org/testingisdocumenting/znai/parser/sphinx/SphinxDocTreeParser.java +++ b/znai-sphinx/src/main/java/org/testingisdocumenting/znai/parser/sphinx/SphinxDocTreeParser.java @@ -17,6 +17,8 @@ package org.testingisdocumenting.znai.parser.sphinx; import org.testingisdocumenting.znai.core.ComponentsRegistry; +import org.testingisdocumenting.znai.markdown.PageMarkdown; +import org.testingisdocumenting.znai.markdown.PageMarkdownSection; import org.testingisdocumenting.znai.parser.MarkupParser; import org.testingisdocumenting.znai.parser.MarkupParserResult; import org.testingisdocumenting.znai.parser.ParserHandlersList; @@ -44,12 +46,15 @@ public MarkupParserResult parse(Path path, String docXml) { DocTreeDomXmlParser xmlParser = new DocTreeDomXmlParser(componentsRegistry, path, parserHandler); xmlParser.parse(docXml); + PageMarkdownSection section = new PageMarkdownSection("", "", docXml); + PageMarkdown pageMarkdown = new PageMarkdown(Collections.singletonList(section)); + return new MarkupParserResult(elementCreationHandler.getDocElement(), Collections.emptyList(), searchCrawler.getSearchEntries(), elementCreationHandler.getAuxiliaryFiles(), new PageMeta(), - docXml); + pageMarkdown); } @Override diff --git a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/extensions/markup/MarkdownBasePlugin.java b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/extensions/markup/MarkdownBasePlugin.java index de208cba0..e64d04610 100644 --- a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/extensions/markup/MarkdownBasePlugin.java +++ b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/extensions/markup/MarkdownBasePlugin.java @@ -23,6 +23,7 @@ import org.testingisdocumenting.znai.extensions.PluginParams; import org.testingisdocumenting.znai.extensions.PluginParamsDefinition; import org.testingisdocumenting.znai.extensions.file.ManipulatedSnippetContentProvider; +import org.testingisdocumenting.znai.markdown.PageMarkdownSection; import org.testingisdocumenting.znai.parser.MarkupParser; import org.testingisdocumenting.znai.parser.MarkupParserResult; import org.testingisdocumenting.znai.preprocessor.RegexpBasedPreprocessor; @@ -32,6 +33,7 @@ import java.nio.file.Path; import java.util.List; +import java.util.stream.Collectors; import java.util.stream.Stream; public abstract class MarkdownBasePlugin implements Plugin { @@ -112,4 +114,15 @@ public Stream auxiliaryFiles(ComponentsRegistry componentsRegistr public List textForSearch() { return List.of(SearchScore.STANDARD.text(parserResult.getAllText())); } + + @Override + public String markdownRepresentation() { + if (parserResult == null || parserResult.markdown() == null) { + return ""; + } + + return parserResult.markdown().sections().stream() + .map(PageMarkdownSection::markdown) + .collect(Collectors.joining("\n\n")); + } } diff --git a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/LlmContentGenerator.java b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/LlmContentGenerator.java index ab6416b8d..05c334d36 100644 --- a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/LlmContentGenerator.java +++ b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/LlmContentGenerator.java @@ -55,11 +55,29 @@ public String generateContent() { return; } - String pageUrl = buildPageUrl(tocItem); - llmContent.append("# ").append(tocItem.getChapterTitle()).append(" :: ").append(tocItem.getPageTitle()).append("\n"); - llmContent.append("answer-link: ").append(pageUrl).append("\n\n"); - llmContent.append(parserResult.markdown()); - llmContent.append("\n\n"); + String basePageUrl = buildPageUrl(tocItem); + + parserResult.markdown().sections().forEach(section -> { + if (section.title().isEmpty() && section.markdown().trim().isEmpty()) { + return; + } + + llmContent.append("# ").append(tocItem.getChapterTitle()).append(" :: ").append(tocItem.getPageTitle()); + + if (!section.title().isEmpty()) { + llmContent.append(" :: ").append(section.title()); + } + llmContent.append("\n"); + + String sectionUrl = section.title().isEmpty() + ? basePageUrl + : basePageUrl + "#" + section.id(); + llmContent.append("answer-link: ").append(sectionUrl).append("\n\n"); + + String sectionMarkdown = section.markdown().stripTrailing(); + llmContent.append(sectionMarkdown); + llmContent.append("\n\n"); + }); }); return llmContent.toString(); diff --git a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/WebSite.java b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/WebSite.java index 2d27d3fd8..d8b9b394f 100644 --- a/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/WebSite.java +++ b/znai-website-gen/src/main/java/org/testingisdocumenting/znai/website/WebSite.java @@ -27,6 +27,7 @@ import org.testingisdocumenting.znai.resources.*; import org.testingisdocumenting.znai.html.*; import org.testingisdocumenting.znai.html.reactjs.ReactJsBundle; +import org.testingisdocumenting.znai.markdown.PageMarkdownSection; import org.testingisdocumenting.znai.parser.MarkupParser; import org.testingisdocumenting.znai.parser.MarkupParserResult; import org.testingisdocumenting.znai.parser.commonmark.MarkdownParser; @@ -587,35 +588,36 @@ private void updateTocItemWithPageMeta(TocItem tocItem, PageMeta pageMeta) { } private void updateSearchEntries(TocItem tocItem, MarkupParserResult parserResult) { - List siteSearchEntries = parserResult.searchEntries().stream() - .map(pageSearchEntry -> + List siteSearchEntries = parserResult.markdown().sections().stream() + .filter(section -> !(section.title().isEmpty() && section.markdown().trim().isEmpty())) + .map(section -> new GlobalSearchEntry( - searchEntryUrl(tocItem, pageSearchEntry), - searchEntryTitle(tocItem, pageSearchEntry), - pageSearchEntry.extractText())) + searchEntryUrl(tocItem, section), + searchEntryTitle(tocItem, section), + section.markdown())) .collect(toList()); globalSearchEntries.addAll(siteSearchEntries); localSearchEntries.add(new PageLocalSearchEntries(tocItem, parserResult.searchEntries())); } - private String searchEntryUrl(TocItem tocItem, PageSearchEntry pageSearchEntry) { + private String searchEntryUrl(TocItem tocItem, PageMarkdownSection section) { DocUrl docUrl = tocItem.isIndex() ? DocUrl.indexUrl(): - new DocUrl(tocItem.getDirName(), tocItem.getFileNameWithoutExtension(), pageSearchEntry.getPageSectionId()); + new DocUrl(tocItem.getDirName(), tocItem.getFileNameWithoutExtension(), section.id()); return docStructure.createUrl(null, docUrl); } - private String searchEntryTitle(TocItem tocItem, PageSearchEntry pageSearchEntry) { + private String searchEntryTitle(TocItem tocItem, PageMarkdownSection section) { if (tocItem.isIndex()) { - return docMeta.getTitle() + " " + (pageSearchEntry.getPageSectionTitle().isEmpty() ? - docMeta.getType() : pageSearchEntry.getPageSectionTitle()); + return docMeta.getTitle() + " " + (section.title().isEmpty() ? + docMeta.getType() : section.title()); } - String pageSectionPart = pageSearchEntry.getPageSectionTitle().isEmpty() ? + String pageSectionPart = section.title().isEmpty() ? "" : - ", " + pageSearchEntry.getPageSectionTitle(); + ", " + section.title(); String chapterPart = tocItem.getChapterTitle().isEmpty() ? "" :