diff --git a/.wokeignore b/.wokeignore deleted file mode 120000 index 19f3fe4..0000000 --- a/.wokeignore +++ /dev/null @@ -1 +0,0 @@ -docs/.wokeignore \ No newline at end of file diff --git a/docs/.gitignore b/docs/.gitignore index bf16e0d..86c41f0 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,16 +1,18 @@ # Environment *env*/ -.sphinx/venv/ +.venv/ +_dev/venv/ # Sphinx -.sphinx/warnings.txt -.sphinx/.wordlist.dic -.sphinx/.doctrees/ -.sphinx/node_modules/ +_dev/warnings.txt +_dev/.wordlist.dic +_dev/.doctrees/ +_dev/update/ +_dev/node_modules/ # Vale -.sphinx/styles/* -.sphinx/vale.ini +_dev/styles/* +#_dev/vale.ini # Build outputs _build diff --git a/docs/.sphinx/get_vale_conf.py b/docs/.sphinx/get_vale_conf.py deleted file mode 100644 index cb73a64..0000000 --- a/docs/.sphinx/get_vale_conf.py +++ /dev/null @@ -1,68 +0,0 @@ -#! /usr/bin/env python - -import requests -import os - -DIR = os.getcwd() - -def main(): - if os.path.exists(f"{DIR}/.sphinx/styles/Canonical"): - print("Vale directory exists") - else: - os.makedirs(f"{DIR}/.sphinx/styles/Canonical") - - url = ( - "https://api.github.com/repos/canonical/praecepta/" - + "contents/styles/Canonical" - ) - r = requests.get(url) - for item in r.json(): - download = requests.get(item["download_url"]) - file = open(".sphinx/styles/Canonical/" + item["name"], "w") - file.write(download.text) - file.close() - - # Update dictionary - if os.path.exists(f"{DIR}/.sphinx/styles/config/dictionaries"): - print("Dictionary directory exists") - else: - os.makedirs(f"{DIR}/.sphinx/styles/config/dictionaries") - url = ( - "https://api.github.com/repos/canonical/praecepta/" - + "contents/styles/config/dictionaries" - ) - r = requests.get(url) - for item in r.json(): - download = requests.get(item["download_url"]) - file = open(".sphinx/styles/config/dictionaries/" + item["name"], "w") - file.write(download.text) - file.close() - - if os.path.exists(f"{DIR}/.sphinx/styles/config/vocabularies/Canonical"): - print("Vocab directory exists") - else: - os.makedirs(f"{DIR}/.sphinx/styles/config/vocabularies/Canonical") - - url = ( - "https://api.github.com/repos/canonical/praecepta/" - + "contents/styles/config/vocabularies/Canonical" - ) - r = requests.get(url) - for item in r.json(): - download = requests.get(item["download_url"]) - file = open( - ".sphinx/styles/config/vocabularies/Canonical/" + item["name"], - "w" - ) - file.write(download.text) - file.close() - config = requests.get( - "https://raw.githubusercontent.com/canonical/praecepta/main/vale.ini" - ) - file = open(".sphinx/vale.ini", "w") - file.write(config.text) - file.close() - - -if __name__ == "__main__": - main() diff --git a/docs/.wokeignore b/docs/.wokeignore deleted file mode 100644 index c64a603..0000000 --- a/docs/.wokeignore +++ /dev/null @@ -1,4 +0,0 @@ -# the cheat sheets contain a link to a repository with a block word which we -# cannot avoid for now, ie -# https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html -doc-cheat-sheet* diff --git a/docs/Makefile b/docs/Makefile index 4557124..0292e6f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -4,23 +4,24 @@ # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXDIR = .sphinx -SPHINXOPTS ?= -c . -d $(SPHINXDIR)/.doctrees -j auto -SPHINXBUILD ?= $(VENVDIR)/bin/sphinx-build -SOURCEDIR = . -BUILDDIR = _build -VENVDIR = $(SPHINXDIR)/venv -PA11Y = $(SPHINXDIR)/node_modules/pa11y/bin/pa11y.js --config $(SPHINXDIR)/pa11y.json -VENV = $(VENVDIR)/bin/activate -TARGET = * -METRICSDIR = $(SOURCEDIR)/.sphinx/metrics -REQPDFPACKS = latexmk fonts-freefont-otf texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-font-utils texlive-lang-cjk texlive-xetex plantuml xindy tex-gyre dvipng -CONFIRM_SUDO ?= N -VALE_CONFIG = $(SPHINXDIR)/vale.ini -VALEDIR = $(SPHINXDIR)/venv/lib/python*/site-packages/vale -VOCAB_CANONICAL = $(SPHINXDIR)/styles/config/vocabularies/Canonical -SPHINX_HOST ?= 127.0.0.1 -SPHINX_PORT ?= 8000 + +DEV_DIR ?= _dev +SPHINX_OPTS ?= -c . -d $(DEV_DIR)/.doctrees -j auto +SPHINX_BUILD ?= $(DOCS_VENVDIR)/bin/sphinx-build +SPHINX_HOST ?= 127.0.0.1 +SPHINX_PORT ?= 8000 +SPHINX_AUTOBUILD_OPTS ?= -D llms_txt_enabled=0 +DOCS_VENVDIR ?= .venv +DOCS_VENV ?= $(DOCS_VENVDIR)/bin/activate +DOCS_SOURCEDIR ?= . +DOCS_BUILDDIR ?= _build +DOCS_PDFPACKAGES ?= latexmk fonts-freefont-otf texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-font-utils texlive-lang-cjk texlive-xetex plantuml xindy tex-gyre dvipng +DOCS_VOCAB ?= $(DEV_DIR)/styles/config/vocabularies/Canonical +VALE_DIR ?= $(DOCS_VENVDIR)/lib/python*/site-packages/vale +VALE_CONFIG ?= $(DEV_DIR)/vale.ini +PA11Y_CMD ?= $(DEV_DIR)/node_modules/pa11y/bin/pa11y.js --config $(DEV_DIR)/pa11y.json +CONFIRM_SUDO ?= N +CHECK_PATH ?= $(filter-out $(DOCS_VENVDIR) $(DOCS_BUILDDIR) $(DEV_DIR),$(wildcard *)) # Put it first so that "make" without argument is like "make help". help: @@ -38,149 +39,154 @@ help: @echo "* check inclusive language: make woke" @echo "* check accessibility: make pa11y" @echo "* check style guide compliance: make vale" - @echo "* check style guide compliance on target: make vale TARGET=*" - @echo "* check metrics for documentation: make allmetrics" + @echo "* check style guide compliance on target: make vale CHECK_PATH=*" @echo "* other possible targets: make " @echo "-------------------------------------------------------------" @echo -.PHONY: help full‑help html epub pdf linkcheck spelling spellcheck woke \ - vale pa11y run serve install pa11y‑install \ - vale‑install pdf‑prep pdf‑prep‑force clean clean‑doc allmetrics \ +.PHONY: help full-help html epub pdf linkcheck spelling spellcheck woke \ + vale pa11y run serve install pa11y-install \ + vale-install pdf-prep pdf-prep-force clean clean-doc \ update lint-md -full-help: $(VENVDIR) - @. $(VENV); $(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +full-help: $(DOCS_VENVDIR) + @. $(DOCS_VENV); $(SPHINX_BUILD) -M help "$(DOCS_SOURCEDIR)" "$(DOCS_BUILDDIR)" $(SPHINX_OPTS) $(O) @echo "\n\033[1;31mNOTE: This help texts shows unsupported targets!\033[0m" @echo "Run 'make help' to see supported targets." # If requirements are updated, venv should be rebuilt and timestamped. -$(VENVDIR): +$(DOCS_VENVDIR): @echo "... setting up virtualenv" - python3 -m venv $(VENVDIR) || { echo "You must install python3-venv before you can build the documentation."; exit 1; } - . $(VENV); pip install $(PIPOPTS) --require-virtualenv \ + python3 -m venv $(DOCS_VENVDIR) || { echo "You must install python3-venv before you can build the documentation."; exit 1; } + . $(DOCS_VENV); pip install $(PIPOPTS) --require-virtualenv \ --upgrade -r requirements.txt \ - --log $(VENVDIR)/pip_install.log - @test ! -f $(VENVDIR)/pip_list.txt || \ - mv $(VENVDIR)/pip_list.txt $(VENVDIR)/pip_list.txt.bak - @. $(VENV); pip list --local --format=freeze > $(VENVDIR)/pip_list.txt - @touch $(VENVDIR) + --log $(DOCS_VENVDIR)/pip_install.log + @test ! -f $(DOCS_VENVDIR)/pip_list.txt || \ + mv $(DOCS_VENVDIR)/pip_list.txt $(DOCS_VENVDIR)/pip_list.txt.bak + @. $(DOCS_VENV); pip list --local --format=freeze > $(DOCS_VENVDIR)/pip_list.txt + @touch $(DOCS_VENVDIR) pa11y-install: - @command -v $(PA11Y) >/dev/null || { \ + @command -v $(PA11Y_CMD) >/dev/null || { \ echo "Installing \"pa11y\" from npm..."; echo; \ - mkdir -p $(SPHINXDIR)/node_modules/ ; \ - npm install --prefix $(SPHINXDIR) pa11y; \ + mkdir -p $(DEV_DIR)/node_modules/ ; \ + npm install --prefix $(DEV_DIR) pa11y; \ } -pymarkdownlnt-install: - @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/pymarkdown || pip install pymarkdownlnt +pymarkdownlnt-install: install + @. $(DOCS_VENV); test -d $(DOCS_VENVDIR)/lib/python*/site-packages/pymarkdown || pip install pymarkdownlnt==0.9.35 -install: $(VENVDIR) +install: $(DOCS_VENVDIR) run: install - . $(VENV); $(VENVDIR)/bin/sphinx-autobuild -b dirhtml --host $(SPHINX_HOST) --port $(SPHINX_PORT) "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) + . $(DOCS_VENV); $(DOCS_VENVDIR)/bin/sphinx-autobuild -b dirhtml --host $(SPHINX_HOST) --port $(SPHINX_PORT) "$(DOCS_SOURCEDIR)" "$(DOCS_BUILDDIR)" $(SPHINX_OPTS) $(SPHINX_AUTOBUILD_OPTS) -# Does not depend on $(BUILDDIR) to rebuild properly at every run. +# Does not depend on $(DOCS_BUILDDIR) to rebuild properly at every run. html: install - . $(VENV); $(SPHINXBUILD) --fail-on-warning --keep-going -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) + . $(DOCS_VENV); $(SPHINX_BUILD) --fail-on-warning --keep-going -b dirhtml "$(DOCS_SOURCEDIR)" "$(DOCS_BUILDDIR)" -w $(DEV_DIR)/warnings.txt $(SPHINX_OPTS) epub: install - . $(VENV); $(SPHINXBUILD) -b epub "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) + . $(DOCS_VENV); $(SPHINX_BUILD) -b epub "$(DOCS_SOURCEDIR)" "$(DOCS_BUILDDIR)" -w $(DEV_DIR)/warnings.txt $(SPHINX_OPTS) serve: html - cd "$(BUILDDIR)"; python3 -m http.server --bind $(SPHINX_HOST) $(SPHINX_PORT) + cd "$(DOCS_BUILDDIR)"; python3 -m http.server --bind $(SPHINX_HOST) $(SPHINX_PORT) clean: clean-doc - @test ! -e "$(VENVDIR)" -o -d "$(VENVDIR)" -a "$(abspath $(VENVDIR))" != "$(VENVDIR)" - rm -rf $(VENVDIR) - rm -rf $(SPHINXDIR)/node_modules/ - rm -rf $(SPHINXDIR)/styles - rm -rf $(VALE_CONFIG) + @test ! -e "$(DOCS_VENVDIR)" -o -d "$(DOCS_VENVDIR)" -a "$(abspath $(DOCS_VENVDIR))" != "$(DOCS_VENVDIR)" + rm -rf $(DOCS_VENVDIR) + rm -rf $(DEV_DIR)/node_modules/ + rm -rf $(DEV_DIR)/styles + #rm -rf $(VALE_CONFIG) clean-doc: - git clean -fx "$(BUILDDIR)" - rm -rf $(SPHINXDIR)/.doctrees + git clean -fx "$(DOCS_BUILDDIR)" + rm -rf $(DEV_DIR)/.doctrees linkcheck: install - . $(VENV) ; $(SPHINXBUILD) -b linkcheck "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) || { grep --color -F "[broken]" "$(BUILDDIR)/output.txt"; exit 1; } + . $(DOCS_VENV) ; $(SPHINX_BUILD) -b linkcheck -q "$(DOCS_SOURCEDIR)" "$(DOCS_BUILDDIR)" $(SPHINX_OPTS) || { grep --color -F "[broken]" "$(DOCS_BUILDDIR)/output.txt"; exit 1; } exit 0 pa11y: pa11y-install html - find $(BUILDDIR) -name *.html -print0 | xargs -n 1 -0 $(PA11Y) + find $(DOCS_BUILDDIR) -name *.html -print0 | xargs -n 1 -0 $(PA11Y_CMD) +# Without --return-code-scheme explicit, pymarkdownlnt returns 1 for multiple scenarios +# By using the explicit scheme, it only returns 1 when no files are found, +# which should not result in failure lint-md: pymarkdownlnt-install - @. $(VENV); pymarkdownlnt --config $(SPHINXDIR)/.pymarkdown.json scan --recurse --exclude=$(SPHINXDIR) --exclude=".github/" $(SOURCEDIR) + @. $(DOCS_VENV); pymarkdownlnt \ + --config $(DEV_DIR)/.pymarkdown.json \ + --return-code-scheme explicit \ + scan \ + --recurse \ + --exclude=$(DEV_DIR)/** \ + --exclude=$(DOCS_VENVDIR)/** \ + $(DOCS_SOURCEDIR); \ + status=$$?; \ + if [ $$status -eq 1 ]; then \ + echo "No Markdown files selected for linting"; \ + exit 0; \ + fi; \ + echo "pymarkdownlnt exited with code $$status"; \ + exit $$status; vale-install: install - @. $(VENV); test -f $(VALE_CONFIG) || python3 $(SPHINXDIR)/get_vale_conf.py - @echo '.Name=="Canonical.400-Enforce-inclusive-terms"' > $(SPHINXDIR)/styles/woke.filter - @echo '.Level=="error" and .Name!="Canonical.500-Repeated-words" and .Name!="Canonical.000-US-spellcheck"' > $(SPHINXDIR)/styles/error.filter - @echo '.Name=="Canonical.000-US-spellcheck"' > $(SPHINXDIR)/styles/spelling.filter - @. $(VENV); find $(VALEDIR)/vale_bin -size 195c -exec vale --version \; + @. $(DOCS_VENV); python3 $(DEV_DIR)/get_vale_conf.py + @echo '.Name=="Canonical.400-Enforce-inclusive-terms"' > $(DEV_DIR)/styles/woke.filter + @echo '.Level=="error" and .Name!="Canonical.500-Repeated-words" and .Name!="Canonical.000-US-spellcheck"' > $(DEV_DIR)/styles/error.filter + @echo '.Name=="Canonical.000-US-spellcheck"' > $(DEV_DIR)/styles/spelling.filter + @. $(DOCS_VENV); find $(VALE_DIR)/vale_bin -size 195c -exec vale --version \; woke: vale-install - @cat $(VOCAB_CANONICAL)/accept.txt > $(VOCAB_CANONICAL)/accept_backup.txt - @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(VOCAB_CANONICAL)/accept.txt - @echo "Running Vale acceptable term check against $(TARGET). To change target set TARGET= with make command" - @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/woke.filter' --glob='*.{md,rst}' $(TARGET) - @cat $(VOCAB_CANONICAL)/accept_backup.txt > $(VOCAB_CANONICAL)/accept.txt && rm $(VOCAB_CANONICAL)/accept_backup.txt + @cat $(DOCS_VOCAB)/accept.txt > $(DOCS_VOCAB)/accept_backup.txt + @cat $(DOCS_SOURCEDIR)/.custom_wordlist.txt >> $(DOCS_VOCAB)/accept.txt + @echo "Running Vale acceptable term check against $(CHECK_PATH). To change target set CHECK_PATH= with make command" + @. $(DOCS_VENV); vale --config="$(VALE_CONFIG)" --filter='$(DEV_DIR)/styles/woke.filter' --glob='*.{md,rst}' $(CHECK_PATH) + @cat $(DOCS_VOCAB)/accept_backup.txt > $(DOCS_VOCAB)/accept.txt && rm $(DOCS_VOCAB)/accept_backup.txt vale: vale-install - @cat $(VOCAB_CANONICAL)/accept.txt > $(VOCAB_CANONICAL)/accept_backup.txt - @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(VOCAB_CANONICAL)/accept.txt - @echo "Running Vale against $(TARGET). To change target set TARGET= with make command" - @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/error.filter' --glob='*.{md,rst}' $(TARGET) - @cat $(VOCAB_CANONICAL)/accept_backup.txt > $(VOCAB_CANONICAL)/accept.txt && rm $(VOCAB_CANONICAL)/accept_backup.txt + @cat $(DOCS_VOCAB)/accept.txt > $(DOCS_VOCAB)/accept_backup.txt + @cat $(DOCS_SOURCEDIR)/.custom_wordlist.txt >> $(DOCS_VOCAB)/accept.txt + @echo "Running Vale against $(CHECK_PATH). To change target set CHECK_PATH= with make command" + @. $(DOCS_VENV); vale --config="$(VALE_CONFIG)" --filter='$(DEV_DIR)/styles/error.filter' --glob='*.{md,rst}' $(CHECK_PATH) + @cat $(DOCS_VOCAB)/accept_backup.txt > $(DOCS_VOCAB)/accept.txt && rm $(DOCS_VOCAB)/accept_backup.txt spelling: vale-install - @cat $(VOCAB_CANONICAL)/accept.txt > $(VOCAB_CANONICAL)/accept_backup.txt - @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(VOCAB_CANONICAL)/accept.txt - @echo "Running Vale against $(TARGET). To change target set TARGET= with make command" - @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/spelling.filter' --glob='*.{md,rst}' $(TARGET) - @cat $(VOCAB_CANONICAL)/accept_backup.txt > $(VOCAB_CANONICAL)/accept.txt && rm $(VOCAB_CANONICAL)/accept_backup.txt + @cat $(DOCS_VOCAB)/accept.txt > $(DOCS_VOCAB)/accept_backup.txt + @cat $(DOCS_SOURCEDIR)/.custom_wordlist.txt >> $(DOCS_VOCAB)/accept.txt + @echo "Running Vale against $(CHECK_PATH). To change target set CHECK_PATH= with make command" + @. $(DOCS_VENV); vale --config="$(VALE_CONFIG)" --filter='$(DEV_DIR)/styles/spelling.filter' --glob='*.{md,rst}' $(CHECK_PATH) + @cat $(DOCS_VOCAB)/accept_backup.txt > $(DOCS_VOCAB)/accept.txt && rm $(DOCS_VOCAB)/accept_backup.txt spellcheck: spelling @echo "Please note that the \`make spellcheck\` command is being deprecated in favor of \`make spelling\`" pdf-prep: install - @for packageName in $(REQPDFPACKS); do (dpkg-query -W -f='$${Status}' $$packageName 2>/dev/null | \ + @for packageName in $(DOCS_PDFPACKAGES); do (dpkg-query -W -f='$${Status}' $$packageName 2>/dev/null | \ grep -c "ok installed" >/dev/null && echo "Package $$packageName is installed") && continue || \ - (echo; echo "PDF generation requires the installation of the following packages: $(REQPDFPACKS)" && \ + (echo; echo "PDF generation requires the installation of the following packages: $(DOCS_PDFPACKAGES)" && \ echo "" && echo "Run 'sudo make pdf-prep-force' to install these packages" && echo "" && echo \ "Please be aware these packages will be installed to your system") && exit 1 ; done pdf-prep-force: apt-get update apt-get upgrade -y - apt-get install --no-install-recommends -y $(REQPDFPACKS) \ + apt-get install --no-install-recommends -y $(DOCS_PDFPACKAGES) \ pdf: pdf-prep - @. $(VENV); sphinx-build -M latexpdf "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) - @rm ./$(BUILDDIR)/latex/front-page-light.pdf || true - @rm ./$(BUILDDIR)/latex/normal-page-footer.pdf || true - @find ./$(BUILDDIR)/latex -name "*.pdf" -exec mv -t ./$(BUILDDIR) {} + - @rm -r $(BUILDDIR)/latex + @. $(DOCS_VENV); $(SPHINX_BUILD) -M latexpdf "$(DOCS_SOURCEDIR)" "$(DOCS_BUILDDIR)" $(SPHINX_OPTS) + @rm ./$(DOCS_BUILDDIR)/latex/front-page-light.pdf || true + @rm ./$(DOCS_BUILDDIR)/latex/normal-page-footer.pdf || true + @find ./$(DOCS_BUILDDIR)/latex -name "*.pdf" -exec mv -t ./$(DOCS_BUILDDIR) {} + + @rm -r $(DOCS_BUILDDIR)/latex @echo - @echo "Output can be found in ./$(BUILDDIR)" + @echo "Output can be found in ./$(DOCS_BUILDDIR)" @echo -allmetrics: html - @echo "Recording documentation metrics..." - @echo "Checking for existence of vale..." - . $(VENV) - @. $(VENV); test -d $(VALEDIR) || pip install vale - @. $(VENV); test -f $(VALE_CONFIG) || python3 $(SPHINXDIR)/get_vale_conf.py - @. $(VENV); find $(VALEDIR)/vale_bin -size 195c -exec vale --config "$(VALE_CONFIG)" $(TARGET) > /dev/null \; - @eval '$(METRICSDIR)/source_metrics.sh $(PWD)' - @. $(VENV); python3 $(METRICSDIR)/build_metrics.py $(BUILDDIR) - update: install - @. $(VENV); .sphinx/update_sp.py + @. $(DOCS_VENV); _dev/update_sp.py # Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +# "make mode" option. $(O) is meant as a shortcut for $(SPHINX_OPTS). %: $(MAKE) --no-print-directory install - . $(VENV); $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - \ No newline at end of file + . $(DOCS_VENV); $(SPHINX_BUILD) -M $@ "$(DOCS_SOURCEDIR)" "$(DOCS_BUILDDIR)" $(SPHINX_OPTS) $(O) diff --git a/docs/.sphinx/.pre-commit-config.yaml b/docs/_dev/.pre-commit-config.yaml similarity index 100% rename from docs/.sphinx/.pre-commit-config.yaml rename to docs/_dev/.pre-commit-config.yaml diff --git a/docs/.sphinx/.pymarkdown.json b/docs/_dev/.pymarkdown.json similarity index 100% rename from docs/.sphinx/.pymarkdown.json rename to docs/_dev/.pymarkdown.json diff --git a/docs/.sphinx/_static/bundle.js b/docs/_dev/_static/bundle.js similarity index 100% rename from docs/.sphinx/_static/bundle.js rename to docs/_dev/_static/bundle.js diff --git a/docs/.sphinx/_static/cookie-banner.css b/docs/_dev/_static/cookie-banner.css similarity index 100% rename from docs/.sphinx/_static/cookie-banner.css rename to docs/_dev/_static/cookie-banner.css diff --git a/docs/.sphinx/_templates/footer.html b/docs/_dev/_templates/footer.html similarity index 100% rename from docs/.sphinx/_templates/footer.html rename to docs/_dev/_templates/footer.html diff --git a/docs/.sphinx/_templates/header.html b/docs/_dev/_templates/header.html similarity index 100% rename from docs/.sphinx/_templates/header.html rename to docs/_dev/_templates/header.html diff --git a/docs/_dev/get_vale_conf.py b/docs/_dev/get_vale_conf.py new file mode 100644 index 0000000..79049c3 --- /dev/null +++ b/docs/_dev/get_vale_conf.py @@ -0,0 +1,151 @@ +#! /usr/bin/env python + +import os +import shutil +import subprocess +import tempfile +import sys +import logging +import argparse + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) + +DEV_DIR = os.path.join(os.getcwd(), "_dev") + +GITHUB_REPO = "canonical/documentation-style-guide" +GITHUB_CLONE_URL = f"https://github.com/{GITHUB_REPO}.git" + +# Source paths to copy from repo +VALE_FILE_LIST = [ + "styles/Canonical", + "styles/config/vocabularies/Canonical", + "styles/config/dictionaries" + #"vale.ini" +] + +def clone_repo_and_copy_paths(file_source_dest, overwrite=False): + """ + Clone the repository to a temporary directory and copy required files + + Args: + file_source_dest: dictionary of file paths to copy from the repository, + and their destination paths + overwrite: boolean flag to overwrite existing files in the destination + + Returns: + bool: True if all files were copied successfully, False otherwise + """ + + if not file_source_dest: + logging.error("No files to copy") + return False + + # Create temporary directory on disk for cloning + temp_dir = tempfile.mkdtemp() + logging.info("Cloning repository <%s> to temporary directory: %s", GITHUB_REPO, temp_dir) + clone_cmd = ["git", "clone", "--depth", "1", GITHUB_CLONE_URL, temp_dir] + + try: + result = subprocess.run( + clone_cmd, + capture_output=True, + text=True, + check=True + ) + logging.debug("Git clone output: %s", result.stdout) + except subprocess.CalledProcessError as e: + logging.error("Git clone failed: %s", e.stderr) + return False + + # Copy files from the cloned repository to the destination paths + is_copy_success = True + for source, dest in file_source_dest.items(): + source_path = os.path.join(temp_dir, source) + + if not os.path.exists(source_path): + is_copy_success = False + logging.error("Source path not found: %s", source_path) + continue + + if not copy_files_to_path(source_path, dest, overwrite): + is_copy_success = False + logging.error("Failed to copy %s to %s", source_path, dest) + + # Clean up temporary directory + logging.info("Cleaning up temporary directory: %s", temp_dir) + shutil.rmtree(temp_dir) + + return is_copy_success + +def copy_files_to_path(source_path, dest_path, overwrite=False): + """ + Copy a file or directory from source to destination + + Args: + source_path: Path to the source file or directory + dest_path: Path to the destination + overwrite: Boolean flag to overwrite existing files in the destination + + Returns: + bool: True if copy was successful, False otherwise + """ + # Skip if source file doesn't exist + if not os.path.exists(source_path): + logging.warning("Source path not found: %s", source_path) + return False + + logging.info("Copying %s to %s", source_path, dest_path) + # Handle existing files + if os.path.exists(dest_path): + if overwrite: + logging.info(" Destination exists, overwriting: %s", dest_path) + if os.path.isdir(dest_path): + shutil.rmtree(dest_path) + else: + os.remove(dest_path) + else: + logging.info(" Destination exists, skip copying (use overwrite=True to replace): %s", + dest_path) + return True # Skip copying + + # Copy the source to destination + try: + if os.path.isdir(source_path): + # entire directory + shutil.copytree(source_path, dest_path) + else: + # individual files + shutil.copy2(source_path, dest_path) + return True + except (shutil.Error, OSError) as e: + logging.error("Copy failed: %s", e) + return False + +def parse_arguments(): + parser = argparse.ArgumentParser(description="Download Vale configuration files") + parser.add_argument("--no-overwrite", action="store_true", help="Don't overwrite existing files") + return parser.parse_args() + +def main(): + # Define local directory paths + vale_files_dict = {file: os.path.join(DEV_DIR, file) for file in VALE_FILE_LIST} + + # Parse command line arguments, default to overwrite_enabled = True + overwrite_enabled = not parse_arguments().no_overwrite + + # Download into /tmp through git clone + if not clone_repo_and_copy_paths(vale_files_dict, overwrite=overwrite_enabled): + logging.error("Failed to download files from repository") + return 1 + + logging.info("Download complete") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) # Keep return code diff --git a/docs/.sphinx/metrics/build_metrics.sh b/docs/_dev/metrics/build_metrics.sh similarity index 100% rename from docs/.sphinx/metrics/build_metrics.sh rename to docs/_dev/metrics/build_metrics.sh diff --git a/docs/.sphinx/metrics/source_metrics.sh b/docs/_dev/metrics/source_metrics.sh similarity index 100% rename from docs/.sphinx/metrics/source_metrics.sh rename to docs/_dev/metrics/source_metrics.sh diff --git a/docs/.sphinx/pa11y.json b/docs/_dev/pa11y.json similarity index 100% rename from docs/.sphinx/pa11y.json rename to docs/_dev/pa11y.json diff --git a/docs/_dev/update_sp.py b/docs/_dev/update_sp.py new file mode 100755 index 0000000..3014a84 --- /dev/null +++ b/docs/_dev/update_sp.py @@ -0,0 +1,270 @@ +#! /usr/bin/env python + +# Initial update script for the Sphinx Stack. +# +# Requires some manual intervention, but makes identifying updates and differences easier. +# +# For debugging, please run this script with DEBUGGING=1 +# e.g. user@device:~/git/Canonical/sphinx-stack/docs$ DEBUGGING=1 python _dev/update_sp.py + + +import glob +import logging +import os +import re +import subprocess +import sys + +import requests +from packaging.version import parse as parse_version +from requests.exceptions import RequestException + +SPHINX_DIR = os.path.abspath(os.path.dirname(__file__)) +DOCS_DIR = os.path.abspath(os.path.join(SPHINX_DIR, "..")) +REQUIREMENTS = os.path.join(DOCS_DIR, "requirements.txt") +SPHINX_UPDATE_DIR = os.path.join(SPHINX_DIR, "update") +GITHUB_REPO = "canonical/sphinx-stack" +GITHUB_API_BASE = f"https://api.github.com/repos/{GITHUB_REPO}" +GITHUB_API_DEV_DIR = f"{GITHUB_API_BASE}/contents/docs/_dev" +GITHUB_RAW_BASE = f"https://raw.githubusercontent.com/{GITHUB_REPO}/main" + +TIMEOUT = 10 # seconds + +# Check if debugging +if os.getenv("DEBUGGING"): + logging.basicConfig(level=logging.DEBUG) + + +def main(): + # Check local version + logging.debug("Checking local version") + try: + with open(os.path.join(SPHINX_DIR, "version")) as f: + local_version = f.read().strip() + except FileNotFoundError: + print("WARNING\nWARNING\nWARNING") + print( + "You need to update to at least version 1.0.0 of the Sphinx Stack to start using the update function." + ) + print("You may experience issues using this functionality.") + logging.debug("No local version found. Setting version to None") + local_version = "None" + except Exception as e: + logging.debug(e) + raise Exception("ERROR executing check local version") + logging.debug(f"Local version = {local_version}") + + # Check release version + latest_release = query_api(GITHUB_API_BASE + "/releases/latest").json()["tag_name"] + logging.debug(f"Latest release = {latest_release}") + + # Perform actions only if local version is older than release version + logging.debug("Comparing versions") + if parse_version(local_version) < parse_version(latest_release): + logging.debug("Local version is older than the release version.") + print("Sphinx Stack is out of date.\n") + + # Identify and download '_dev' dir files to '_dev/update' + files_updated, new_files = update_static_files() + + # Write new version to file to '_dev/update' + + download_file( + GITHUB_RAW_BASE + "/docs/_dev/version", + os.path.join(SPHINX_UPDATE_DIR, "version"), + ) + + # Provide changelog to identify other significant changes + changelog = query_api(GITHUB_RAW_BASE + "/CHANGELOG.md") + logging.debug("Changelog obtained") + version_regex = re.compile(r"#+ +" + re.escape(local_version) + r" *\n") + print("SEE CURRENT CHANGELOG:") + print(re.split(version_regex, changelog.text)[0]) + + # Provide information on any files identified for updates + if files_updated: + logging.debug("Updated files found and downloaded") + print("Differences have been identified in static files.") + print("Updated files have been downloaded to '_dev/update'.") + print("Validate and move these files into your '_dev/' directory.") + else: + logging.debug("No files found to update") + # Provide information on NEW files + if new_files: + logging.debug("New files found and downloaded") + print( + "NOTE: New files have been downloaded\n", + "See 'NEWFILES.txt' for all downloaded files\n", + "Validate and merge these files into your '_dev/' directory", + ) + else: + logging.debug("No new files found to download") + else: + logging.debug("Local version and release version are the same") + print("This version is up to date.") + + # Check requirements are the same + new_requirements = [] + try: + with open(REQUIREMENTS, "r") as file: + logging.debug("Checking requirements") + + local_reqs = set(file.read().splitlines()) - {""} + requirements = set( + query_api(GITHUB_RAW_BASE + "/docs/requirements.txt").text.splitlines() + ) + + new_requirements = requirements - local_reqs + + for req in new_requirements: + logging.debug(f"{req} not found in local requirements.txt") + + for req in requirements & local_reqs: + logging.debug(f"{req} already exists in local requirements.txt") + + if new_requirements != set(): + print( + "You may need to add the following packages to your requirements.txt file:" + ) + for r in new_requirements: + print(f"{r}\n") + except FileNotFoundError: + print("requirements.txt not found") + print( + "The updated Sphinx Stack has moved requirements.txt out of the '_dev' dir" + ) + print("requirements.txt not checked, please update your requirements manually") + + +def update_static_files(): + """Checks local files against remote for new and different files, downloads to '_dev/updates'""" + files, paths = get_local_files_and_paths() + new_file_list = [] + + for item in query_api(GITHUB_API_DEV_DIR).json(): + logging.debug(f"Checking {item['name']}") + # Checks existing files in '_dev' Sphinx Stack static root for changed SHA + if item["name"] in files and item["type"] == "file": + index = files.index(item["name"]) + if item["sha"] != get_git_revision_hash(paths[index]): + logging.debug(f"Local {item['name']} is different to remote") + download_file( + item["download_url"], os.path.join(SPHINX_UPDATE_DIR, item["name"]) + ) + if item["name"] == "update_sp.py": + # Indicate update script needs to be updated and re-run + print("WARNING") + print( + "THIS UPDATE SCRIPT IS OUT OF DATE. YOU MAY NEED TO RUN ANOTHER UPDATE AFTER UPDATING TO THE FILE IN '_dev/updates'." + ) + print("WARNING\n") + else: + logging.debug("File hashes are equal") + # Checks nested files '_dev/**/**.*' for changed SHA (single level of depth) + elif item["type"] == "dir": + logging.debug(item["name"] + " is a directory") + for nested_item in query_api(f"{GITHUB_API_DEV_DIR}/{item['name']}").json(): + logging.debug(f"Checking {nested_item['name']}") + if nested_item["name"] in files: + index = files.index(nested_item["name"]) + if nested_item["sha"] != get_git_revision_hash(paths[index]): + logging.debug( + f"Local {nested_item['name']} is different to remote" + ) + download_file( + nested_item["download_url"], + os.path.join( + SPHINX_UPDATE_DIR, item["name"], nested_item["name"] + ), + ) + # Downloads NEW nested files + else: + logging.debug(f"No local version found of {nested_item['name']}") + if nested_item["type"] == "file": + new_file_list.append(nested_item["name"]) + download_file( + nested_item["download_url"], + os.path.join( + SPHINX_UPDATE_DIR, item["name"], nested_item["name"] + ), + ) + # Downloads NEW files in '_dev' Sphinx Stack static root + else: + if item["type"] == "file": + logging.debug(f"No local version found of {item['name']}") + download_file( + item["download_url"], os.path.join(SPHINX_UPDATE_DIR, item["name"]) + ) + if item["name"] != "version": + new_file_list.append(item["name"]) + # Writes return value for parent function + if os.path.exists(os.path.join(SPHINX_UPDATE_DIR)): + logging.debug("Files have been downloaded") + files_updated = True + else: + logging.debug("No downloads found") + files_updated = False + # Writes return value for parent function + if new_file_list != []: + # Provides more information on new files + with open(f"{SPHINX_DIR}/NEWFILES.txt", "w") as f: + for entry in new_file_list: + f.write(f"{entry}\n") + logging.debug("Some downloaded files are new") + return files_updated, True + return files_updated, False + + +# Checks git hash of a file +def get_git_revision_hash(file) -> str: + """Get SHA of local files""" + logging.debug(f"Getting hash of {os.path.basename(file)}") + return subprocess.check_output(["git", "hash-object", file]).decode("ascii").strip() + + +# Examines local files +def get_local_files_and_paths(): + """Identify '_dev' local files and paths""" + logging.debug("Checking local files and paths") + try: + files = [] + paths = [] + patterns = [".*", "**.*"] + files, paths = [], [] + + for pattern in patterns: + for file in glob.iglob(os.path.join(SPHINX_DIR, pattern), recursive=True): + files.append(os.path.basename(file)) + paths.append(file) + return files, paths + except Exception as e: + logging.debug(e) + raise RuntimeError("get_local_files_and_paths()") from e + + +# General API query with timeout and RequestException +def query_api(url): + """Query an API with a globally set timeout""" + logging.debug(f"Querying {url}") + try: + r = requests.get(url, timeout=TIMEOUT) + return r + except RequestException as e: + raise RuntimeError(f"Failed query_api(): {url}") from e + + +# General file download function +def download_file(url, output_path): + """Download a file to a specified path""" + logging.debug(f"Downloading {os.path.basename(output_path)}") + try: + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, "wb") as file: + file.write(query_api(url).content) + except Exception as e: + logging.debug(e) + raise RuntimeError(f"Failed download_file(): {url}") from e + + +if __name__ == "__main__": + sys.exit(main()) # Keep return code diff --git a/docs/_dev/vale.ini b/docs/_dev/vale.ini new file mode 100644 index 0000000..417a395 --- /dev/null +++ b/docs/_dev/vale.ini @@ -0,0 +1,69 @@ +StylesPath = styles +MinAlertLevel = suggestion +Vocab = Canonical +IgnoredClasses = woke-ignore, vale-ignore, spellexception + +[*.{md,txt,rst,html}] +BasedOnStyles = Canonical + +# this enumerates all of the current rules with their suggested +# severity level. This makes it easier to edit for specific use cases + +Canonical.000-US-spellcheck = error +Canonical.003-Ubuntu-names-versions = error +Canonical.005-Industry-product-names = warning +Canonical.006-Contractions-forbidden = warning +Canonical.007-Headings-sentence-case = warning +Canonical.008-Headings-no-punctuation = suggestion +Canonical.009-Headings-no-links = warning +Canonical.010-Punctuation-double-spaces = warning +Canonical.012-Date-format = warning +Canonical.013-Spell-out-numbers-below-10 = suggestion +Canonical.014a-Numbers-greater-than-nine-should-be-in-numeric-form = suggestion +Canonical.014b-Numbers-with-five-or-more-digits-must-have-comma-separators = suggestion +Canonical.015-No-prompts-in-comments = warning +Canonical.016-No-inline-comments = suggestion +Canonical.017a-Suggest-code-over-25-lines = suggestion +Canonical.017b-Warn-code-over-35-lines = warning +Canonical.019-no-external-images = error +Canonical.025a-latinisms-with-english-equivalents = suggestion +Canonical.025b-latinisms-to-reconsider = suggestion +Canonical.025c-latinisms-to-avoid = suggestion +Canonical.026-hints-tips = suggestion +Canonical.027-bad-descriptive-links = warning +Canonical.400-Enforce-inclusive-terms = error +Canonical.500-Repeated-words = warning + + +[*.md] +BasedOnStyles = Canonical + +# Ignore custom spelling role +TokenIgnores = ({(vale-ignore|woke-ignore|spellexception)}`.+?`) +# Ignore technical roles +TokenIgnores = (\{external[\w+:]+?\}`.+?`) + +# Ignore opening colon-fenced (:::) lines and colon-fenced literal directives +# The listed directives must be in sync with the [*.rst] `BlockIgnores` +BlockIgnores = (::::*[^\n]*), \ +(?s)(::::*) *{(?:code-block|code|prompt|sourcecode|terminal|toctree)}[^\n]*\n(?:(?!\1$)[^\n]*\n)*\1 * + + +[*.rst] +BasedOnStyles = Canonical + +# Ignore custom spelling role +TokenIgnores = (:(vale-ignore|woke-ignore|spellexception):`.+?`) +# Ignore technical roles +TokenIgnores = (:relatedlinks:) +TokenIgnores = (:(command|envvar|file|kbd|literal|literalref|math|option|program|regexp|samp|token|external[\w+:]+?):`.+?`) +# Ignore directives +TokenIgnores = (.. \w+::) +# Ignore options +TokenIgnores = (:\w+: \w+), (:\w+:) +# Ignore external links +TokenIgnores = (\`\w+\`_) + +# Spelling not respecting code blocks +# The listed directives must be in sync with the [*.md] `BlockIgnores` +BlockIgnores = (?s)\.\. (code-block|code|prompt|sourcecode|terminal|toctree):: ?[^\n]*\n( *:[\w-]+: *[^\n]+\n)*\n( +[^\n]+\n|\n)* diff --git a/docs/_dev/version b/docs/_dev/version new file mode 100644 index 0000000..cd5ac03 --- /dev/null +++ b/docs/_dev/version @@ -0,0 +1 @@ +2.0 diff --git a/docs/conf.py b/docs/conf.py index 8136259..bc20aa8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,6 +2,7 @@ import ast import os import yaml +import textwrap # Configuration for the Sphinx documentation builder. # All configuration specific to your project should be done in this file. @@ -61,7 +62,7 @@ # -H 'Accept: application/vnd.github.v3.raw' \ # https://api.github.com/repos/canonical/ | jq '.created_at' -copyright = "%s CC-BY-SA, %s" % (datetime.date.today().year, author) +copyright = f"{datetime.date.today().year}" # Documentation website URL @@ -188,31 +189,41 @@ # Template and asset locations -html_static_path = [".sphinx/_static"] -templates_path = [".sphinx/_templates"] +html_static_path = ["_dev/_static"] +templates_path = ["_dev/_templates"] ############# # Redirects # ############# -# To set up redirects: https://documatt.gitlab.io/sphinx-reredirects/usage.html -# For example: 'explanation/old-name.html': '../how-to/prettify.html', +# Add redirects to the 'redirects.txt' file +# https://sphinxext-rediraffe.readthedocs.io/en/latest/ -# To set up redirects in the Read the Docs project dashboard: -# https://docs.readthedocs.io/en/stable/guides/redirects.html +rediraffe_redirects = "redirects.txt" -# NOTE: If undefined, set to None, or empty, -# the sphinx_reredirects extension will be disabled. +# Strips '/index.html' from destination URLs when building with 'dirhtml' +rediraffe_dir_only = True -redirects = { - "reference/patch_acceptance_criteria/index.html": "../patch-acceptance-criteria/", - "how-to/develop-customize/": "how-to/", - "how-to/develop-customise/": "how-to/", - "how-to/source-code/": "how-to/", - "how-to/develop-customize/build-kernel/": "how-to/develop-customise/build-kernel/", - "how-to/develop-customize/build-kernel-snap/": "how-to/develop-customise/build-kernel-snap/", -} + +############################ +# sphinx-llm configuration # +############################ + +# This description is included in llms.txt to provide some initial context for your +# product docs. +# TODO: Add a description in the form "This is the documentation for , +# ". +llms_txt_description = textwrap.dedent( + """\ + This documentation is about the Ubuntu Linux kernel, mainly focused on its + development processes, tools, schedules, and more. + """ +) + +# The base URL for references built by sphinx-markdown-builder. +if os.environ.get("READTHEDOCS"): + markdown_http_base = html_baseurl ########################### @@ -292,12 +303,16 @@ "sphinx_last_updated_by_git", "sphinx.ext.intersphinx", "sphinx_sitemap", + "sphinx_rerediraffe", + "sphinx_llm.txt", ] # Excludes files or directories from processing exclude_patterns = [ ".github/**", + ".venv*", + "_dev/*", ] # Adds custom CSS files, located under 'html_static_path' diff --git a/docs/redirects.txt b/docs/redirects.txt new file mode 100644 index 0000000..407f3d4 --- /dev/null +++ b/docs/redirects.txt @@ -0,0 +1,6 @@ +"reference/patch_acceptance_criteria/index.html" "reference/patch-acceptance-criteria/" +"how-to/develop-customize/" "how-to/" +"how-to/develop-customise/" "how-to/" +"how-to/source-code/" "how-to/" +"how-to/develop-customize/build-kernel/" "how-to/develop-customise/build-kernel/" +"how-to/develop-customize/build-kernel-snap/" "how-to/develop-customise/build-kernel-snap/" \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index caaa865..69c0782 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,32 +1,34 @@ # Canonical theme (still needed for Furo theme and custom templates) -canonical-sphinx>=0.5.1 +canonical-sphinx~=0.6 # Extensions previously auto-loaded by canonical-sphinx -myst-parser~=4.0 +myst-parser~=4.0 # v5.0.0 causes version conflicts sphinx-autobuild -sphinx-design -sphinx-notfound-page -sphinx-reredirects -sphinx-tabs -sphinxcontrib-jquery -sphinxext-opengraph +sphinx-design==0.6.1 +sphinx-notfound-page~=1.1 +sphinx-reredirects==0.1.6 +sphinx-tabs~=3.5 +sphinxcontrib-jquery~=4.1 +sphinxext-opengraph~=0.13 +sphinx-rerediraffe>=0.0.3, <1.0.0 # Extra extensions, previously bundled as canonical-sphinx-extensions -sphinx-config-options>=0.1.0 -sphinx-contributor-listing>=0.1.0 -sphinx-filtered-toctree>=0.1.0 -sphinx-related-links>=0.1.1 -sphinx-roles>=0.1.0 -sphinx-terminal>=1.0.2 -sphinx-ubuntu-images>=0.1.0 -sphinx-youtube-links>=0.1.0 +sphinx-config-options~=0.1 +sphinx-contributor-listing~=0.1 +sphinx-filtered-toctree~=0.1 +sphinx-related-links~=0.1 +sphinx-roles~=0.1 +sphinx-terminal~=1.0 +sphinx-ubuntu-images~=0.1 +sphinx-youtube-links~=0.1 # Other dependencies -packaging -sphinxcontrib-svg2pdfconverter[CairoSVG] -sphinx-last-updated-by-git -sphinx-sitemap +packaging~=26.1 +sphinxcontrib-svg2pdfconverter[CairoSVG]~=2.1 +sphinx-last-updated-by-git~=0.3 +sphinx-sitemap~=2.9 +sphinx-llm~=0.4 # Vale dependencies rst2html -vale +vale~=3.13 \ No newline at end of file