Misc documentation and build fixes

.gitignore

@@ -67,6 +67,7 @@ tests/tmp
 
 # doc build outputs
 docs/_build
+docs/_ext/__pycache__
 
 # coverage outputs
 *-coverage.info

README.rst

@@ -126,7 +126,7 @@ to a new file:
        return 0;
    }
 
-With libasdf installed on your system (see :ref:`development`) you can compile
+With libasdf installed on your system (see `Development`_) you can compile
 and run this test like:
 
 .. code:: console

changes/+broken-readme-link.doc

@@ -0,0 +1 @@
+Fixed broken link in the README to the "development" section.

changes/+disable-debug.bugfix

@@ -0,0 +1,2 @@
+Fixed incorrect handling of ``--disable-debug``, ``--disable-asan`` and
+``--disable-ubsan`` flags to ``configure``.

changes/+improved-index.doc

@@ -0,0 +1,6 @@
+Improved the documentation index such that entries are grouped ignoring the
+``asdf_`` prefix.
+
+For example, ``asdf_block_open()`` is grouped under "B", not "A" (previously
+almost all documented symbols are just grouped under "A" which is not as useful
+for a C library).

configure.ac

@@ -119,7 +119,7 @@ AM_CONDITIONAL([HAVE_LINKER_WRAP], [test "x$have_linker_wrap" = "xyes"])
 # --enable-debug
 AC_ARG_ENABLE([debug],
   [AS_HELP_STRING([--enable-debug], [Enable debug build (-g -O0)])],
-  [enable_debug=yes],
+  [],
   [enable_debug=no]
 )
 
@@ -145,7 +145,7 @@ AM_CONDITIONAL([ASDF_BUILD_TOOL], [test "x$asdf_build_tool" = xyes])
 # --with-asan
 AC_ARG_WITH([asan],
   [AS_HELP_STRING([--with-asan], [Build with AddressSanitizer support])],
-  [with_asan=yes],
+  [],
   [with_asan=no])
 
 AS_IF([test "x$with_asan" = "xyes"], [
@@ -162,7 +162,7 @@ AM_CONDITIONAL([HOST_LINUX], [case "$host_os" in linux*) true;; *) false;; esac]
 # --with-ubsan
 AC_ARG_WITH([ubsan],
   [AS_HELP_STRING([--with-ubsan], [Build with UndefinedBehaviorSanitizer support])],
-  [with_ubsan=yes],
+  [],
   [with_ubsan=no])
 
 AS_IF([test "x$with_ubsan" = "xyes"], [

docs/Makefile.am

@@ -1,8 +1,13 @@
 # CMake-related files to include in the distribution
 CMAKE_DIST = CMakeLists.txt
 
+# Local Sphinx extensions; see docs/_ext/
+SPHINX_EXT_DIST = \
+    _ext/index_grouping.py
+
 EXTRA_DIST = \
     $(CMAKE_DIST) \
+    $(SPHINX_EXT_DIST) \
     conf.py \
     api/asdf/block.h.rst \
     api/asdf/core/datatype.h.rst \

docs/_ext/index_grouping.py

@@ -0,0 +1,81 @@
+"""
+Group index entries by the first letter that actually distinguishes them.
+
+In a C library where there is only a global namespace, and nearly every symbol
+shares a prefix (i.e. ``asdf_``, ``ASDF_``, and so on) the generated index
+files all of them under a single letter and is close to useless for finding
+anything.
+
+Sphinx builds each index heading from the *category key* of an index entry: the
+fifth element of the ``(type, value, target_id, main, key)`` tuples the domains
+emit.  It falls back to the entry's first letter only when that key is ``None``
+(see ``sphinx.environment.adapters.indexentries._group_by_func``), and when the
+key is set it sorts by it too.  That hook exists for CJK indexes, where a
+first-letter heading is meaningless, but it serves just as well here.
+
+Setting it lets each symbol be filed under the first letter after its common
+prefix, so ``asdf_value_as_string`` lands under "V".  The entry text itself is
+left alone, so searching still matches the full name.
+
+Configuration
+-------------
+
+``index_strip_prefixes``
+    Prefixes to look past when choosing a heading, e.g.
+    ``['asdf_gwcs_', 'ASDF_GWCS_', 'asdf_', 'ASDF_']``.  Longest match wins.
+    Empty (the default) disables the extension.
+"""
+
+
+def _group_key(entry_text, prefixes):
+    """The letter to file an entry under, or None for Sphinx's default."""
+    # Index text looks like "asdf_value_as_string (C function)", and for a
+    # struct member "asdf_time_t.value (C member)".  Group members with their
+    # parent so a struct and its fields never land under different letters.
+    name = entry_text.split(' ', 1)[0].split('.', 1)[0].strip()
+
+    # Longest first, so 'asdf_gwcs_' wins over 'asdf_'.  A prefix that would
+    # leave a single character behind is skipped in favour of a shorter one:
+    # 'asdf_gwcs_t' is far easier to find under G, alongside its own members,
+    # than alone under T.
+    for prefix in sorted(prefixes, key=len, reverse=True):
+        if not name.startswith(prefix):
+            continue
+
+        rest = name[len(prefix):].lstrip('_')
+
+        if len(rest) > 1 and rest[:1].isalpha():
+            return rest[0].upper()
+
+    return None
+
+
+def _regroup_index_entries(app, env):
+    prefixes = app.config.index_strip_prefixes
+
+    if not prefixes:
+        return
+
+    domain = env.domains['index']
+
+    for docname, entries in domain.entries.items():
+        regrouped = []
+
+        for entry_type, value, target_id, main, category_key in entries:
+            if entry_type == 'single' and category_key is None:
+                category_key = _group_key(value, prefixes)
+
+            regrouped.append((entry_type, value, target_id, main, category_key))
+
+        domain.entries[docname] = regrouped
+
+
+def setup(app):
+    app.add_config_value('index_strip_prefixes', [], 'env')
+    app.connect('env-check-consistency', _regroup_index_entries)
+
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

docs/conf.py

@@ -3,9 +3,13 @@
 from datetime import datetime
 from pathlib import Path
 
+import sys
+
 from docutils.parsers.rst import directives
 from sphinx.directives.patches import Code
 
+sys.path.insert(0, str(Path(__file__).parent / '_ext'))
+
 
 # -- Project information ------------------------------------------------------
 def read_config_h() -> tuple[str, str, str]:
@@ -139,7 +143,13 @@ def read_config_h() -> tuple[str, str, str]:
     'numpy': ('https://numpy.org/doc/stable/', None)
 }
 
-extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.todo', 'hawkmoth']
+extensions = [
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'hawkmoth',
+    # Local; see docs/_ext/
+    'index_grouping',
+]
 
 # -- Options for hawkmoth extension --------------------------------------------
 
@@ -229,6 +239,11 @@ def _config_h_includedir():
 latex_logo = "_static/images/logo-light-mode.png"
 
 
+# -- Local extensions ----------------------------------------------------------
+# See docs/_ext/index_grouping.py
+index_strip_prefixes = ['asdf_', 'ASDF_']
+
+
 # -- Doc-example test directive options ----------------------------------------
 # The tests/scripts/extract_doc_examples.py script extracts ``.. code:: c``
 # blocks from the documentation and compiles/runs them as part of the test

docs/development.rst

@@ -5,7 +5,8 @@ Development resources
 
 This page covers building libasdf from a git checkout, the conventions the
 project follows, and how releases are made.  If you only want to *use* the
-library, the build instructions in the README *should* be sufficient.
+library, the build instructions in the :ref:`README <development>` *should*
+be sufficient.
 
 
 .. _build-systems:
@@ -38,19 +39,27 @@ distribution tarball* (the ``distcheck-cmake`` target in the top-level
 a file CMake needs, fails the autotools release check.  Both are also built and
 tested in CI, by the ``Build`` and ``CMake Build`` workflows respectively.
 
-The practical consequence: **when you add a source file, add it to both build
-systems.**
+The practical consequences:
 
-CMake is generally more permissive and picks up most new source files
-automatically, whereas automake tends to require anything you want built to be
-listed explicitly.
+* **When you add a source file, add it to both build systems.**  Both list
+  their sources explicitly--``src_files`` in the top-level ``Makefile.am`` and
+  ``libasdf_sources`` in ``src/CMakeLists.txt``.
+
+* **When you add a public header, add it to both install lists**--
+  ``include/Makefile.am`` and ``include/CMakeLists.txt``.  Drift between these
+  two breaks the *installed* library without breaking the build tree, so it
+  tends to be noticed late.
+
+* **When you add a documentation page, add it to EXTRA_DIST in
+  docs/Makefile.am**, or it will be missing from the release tarball and
+  the documentation build inside ``make distcheck`` will fail.
 
 
 Building with autotools
 =======================
 
 A git checkout has no ``configure`` script; generate it first with
-``autogen.sh`` (a one-line wrapper around ``autoreconf --install``).  This is
+``autogen.sh`` (a wrapper script around ``autoreconf --install``).  This is
 normally only needed once, or again after editing ``configure.ac`` or any
 ``Makefile.am``, though the generated makefiles normally re-run the necessary
 steps by themselves:
@@ -127,6 +136,10 @@ convenience:
 
 Other options of note:
 
+``-D ENABLE_TESTING_DOCS=YES``
+    Additionally build and run the example programs embedded in the
+    documentation (see `Documentation examples`_).
+
 ``-D ENABLE_TESTING_ALL=YES``
     Enable every test target, including the shell-based integration tests.
     This is what CI uses.
@@ -161,9 +174,11 @@ with some custom wrappers around it (helper macros) defined in
 ``tests/munit.h``.
 
 Test binaries are named ``test-<name>.unit`` and live in the ``tests/``
-directory of the build tree.  Some of them compile a subset of the sources
-directly, rather than linking against the library, so that internal components
-can be tested in isolation.
+directory of the build tree.  Tests that need to reach internals link against
+``libasdf_static.la``, a static convenience archive of the whole library, which
+makes ``ASDF_LOCAL`` (hidden-visibility) symbols reachable from the test binary.
+A couple of the narrower ones instead compile the single source under test
+directly, so that it can be exercised in isolation.
 
 From a build directory:
 
@@ -200,6 +215,50 @@ Two more targets, both requiring the corresponding ``configure`` option:
 - ``make check-code-coverage`` (``--enable-code-coverage``)
 
 
+.. _documentation examples:
+
+Documentation examples
+======================
+
+The example programs in ``README.rst`` and under ``docs/usage/`` are compiled
+and executed as part of the test suite, so they cannot drift away from the API.
+
+A code block opts in with the ``:test:`` option, naming the test, and may
+declare an input file with ``:fixture:``:
+
+.. code:: rst
+
+    .. code:: c
+       :test: test-open-close-file
+       :fixture: cube.asdf
+
+       #include <asdf.h>
+       ...
+
+``tests/scripts/extract_doc_examples.py`` pulls each marked block out into a
+``.c`` file under ``tests/doc_examples/``, compiles it against the freshly
+built library, and runs it with the resolved fixture path as its first
+argument.  A block with no ``:fixture:`` is run with no arguments; a fixture of
+``temp`` or ``temp:<name>`` resolves to a throwaway output path instead of an
+input file.
+
+The set of files scanned is listed explicitly, as ``DOC_EXAMPLE_FILES`` in
+``tests/Makefile.am`` and ``DOC_FILES`` in ``tests/CMakeLists.txt``; **a new
+documentation page containing examples must be added to both.**
+
+The examples are run for their exit status, not compared against the output
+quoted in the documentation.  When you change one, re-run it and paste its real
+output back into the surrounding prose:
+
+.. code:: console
+
+    $ make check
+    $ ./tests/doc_examples/test-open-close-file tests/fixtures/cube.asdf
+
+Under autotools this needs Python 3, and is skipped if none is found; under
+CMake it is gated on ``-D ENABLE_TESTING_DOCS=YES``.
+
+
 Code style
 ==========
 
@@ -260,6 +319,29 @@ CI builds the docs with ``-W``, so warnings are errors; if you add a page,
 make sure it is referenced from a ``toctree`` and that every cross-reference
 resolves.
 
+Two things about hawkmoth are worth knowing before writing header comments:
+
+* **An undocumented declaration is dropped entirely, and takes its members with
+  it.**  A struct whose fields all carry ``/** ... */`` comments will still be
+  absent from the rendered API unless the struct *itself* has one.  If a type
+  you expect is missing from the output, this is almost always why.
+
+* **A header that clang cannot parse loses its declarations silently.**  The
+  parse needs the same include path the compiler gets; when a declaration
+  vanishes from the output for no apparent reason, check that first.
+
+Because ``conf.py`` sets ``nitpicky = True`` and uses ``c:expr`` as the default
+role, *any* bare identifier written in single backticks is looked up in the C
+domain.  Standard C names have no inventory to resolve against and are listed
+in ``nitpick_ignore``; for anything else that is not a real API symbol--a file
+name, a schema name, a field mentioned in passing--use
+
+.. code:: rst
+
+    ``double backticks``
+
+instead.
+
 The ``asdf(1)`` man page is generated from ``docs/usage/cli.rst`` but is
 **committed to the repository** (as ``docs/man/asdf.1``), so that building or
 installing from a release tarball does not require Sphinx.  After changing
@@ -269,10 +351,6 @@ installing from a release tarball does not require Sphinx.  After changing
 
     $ make man-page
 
-Adding a new documentation page also means adding it to ``EXTRA_DIST`` in
-``docs/Makefile.am``, or it will be missing from the release tarball and the
-documentation build inside ``make distcheck`` will fail.
-
 
 Changelog entries
 =================

tests/CMakeLists.txt

@@ -53,6 +53,17 @@ add_compile_definitions(REFERENCE_FILES_DIR=\"${REFERENCE_FILES_DIR}\")
 add_compile_definitions(FIXTURES_DIR=\"${FIXTURES_DIR}\")
 add_compile_definitions(TEMP_DIR=\"${TEMP_DIR}\")
 
+# Helper used by the shell tests to discover the run directory chosen by
+# util.c's constructor
+add_executable(run-dir run-dir.c)
+target_include_directories(run-dir PRIVATE
+    ${CMAKE_BINARY_DIR}/include
+    ${CMAKE_SOURCE_DIR}/include
+)
+target_include_directories(run-dir SYSTEM PRIVATE ${STATGRAB_INCLUDEDIR})
+target_link_directories(run-dir PRIVATE ${STATGRAB_LIBDIR})
+target_link_libraries(run-dir PRIVATE util ${STATGRAB_LIBRARIES})
+
 set(runtime "WITH_CMAKE=set:YES;OBJC_DISABLE_INITIALIZE_FORK_SAFETY=set:YES;srcdir=set:${CMAKE_CURRENT_SOURCE_DIR};top_srcdir=set:${CMAKE_SOURCE_DIR};top_builddir=set:${CMAKE_BINARY_DIR};LD_LIBRARY_PATH=path_list_prepend:${CMAKE_BINARY_DIR}/src")
 
 set(sanitizer_options "ASAN_OPTIONS=detect_leaks=1:suppressions=${CMAKE_CURRENT_SOURCE_DIR}/libfyaml.asan.supp;LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/libfyaml.lsan.supp")