From 0e23a983f134e9ff44f2355899031815beb3cca5 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:44:52 +0200 Subject: [PATCH 1/5] docs: fix broken link in README.rst when rendered in GitHub --- README.rst | 2 +- changes/+broken-readme-link.doc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/+broken-readme-link.doc diff --git a/README.rst b/README.rst index 95462e7..43f1b4b 100644 --- a/README.rst +++ b/README.rst @@ -126,7 +126,7 @@ to a new file: return 0; } -With libasdf installed on your system (see :ref:`development`) you can compile +With libasdf installed on your system (see `Development`_) you can compile and run this test like: .. code:: console diff --git a/changes/+broken-readme-link.doc b/changes/+broken-readme-link.doc new file mode 100644 index 0000000..b578eec --- /dev/null +++ b/changes/+broken-readme-link.doc @@ -0,0 +1 @@ +Fixed broken link in the README to the "development" section. From 42552c0980cee9ca2f20e7b9d444a302723191c8 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:53:20 +0200 Subject: [PATCH 2/5] docs: improved index entry grouping ignoring common symbols prefixes Borrowed from libasdf-gwcs where I added this while working on its docs. --- .gitignore | 1 + changes/+improved-index.doc | 6 +++ docs/Makefile.am | 5 +++ docs/_ext/index_grouping.py | 81 +++++++++++++++++++++++++++++++++++++ docs/conf.py | 17 +++++++- 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 changes/+improved-index.doc create mode 100644 docs/_ext/index_grouping.py diff --git a/.gitignore b/.gitignore index a6fb0b2..a553af0 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,7 @@ tests/tmp # doc build outputs docs/_build +docs/_ext/__pycache__ # coverage outputs *-coverage.info diff --git a/changes/+improved-index.doc b/changes/+improved-index.doc new file mode 100644 index 0000000..76a574a --- /dev/null +++ b/changes/+improved-index.doc @@ -0,0 +1,6 @@ +Improved the documentation index such that entries are grouped ignoring the +``asdf_`` prefix. + +For example, ``asdf_block_open()`` is grouped under "B", not "A" (previously +almost all documented symbols are just grouped under "A" which is not as useful +for a C library). diff --git a/docs/Makefile.am b/docs/Makefile.am index 23abf98..ac5a4e2 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -1,8 +1,13 @@ # CMake-related files to include in the distribution CMAKE_DIST = CMakeLists.txt +# Local Sphinx extensions; see docs/_ext/ +SPHINX_EXT_DIST = \ + _ext/index_grouping.py + EXTRA_DIST = \ $(CMAKE_DIST) \ + $(SPHINX_EXT_DIST) \ conf.py \ api/asdf/block.h.rst \ api/asdf/core/datatype.h.rst \ diff --git a/docs/_ext/index_grouping.py b/docs/_ext/index_grouping.py new file mode 100644 index 0000000..c5478cd --- /dev/null +++ b/docs/_ext/index_grouping.py @@ -0,0 +1,81 @@ +""" +Group index entries by the first letter that actually distinguishes them. + +In a C library where there is only a global namespace, and nearly every symbol +shares a prefix (i.e. ``asdf_``, ``ASDF_``, and so on) the generated index +files all of them under a single letter and is close to useless for finding +anything. + +Sphinx builds each index heading from the *category key* of an index entry: the +fifth element of the ``(type, value, target_id, main, key)`` tuples the domains +emit. It falls back to the entry's first letter only when that key is ``None`` +(see ``sphinx.environment.adapters.indexentries._group_by_func``), and when the +key is set it sorts by it too. That hook exists for CJK indexes, where a +first-letter heading is meaningless, but it serves just as well here. + +Setting it lets each symbol be filed under the first letter after its common +prefix, so ``asdf_value_as_string`` lands under "V". The entry text itself is +left alone, so searching still matches the full name. + +Configuration +------------- + +``index_strip_prefixes`` + Prefixes to look past when choosing a heading, e.g. + ``['asdf_gwcs_', 'ASDF_GWCS_', 'asdf_', 'ASDF_']``. Longest match wins. + Empty (the default) disables the extension. +""" + + +def _group_key(entry_text, prefixes): + """The letter to file an entry under, or None for Sphinx's default.""" + # Index text looks like "asdf_value_as_string (C function)", and for a + # struct member "asdf_time_t.value (C member)". Group members with their + # parent so a struct and its fields never land under different letters. + name = entry_text.split(' ', 1)[0].split('.', 1)[0].strip() + + # Longest first, so 'asdf_gwcs_' wins over 'asdf_'. A prefix that would + # leave a single character behind is skipped in favour of a shorter one: + # 'asdf_gwcs_t' is far easier to find under G, alongside its own members, + # than alone under T. + for prefix in sorted(prefixes, key=len, reverse=True): + if not name.startswith(prefix): + continue + + rest = name[len(prefix):].lstrip('_') + + if len(rest) > 1 and rest[:1].isalpha(): + return rest[0].upper() + + return None + + +def _regroup_index_entries(app, env): + prefixes = app.config.index_strip_prefixes + + if not prefixes: + return + + domain = env.domains['index'] + + for docname, entries in domain.entries.items(): + regrouped = [] + + for entry_type, value, target_id, main, category_key in entries: + if entry_type == 'single' and category_key is None: + category_key = _group_key(value, prefixes) + + regrouped.append((entry_type, value, target_id, main, category_key)) + + domain.entries[docname] = regrouped + + +def setup(app): + app.add_config_value('index_strip_prefixes', [], 'env') + app.connect('env-check-consistency', _regroup_index_entries) + + return { + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/conf.py b/docs/conf.py index b55898a..b93d7dd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -3,9 +3,13 @@ from datetime import datetime from pathlib import Path +import sys + from docutils.parsers.rst import directives from sphinx.directives.patches import Code +sys.path.insert(0, str(Path(__file__).parent / '_ext')) + # -- Project information ------------------------------------------------------ def read_config_h() -> tuple[str, str, str]: @@ -139,7 +143,13 @@ def read_config_h() -> tuple[str, str, str]: 'numpy': ('https://numpy.org/doc/stable/', None) } -extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.todo', 'hawkmoth'] +extensions = [ + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'hawkmoth', + # Local; see docs/_ext/ + 'index_grouping', +] # -- Options for hawkmoth extension -------------------------------------------- @@ -229,6 +239,11 @@ def _config_h_includedir(): latex_logo = "_static/images/logo-light-mode.png" +# -- Local extensions ---------------------------------------------------------- +# See docs/_ext/index_grouping.py +index_strip_prefixes = ['asdf_', 'ASDF_'] + + # -- Doc-example test directive options ---------------------------------------- # The tests/scripts/extract_doc_examples.py script extracts ``.. code:: c`` # blocks from the documentation and compiles/runs them as part of the test From b4c477bb2c304be42f9ee2945b559b2118124dd5 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:54:06 +0200 Subject: [PATCH 3/5] docs: sync up development.rst with libasdf-gwcs In the libasdf-gwcs docs I also copied libasdf's development.rst and made some improvements to it--those improvements, such that are applicable also in libasdf, I tried to sync back up here. --- docs/development.rst | 106 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 14 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 2ca9986..4a95609 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -5,7 +5,8 @@ Development resources This page covers building libasdf from a git checkout, the conventions the project follows, and how releases are made. If you only want to *use* the -library, the build instructions in the README *should* be sufficient. +library, the build instructions in the :ref:`README ` *should* +be sufficient. .. _build-systems: @@ -38,19 +39,27 @@ distribution tarball* (the ``distcheck-cmake`` target in the top-level a file CMake needs, fails the autotools release check. Both are also built and tested in CI, by the ``Build`` and ``CMake Build`` workflows respectively. -The practical consequence: **when you add a source file, add it to both build -systems.** +The practical consequences: -CMake is generally more permissive and picks up most new source files -automatically, whereas automake tends to require anything you want built to be -listed explicitly. +* **When you add a source file, add it to both build systems.** Both list + their sources explicitly--``src_files`` in the top-level ``Makefile.am`` and + ``libasdf_sources`` in ``src/CMakeLists.txt``. + +* **When you add a public header, add it to both install lists**-- + ``include/Makefile.am`` and ``include/CMakeLists.txt``. Drift between these + two breaks the *installed* library without breaking the build tree, so it + tends to be noticed late. + +* **When you add a documentation page, add it to EXTRA_DIST in + docs/Makefile.am**, or it will be missing from the release tarball and + the documentation build inside ``make distcheck`` will fail. Building with autotools ======================= A git checkout has no ``configure`` script; generate it first with -``autogen.sh`` (a one-line wrapper around ``autoreconf --install``). This is +``autogen.sh`` (a wrapper script around ``autoreconf --install``). This is normally only needed once, or again after editing ``configure.ac`` or any ``Makefile.am``, though the generated makefiles normally re-run the necessary steps by themselves: @@ -127,6 +136,10 @@ convenience: Other options of note: +``-D ENABLE_TESTING_DOCS=YES`` + Additionally build and run the example programs embedded in the + documentation (see `Documentation examples`_). + ``-D ENABLE_TESTING_ALL=YES`` Enable every test target, including the shell-based integration tests. This is what CI uses. @@ -161,9 +174,11 @@ with some custom wrappers around it (helper macros) defined in ``tests/munit.h``. Test binaries are named ``test-.unit`` and live in the ``tests/`` -directory of the build tree. Some of them compile a subset of the sources -directly, rather than linking against the library, so that internal components -can be tested in isolation. +directory of the build tree. Tests that need to reach internals link against +``libasdf_static.la``, a static convenience archive of the whole library, which +makes ``ASDF_LOCAL`` (hidden-visibility) symbols reachable from the test binary. +A couple of the narrower ones instead compile the single source under test +directly, so that it can be exercised in isolation. From a build directory: @@ -200,6 +215,50 @@ Two more targets, both requiring the corresponding ``configure`` option: - ``make check-code-coverage`` (``--enable-code-coverage``) +.. _documentation examples: + +Documentation examples +====================== + +The example programs in ``README.rst`` and under ``docs/usage/`` are compiled +and executed as part of the test suite, so they cannot drift away from the API. + +A code block opts in with the ``:test:`` option, naming the test, and may +declare an input file with ``:fixture:``: + +.. code:: rst + + .. code:: c + :test: test-open-close-file + :fixture: cube.asdf + + #include + ... + +``tests/scripts/extract_doc_examples.py`` pulls each marked block out into a +``.c`` file under ``tests/doc_examples/``, compiles it against the freshly +built library, and runs it with the resolved fixture path as its first +argument. A block with no ``:fixture:`` is run with no arguments; a fixture of +``temp`` or ``temp:`` resolves to a throwaway output path instead of an +input file. + +The set of files scanned is listed explicitly, as ``DOC_EXAMPLE_FILES`` in +``tests/Makefile.am`` and ``DOC_FILES`` in ``tests/CMakeLists.txt``; **a new +documentation page containing examples must be added to both.** + +The examples are run for their exit status, not compared against the output +quoted in the documentation. When you change one, re-run it and paste its real +output back into the surrounding prose: + +.. code:: console + + $ make check + $ ./tests/doc_examples/test-open-close-file tests/fixtures/cube.asdf + +Under autotools this needs Python 3, and is skipped if none is found; under +CMake it is gated on ``-D ENABLE_TESTING_DOCS=YES``. + + Code style ========== @@ -260,6 +319,29 @@ CI builds the docs with ``-W``, so warnings are errors; if you add a page, make sure it is referenced from a ``toctree`` and that every cross-reference resolves. +Two things about hawkmoth are worth knowing before writing header comments: + +* **An undocumented declaration is dropped entirely, and takes its members with + it.** A struct whose fields all carry ``/** ... */`` comments will still be + absent from the rendered API unless the struct *itself* has one. If a type + you expect is missing from the output, this is almost always why. + +* **A header that clang cannot parse loses its declarations silently.** The + parse needs the same include path the compiler gets; when a declaration + vanishes from the output for no apparent reason, check that first. + +Because ``conf.py`` sets ``nitpicky = True`` and uses ``c:expr`` as the default +role, *any* bare identifier written in single backticks is looked up in the C +domain. Standard C names have no inventory to resolve against and are listed +in ``nitpick_ignore``; for anything else that is not a real API symbol--a file +name, a schema name, a field mentioned in passing--use + +.. code:: rst + + ``double backticks`` + +instead. + The ``asdf(1)`` man page is generated from ``docs/usage/cli.rst`` but is **committed to the repository** (as ``docs/man/asdf.1``), so that building or installing from a release tarball does not require Sphinx. After changing @@ -269,10 +351,6 @@ installing from a release tarball does not require Sphinx. After changing $ make man-page -Adding a new documentation page also means adding it to ``EXTRA_DIST`` in -``docs/Makefile.am``, or it will be missing from the release tarball and the -documentation build inside ``make distcheck`` will fail. - Changelog entries ================= From 74bf11d26d85bb0a15b40c28b7b68c823bf84b4d Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:57:03 +0200 Subject: [PATCH 4/5] build: fix incorrect handling of --disable-debug and so on I had forgotten the third argument to AC_ARG_ENABLE/WITH is *not* the "enabled" case, it's whether an --enable-foo or --disable-foo was given at all as opposed to absent (the 4th argument). This often trips me up. --- changes/+disable-debug.bugfix | 2 ++ configure.ac | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changes/+disable-debug.bugfix diff --git a/changes/+disable-debug.bugfix b/changes/+disable-debug.bugfix new file mode 100644 index 0000000..481edd3 --- /dev/null +++ b/changes/+disable-debug.bugfix @@ -0,0 +1,2 @@ +Fixed incorrect handling of ``--disable-debug``, ``--disable-asan`` and +``--disable-ubsan`` flags to ``configure``. diff --git a/configure.ac b/configure.ac index 03040e3..ea1917a 100644 --- a/configure.ac +++ b/configure.ac @@ -119,7 +119,7 @@ AM_CONDITIONAL([HAVE_LINKER_WRAP], [test "x$have_linker_wrap" = "xyes"]) # --enable-debug AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], [Enable debug build (-g -O0)])], - [enable_debug=yes], + [], [enable_debug=no] ) @@ -145,7 +145,7 @@ AM_CONDITIONAL([ASDF_BUILD_TOOL], [test "x$asdf_build_tool" = xyes]) # --with-asan AC_ARG_WITH([asan], [AS_HELP_STRING([--with-asan], [Build with AddressSanitizer support])], - [with_asan=yes], + [], [with_asan=no]) AS_IF([test "x$with_asan" = "xyes"], [ @@ -162,7 +162,7 @@ AM_CONDITIONAL([HOST_LINUX], [case "$host_os" in linux*) true;; *) false;; esac] # --with-ubsan AC_ARG_WITH([ubsan], [AS_HELP_STRING([--with-ubsan], [Build with UndefinedBehaviorSanitizer support])], - [with_ubsan=yes], + [], [with_ubsan=no]) AS_IF([test "x$with_ubsan" = "xyes"], [ From 4ddf5cc168ade1509af3b4e890af4e636d1a62b4 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:12:38 +0200 Subject: [PATCH 5/5] test: fix race condition when running the tests with `make -j check` The destructor that cleaned up the test directory at the end of a test executable was *too* destructive and could lead to race conditions where the test directory gets deleted between test programs like: A: get_run_dir() (creates run dir) B: get_run_dir() (reuses existing run dir) A: finishes (deletes run dir) B: tries to write file to run_dir (gets ENOENT) Better to just not delete them, even if they happen to be empty. The `latest` symlink will still point to the most recent. This makes a few additional attempts as well to more carefully avoid race conditions (e.g. a partially written or empty coordination file). It also adds a `run-dir` helper program that can be used in the shell tests so that they also write cleanly into per-test-run directories. --- tests/CMakeLists.txt | 11 ++++++ tests/Makefile.am | 15 ++++++-- tests/run-dir.c | 19 ++++++++++ tests/shell-test.sh | 22 ++++++++++-- tests/util.c | 82 +++++++++++++++++++++++++++----------------- 5 files changed, 113 insertions(+), 36 deletions(-) create mode 100644 tests/run-dir.c diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 291af02..31df42b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -53,6 +53,17 @@ add_compile_definitions(REFERENCE_FILES_DIR=\"${REFERENCE_FILES_DIR}\") add_compile_definitions(FIXTURES_DIR=\"${FIXTURES_DIR}\") add_compile_definitions(TEMP_DIR=\"${TEMP_DIR}\") +# Helper used by the shell tests to discover the run directory chosen by +# util.c's constructor +add_executable(run-dir run-dir.c) +target_include_directories(run-dir PRIVATE + ${CMAKE_BINARY_DIR}/include + ${CMAKE_SOURCE_DIR}/include +) +target_include_directories(run-dir SYSTEM PRIVATE ${STATGRAB_INCLUDEDIR}) +target_link_directories(run-dir PRIVATE ${STATGRAB_LIBDIR}) +target_link_libraries(run-dir PRIVATE util ${STATGRAB_LIBRARIES}) + set(runtime "WITH_CMAKE=set:YES;OBJC_DISABLE_INITIALIZE_FORK_SAFETY=set:YES;srcdir=set:${CMAKE_CURRENT_SOURCE_DIR};top_srcdir=set:${CMAKE_SOURCE_DIR};top_builddir=set:${CMAKE_BINARY_DIR};LD_LIBRARY_PATH=path_list_prepend:${CMAKE_BINARY_DIR}/src") set(sanitizer_options "ASAN_OPTIONS=detect_leaks=1:suppressions=${CMAKE_CURRENT_SOURCE_DIR}/libfyaml.asan.supp;LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/libfyaml.lsan.supp") diff --git a/tests/Makefile.am b/tests/Makefile.am index 6e1b775..40892c9 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -26,7 +26,7 @@ $(asdf_standard_submodule) $(munit_submodule): @VALGRIND_CHECK_RULES@ -check_PROGRAMS = \ +unit_tests = \ test-block.unit \ test-compression.unit \ test-core-extensions.unit \ @@ -48,6 +48,10 @@ check_PROGRAMS = \ test-version.unit \ test-yaml.unit +# run-dir is a helper for the shell tests, not a test itself, so it goes in +# check_PROGRAMS (built by `make check`) but not in TESTS. +check_PROGRAMS = $(unit_tests) run-dir + TESTS_ENVIRONMENT = top_builddir=$(top_builddir) top_srcdir=$(top_srcdir) if UBSAN @@ -72,7 +76,7 @@ TESTS_ENVIRONMENT += \ endif endif -TESTS = $(check_PROGRAMS) +TESTS = $(unit_tests) unit_test_base_cppflags = \ -DREFERENCE_FILES_DIR=\"$(asdf_standard_dir)/reference_files\" \ @@ -109,6 +113,12 @@ libmunit_a_SOURCES = util.c $(munit_dir)/munit.c libmunit_a_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include $(unit_test_base_cppflags) libmunit_a_CFLAGS = $(unit_test_cflags) -w +# run-dir (shell test helper, not a test) +run_dir_SOURCES = run-dir.c +run_dir_CPPFLAGS = $(unit_test_cppflags) +run_dir_CFLAGS = $(unit_test_cflags) +run_dir_LDADD = $(munit_ldflags) $(ASDF_LIBS) + # test-block.unit test_block_unit_SOURCES = test-block.c test_block_unit_CPPFLAGS = $(unit_test_cppflags) @@ -195,6 +205,7 @@ if HAVE_LINKER_WRAP # static archive (rather than the shared libasdf) lets the --wrap flags # intercept malloc/calloc/strdup/strndup calls anywhere in the library, not # just in a hand-maintained subset of translation units. +unit_tests += test-malloc-fail.unit check_PROGRAMS += test-malloc-fail.unit test_malloc_fail_unit_SOURCES = test-malloc-fail.c test_malloc_fail_unit_CPPFLAGS = $(unit_test_cppflags) diff --git a/tests/run-dir.c b/tests/run-dir.c new file mode 100644 index 0000000..f9db930 --- /dev/null +++ b/tests/run-dir.c @@ -0,0 +1,19 @@ +/** + * Print the temporary directory for the current test run. + * + * The run directory is selected by a constructor in util.c and shared by + * every process in the same process group (see the comment there), so the + * path printed here is the same one the unit test binaries of the current + * `make check` are using. This lets the shell tests write their output + * alongside the unit tests' temp files. + */ + +#include + +#include "util.h" + + +int main(void) { + printf("%s\n", get_run_dir()); + return 0; +} diff --git a/tests/shell-test.sh b/tests/shell-test.sh index 8b749e7..7753dda 100755 --- a/tests/shell-test.sh +++ b/tests/shell-test.sh @@ -53,12 +53,23 @@ fi fixtures_dir="${srcdir}/fixtures/${SUBCOMMAND}" +# Write output to the same per-run directory the unit tests use. The helper +# joins the current run by process group; fall back to tmp/ if it is missing +# (e.g. running this script by hand from an unbuilt tree). +run_dir_prog="${top_builddir}/tests/run-dir" + +if [ -x "${run_dir_prog}" ]; then + run_dir=$("${run_dir_prog}") +else + run_dir="$(pwd)/tmp" +fi + +mkdir -p "${run_dir}" + for input in $@; do base=$(basename "$input" .asdf) expected="${fixtures_dir}/${base}.${SUBCOMMAND}.txt" - actual="$(pwd)/tmp/${base}.${SUBCOMMAND}.out.txt" - - mkdir -p tmp + actual="${run_dir}/${base}.${SUBCOMMAND}.out.txt" asdfprog="${top_builddir}/asdf" if [ "x${WITH_CMAKE}" != "x" ]; then @@ -77,6 +88,11 @@ for input in $@; do fail=1 else echo "✅ Test passed: $base" + # Keep the output of failing tests for inspection; discard it otherwise, + # matching the unit tests' teardown. + if [ -z "${ASDF_TEST_KEEP_TEMP}" ]; then + rm -f "$actual" + fi fi fi done diff --git a/tests/util.c b/tests/util.c index b566894..97b0af1 100644 --- a/tests/util.c +++ b/tests/util.c @@ -89,6 +89,12 @@ static void ensure_tmp_dir(void) { * At startup each binary also lazily removes coordination files whose process * groups are no longer alive (kill(-pgid, 0) == ESRCH). * + * The run directory is never removed while the run is in progress. It is + * shared by every binary in the run (and, because munit forks a child per + * test, by every one of those children). + * + * `make distclean` removes tmp/ entirely. + * * Note: in non-interactive shells without job control, `make` inherits the * invoking shell's PGID rather than creating its own, so two sequential * `make check` calls in the same shell session may share a PGID. The stale @@ -142,7 +148,9 @@ static int join_existing_run(const char *pgid_file) { ssize_t n = read(fd, serial_str, sizeof(serial_str) - 1); close(fd); - if (n <= 0) + /* A short read means the pioneer has created the coordination file but + * has not yet finished writing to it; treat it as "not ready". */ + if (n != TEST_SERIAL_LEN) return 0; snprintf(run_dir_storage, sizeof(run_dir_storage), TEMP_DIR "/%s", serial_str); @@ -150,22 +158,33 @@ static int join_existing_run(const char *pgid_file) { } -#define WAIT_FOR_PIONEER_ATTEMPTS 200 +#define WAIT_FOR_PIONEER_ATTEMPTS 1000 #define WAIT_FOR_PIONEER_DELAY 5000 // usec +#define PIONEER_JOINED 1 +#define PIONEER_GONE 0 +#define PIONEER_TIMEOUT (-1) + /* * Retry joining a run after losing the O_EXCL race to the pioneer. * Polls the coordination file with a short backoff until the pioneer * writes the serial. + * + * Returns PIONEER_JOINED if the serial was read, PIONEER_GONE if the + * coordination file vanished (the pioneer failed, so the caller should try + * to claim it), or PIONEER_TIMEOUT if the pioneer never published one. */ -static void wait_for_pioneer(const char *pgid_file) { +static int wait_for_pioneer(const char *pgid_file) { for (int attempt = 0; attempt < WAIT_FOR_PIONEER_ATTEMPTS; attempt++) { usleep(WAIT_FOR_PIONEER_DELAY); if (join_existing_run(pgid_file)) - return; + return PIONEER_JOINED; + if (access(pgid_file, F_OK) == -1 && errno == ENOENT) + return PIONEER_GONE; } /* Timed out; get_run_dir() falls back to TEMP_DIR. */ + return PIONEER_TIMEOUT; } @@ -278,6 +297,9 @@ static void pioneer_setup(int fd_create, const char *pgid_file) { } +#define CLAIM_RUN_ATTEMPTS 3 + + __attribute__((constructor)) static void init_run_dir(void) { ensure_tmp_dir(); @@ -287,30 +309,26 @@ static void init_run_dir(void) { char pgid_file[PATH_MAX]; snprintf(pgid_file, sizeof(pgid_file), TEMP_DIR "/" PGID_FILE_TEMPLATE, (int)pgid); - /* Follower: join an existing run for this PGID. */ - if (join_existing_run(pgid_file)) - return; - - /* Pioneer: atomically claim the coordination file. */ - int fd_create = open(pgid_file, O_WRONLY | O_CREAT | O_EXCL, 0600); - if (fd_create < 0) { - if (errno == EEXIST) - wait_for_pioneer(pgid_file); /* lost the race; become a follower */ - return; - } - - pioneer_setup(fd_create, pgid_file); -} + for (int attempt = 0; attempt < CLAIM_RUN_ATTEMPTS; attempt++) { + /* Follower: join an existing run for this PGID. */ + if (join_existing_run(pgid_file)) + return; + /* Pioneer: atomically claim the coordination file. */ + int fd_create = open(pgid_file, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (fd_create >= 0) { + pioneer_setup(fd_create, pgid_file); + return; + } + if (errno != EEXIST) + return; -/* On exit, attempt to remove the run directory if it is empty. This is - * best-effort: rmdir silently fails on a non-empty directory, and with - * parallel test execution (make -jN) multiple binaries share the directory, - * so only the last binary to exit will succeed if no files were left. */ -__attribute__((destructor)) -static void cleanup_run_dir(void) { - if (run_dir_storage[0]) - rmdir(run_dir_storage); /* no-op if non-empty */ + /* Lost the race; wait for the pioneer to publish the serial. If the + * pioneer failed it removes the coordination file, in which case we + * loop around and try to claim it ourselves. */ + if (wait_for_pioneer(pgid_file) != PIONEER_GONE) + return; + } } @@ -336,12 +354,14 @@ const char *get_temp_file_path(const char *prefix, const char *suffix) { if (n < 0 || n >= (int)sizeof(fullpath)) return NULL; - /* Ensure the run directory exists: the destructor may have removed it if - * it was empty between the last test binary's exit and this call. */ - mkdir(get_run_dir(), 0777); /* no-op if it already exists */ - - /* Create the file so it exists (matching the old mkstemp-based behaviour). */ + /* Create the file so it exists (matching the old mkstemp-based behaviour). + * The run directory should already exist, but recreate it and retry once + * if something outside the test run removed it. */ int fd = open(fullpath, O_CREAT | O_WRONLY, 0600); + if (fd < 0 && errno == ENOENT) { + mkdir(get_run_dir(), 0777); + fd = open(fullpath, O_CREAT | O_WRONLY, 0600); + } if (fd >= 0) close(fd);