Backport upstream doctest fixes and add pytest 7/8/9 compatibility (#56)

why: Fix critical bugs in autouse fixture discovery and doctest
directive parsing. Add pytest version compatibility with new API
support.

what:
- Backport pytest autouse fixtures fix (9cd14b4ff, 2024-02-06)
- Backport Sphinx doctestopt_re whitespace fix (ad0c343d3, 2025-01-04)
- Add _unblock_doctest() helper for pytest 8.1+ unblock() API
- Add pytest 9.x to CI test matrix with 7.x/8.x compatibility
- Remove dead _from_module() method from DocutilsDocTestFinder
- Add upstream audit report for CPython, pytest, and Sphinx

Author: tony

.github/workflows/tests.yml

@@ -11,6 +11,11 @@ jobs:
       matrix:
         python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
         docutils-version: ['0.18', '0.19']
+        pytest-version: ['7', '8', '9']
+        exclude:
+          # Exclude pytest 7 from Python 3.14 to reduce matrix size
+          - python-version: '3.14'
+            pytest-version: '7'
     steps:
       - uses: actions/checkout@v5
 
@@ -25,10 +30,14 @@ jobs:
       - name: Install dependencies
         run: uv sync --all-extras --dev
 
-      - name: Print python versions
+      - name: Install specific pytest version
+        run: uv pip install "pytest~=${{ matrix.pytest-version }}.0"
+
+      - name: Print python and pytest versions
         run: |
           python -V
           uv run python -V
+          uv run pytest --version
 
       - name: Lint with ruff check
         run: uv run ruff check .

.gitignore

@@ -83,7 +83,6 @@ pip-wheel-metadata/
 monkeytype.sqlite3
 
 # Claude code
-**/CLAUDE.md
 **/CLAUDE.local.md
 **/CLAUDE.*.md
 **/.claude/settings.local.json

CHANGES

@@ -28,7 +28,32 @@ $ uvx --from 'gp-libs' --prerelease allow gp-libs
 
 ## gp-libs 0.0.16 (unreleased)
 
-- _Add your latest changes from PRs here_
+### Features
+
+#### pytest_doctest_docutils
+
+- Add `_unblock_doctest()` helper for programmatic re-enabling of built-in doctest plugin (#56)
+
+  Uses the public `pluginmanager.unblock()` API introduced in pytest 8.1.0, with graceful
+  fallback for older versions.
+
+### Bug fixes
+
+#### pytest_doctest_docutils
+
+- Autouse fixtures from `conftest.py` are now properly discovered for doctest files (#56)
+
+  Backported from pytest commit [9cd14b4ff](https://github.com/pytest-dev/pytest/commit/9cd14b4ff) (2024-02-06).
+
+#### doctest_docutils
+
+- Doctest directive comments with leading whitespace (e.g., `  # doctest: +SKIP`) are now properly matched (#56)
+
+  Backported from Sphinx commit [ad0c343d3](https://github.com/sphinx-doc/sphinx/commit/ad0c343d3) (2025-01-04).
+
+### Development
+
+- CI: Add pytest 9.x to test matrix, with pytest 7.x/8.x compatibility testing (#56)
 
 ## gp-libs 0.0.15 (2025-11-01)
 

notes/2025-11-25-upstream-backports.md

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import doctest
-import functools
 import linecache
 import logging
 import os
@@ -30,7 +29,10 @@
 
 
 blankline_re = re.compile(r"^\s*<BLANKLINE>", re.MULTILINE)
-doctestopt_re = re.compile(r"#\s*doctest:.+$", re.MULTILINE)
+# Backported from Sphinx commit ad0c343d3 (2025-01-04).
+# https://github.com/sphinx-doc/sphinx/commit/ad0c343d3
+# Allow optional leading whitespace before doctest directive comments.
+doctestopt_re = re.compile(r"[ \t]*#\s*doctest:.+$", re.MULTILINE)
 
 
 def is_allowed_version(version: str, spec: str) -> bool:
@@ -399,32 +401,6 @@ def condition(node: Node) -> bool:
             if test is not None:
                 tests.append(test)
 
-    if sys.version_info < (3, 13):
-
-        def _from_module(
-            self,
-            module: str | types.ModuleType | None,
-            object: object,  # NOQA: A002
-        ) -> bool:
-            """Return true if the given object lives in the given module.
-
-            `cached_property` objects are never considered a part
-            of the 'current module'. As such they are skipped by doctest.
-            Here we override `_from_module` to check the underlying
-            function instead. https://github.com/python/cpython/issues/107995.
-            """
-            if isinstance(object, functools.cached_property):
-                object = object.func  # noqa: A001
-
-            # Type ignored because this is a private function.
-            return t.cast(
-                "bool",
-                super()._from_module(module, object),  # type:ignore[misc]
-            )
-
-    else:  # pragma: no cover
-        pass
-
     def _get_test(
         self,
         string: str,

src/doctest_docutils.py

@@ -37,6 +37,9 @@
 
 logger = logging.getLogger(__name__)
 
+# Parse pytest version for version-specific features
+PYTEST_VERSION = tuple(int(x) for x in pytest.__version__.split(".")[:2])
+
 # Lazy definition of runner class
 RUNNER_CLASS = None
 
@@ -68,6 +71,28 @@ def pytest_configure(config: pytest.Config) -> None:
         config.pluginmanager.set_blocked("doctest")
 
 
+def _unblock_doctest(config: pytest.Config) -> bool:
+    """Unblock doctest plugin (pytest 8.1+ only).
+
+    Re-enables the built-in doctest plugin after it was blocked by
+    pytest_configure. Uses the public unblock() API introduced in pytest 8.1.0.
+
+    Parameters
+    ----------
+    config : pytest.Config
+        The pytest configuration object
+
+    Returns
+    -------
+    bool
+        True if unblocked successfully, False if API not available
+    """
+    pm = config.pluginmanager
+    if PYTEST_VERSION >= (8, 1) and hasattr(pm, "unblock"):
+        return pm.unblock("doctest")
+    return False
+
+
 def pytest_unconfigure() -> None:
     """Unconfigure hook for pytest-doctest-docutils."""
     global RUNNER_CLASS
@@ -306,6 +331,12 @@ def collect(self) -> Iterable[DoctestItem]:
         # Uses internal doctest module parsing mechanism.
         finder = DocutilsDocTestFinder()
 
+        # While doctests in .rst/.md files don't support fixtures directly,
+        # we still need to pick up autouse fixtures.
+        # Backported from pytest commit 9cd14b4ff (2024-02-06).
+        # https://github.com/pytest-dev/pytest/commit/9cd14b4ff
+        self.session._fixturemanager.parsefactories(self)
+
         optionflags = get_optionflags(self.config)
 
         runner = _get_runner(

src/pytest_doctest_docutils.py

@@ -0,0 +1,136 @@
+"""Regression test for autouse fixtures with doctest files.
+
+Backported from pytest commit 9cd14b4ff (2024-02-06).
+https://github.com/pytest-dev/pytest/commit/9cd14b4ff
+
+The original pytest test verified autouse fixtures defined in the same .py module
+as the doctest get picked up properly. For gp-libs, DocTestDocutilsFile handles
+.rst/.md files where fixtures can only come from conftest.py (not from the
+document itself).
+
+This test verifies that autouse fixtures from conftest.py are properly discovered
+for .rst and .md doctest files across different fixture scopes.
+
+Refs: pytest-dev/pytest#11929
+"""
+
+from __future__ import annotations
+
+import textwrap
+import typing as t
+
+import _pytest.pytester
+import pytest
+
+
+class AutouseFixtureTestCase(t.NamedTuple):
+    """Test fixture for autouse fixtures with doctest files."""
+
+    test_id: str
+    scope: str
+    file_ext: str
+    file_content: str
+
+
+RST_DOCTEST_CONTENT = textwrap.dedent(
+    """
+Example
+=======
+
+.. doctest::
+
+   >>> get_value()
+   'fixture ran'
+    """,
+)
+
+MD_DOCTEST_CONTENT = textwrap.dedent(
+    """
+# Example
+
+```{doctest}
+>>> get_value()
+'fixture ran'
+```
+    """,
+)
+
+SCOPES = ["module", "session", "class", "function"]
+
+FIXTURES = [
+    AutouseFixtureTestCase(
+        test_id=f"{scope}-rst",
+        scope=scope,
+        file_ext=".rst",
+        file_content=RST_DOCTEST_CONTENT,
+    )
+    for scope in SCOPES
+] + [
+    AutouseFixtureTestCase(
+        test_id=f"{scope}-md",
+        scope=scope,
+        file_ext=".md",
+        file_content=MD_DOCTEST_CONTENT,
+    )
+    for scope in SCOPES
+]
+
+
+@pytest.mark.parametrize(
+    AutouseFixtureTestCase._fields,
+    FIXTURES,
+    ids=[f.test_id for f in FIXTURES],
+)
+def test_autouse_fixtures_with_doctest_files(
+    pytester: _pytest.pytester.Pytester,
+    test_id: str,
+    scope: str,
+    file_ext: str,
+    file_content: str,
+) -> None:
+    """Autouse fixtures from conftest.py work with .rst/.md doctest files.
+
+    Regression test for pytest-dev/pytest#11929.
+    Backported from pytest commit 9cd14b4ff (2024-02-06).
+    """
+    pytester.plugins = ["pytest_doctest_docutils"]
+    pytester.makefile(
+        ".ini",
+        pytest=textwrap.dedent(
+            """
+[pytest]
+addopts=-p no:doctest -vv
+            """.strip(),
+        ),
+    )
+
+    # Create conftest with autouse fixture that sets a global value
+    pytester.makeconftest(
+        textwrap.dedent(
+            f"""
+import pytest
+
+VALUE = "fixture did not run"
+
+@pytest.fixture(autouse=True, scope="{scope}")
+def set_value():
+    global VALUE
+    VALUE = "fixture ran"
+
+@pytest.fixture(autouse=True)
+def add_get_value_to_doctest_namespace(doctest_namespace):
+    def get_value():
+        return VALUE
+    doctest_namespace["get_value"] = get_value
+            """,
+        ),
+    )
+
+    # Create the doctest file
+    tests_path = pytester.path / "tests"
+    tests_path.mkdir()
+    test_file = tests_path / f"example{file_ext}"
+    test_file.write_text(file_content, encoding="utf-8")
+
+    result = pytester.runpytest(str(test_file))
+    result.assert_outcomes(passed=1)

tests/regressions/test_autouse_fixtures.py

@@ -250,3 +250,74 @@ def test_DocutilsDocTestFinder(
 
     for test in tests:
         doctest.DebugRunner(verbose=False).run(test)
+
+
+class DoctestOptReTestCase(t.NamedTuple):
+    """Test fixture for doctestopt_re regex.
+
+    Backported from Sphinx commit ad0c343d3 (2025-01-04).
+    https://github.com/sphinx-doc/sphinx/commit/ad0c343d3
+
+    The original Sphinx test verified HTML output doesn't have trailing
+    whitespace after flag trimming. This test verifies the regex correctly
+    matches and removes leading whitespace before doctest flags.
+
+    Refs: sphinx-doc/sphinx#13164
+    """
+
+    test_id: str
+    input_code: str
+    expected_output: str
+
+
+DOCTESTOPT_RE_FIXTURES = [
+    DoctestOptReTestCase(
+        test_id="trailing-spaces-before-flag",
+        input_code="result = func()   # doctest: +SKIP",
+        expected_output="result = func()",
+    ),
+    DoctestOptReTestCase(
+        test_id="tab-before-flag",
+        input_code="result = func()\t# doctest: +SKIP",
+        expected_output="result = func()",
+    ),
+    DoctestOptReTestCase(
+        test_id="no-space-before-flag",
+        input_code="result = func()# doctest: +SKIP",
+        expected_output="result = func()",
+    ),
+    DoctestOptReTestCase(
+        test_id="multiline-with-leading-whitespace",
+        input_code="line1\nresult = func()   # doctest: +SKIP\nline3",
+        expected_output="line1\nresult = func()\nline3",
+    ),
+    DoctestOptReTestCase(
+        test_id="multiple-flags-on-separate-lines",
+        input_code="a = 1  # doctest: +SKIP\nb = 2  # doctest: +ELLIPSIS",
+        expected_output="a = 1\nb = 2",
+    ),
+    DoctestOptReTestCase(
+        test_id="mixed-tabs-and-spaces",
+        input_code="result = func() \t # doctest: +NORMALIZE_WHITESPACE",
+        expected_output="result = func()",
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    DoctestOptReTestCase._fields,
+    DOCTESTOPT_RE_FIXTURES,
+    ids=[f.test_id for f in DOCTESTOPT_RE_FIXTURES],
+)
+def test_doctestopt_re_whitespace_trimming(
+    test_id: str,
+    input_code: str,
+    expected_output: str,
+) -> None:
+    """Verify doctestopt_re removes leading whitespace before doctest flags.
+
+    Regression test for Sphinx PR #13164.
+    Backported from Sphinx commit ad0c343d3 (2025-01-04).
+    """
+    result = doctest_docutils.doctestopt_re.sub("", input_code)
+    assert result == expected_output