Introspection: add a CLI to generate stubs

Useful for setuptools-rust

Use it also for the integration tests

Author: Tpt

newsfragments/5904.added.md

@@ -0,0 +1 @@
+`pyo3-introspection`: add a small CLI to generate stubs

noxfile.py

@@ -1413,35 +1413,53 @@ def update_ui_tests(session: nox.Session):
 
 @nox.session(name="test-introspection")
 def test_introspection(session: nox.Session):
-    session.install("maturin")
-    session.install("ruff")
-    options = []
-    target = os.environ.get("CARGO_BUILD_TARGET")
-    if target is not None:
-        options += ("--target", target)
-    profile = os.environ.get("CARGO_BUILD_PROFILE")
-    if profile == "release":
-        options.append("--release")
-    session.run_always(
-        "maturin",
-        "develop",
-        "-m",
-        "./pytests/Cargo.toml",
-        "--features",
-        "experimental-async,experimental-inspect",
-        *options,
-    )
-    lib_file = session.run(
-        "python",
-        "-c",
-        "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)",
-        silent=True,
-    ).strip()
-    _run_cargo_test(
-        session,
-        package="pyo3-introspection",
-        env={"PYO3_PYTEST_LIB_PATH": lib_file},
-    )
+    with tempfile.TemporaryDirectory() as stub_dir:
+        session.install("maturin")
+        session.install("ruff")
+        options = []
+        target = os.environ.get("CARGO_BUILD_TARGET")
+        if target is not None:
+            options += ("--target", target)
+        profile = os.environ.get("CARGO_BUILD_PROFILE")
+        if profile == "release":
+            options.append("--release")
+        _run(
+            session,
+            "maturin",
+            "develop",
+            "-m",
+            "./pytests/Cargo.toml",
+            "--features",
+            "experimental-async,experimental-inspect",
+            *options,
+        )
+        lib_file = session.run(
+            "python",
+            "-c",
+            "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)",
+            silent=True,
+        ).strip()
+        _run_cargo(session, "run", "-p", "pyo3-introspection", "--", lib_file, stub_dir)
+        _run(session, "ruff", "format", stub_dir)
+        _ensure_directory_equals(Path(stub_dir), Path("pytests/stubs"))
+
+
+def _ensure_directory_equals(expected_dir: Path, actual_dir: Path):
+    # Assert all expected files are in actual and are equals
+    for expected_file_path in expected_dir.rglob("*"):
+        file_path = expected_file_path.relative_to(expected_dir)
+        actual_file_path = actual_dir / file_path
+        assert actual_file_path.exists(), f"File {file_path} does not exist"
+        assert expected_file_path.read_text() == actual_file_path.read_text(), (
+            f"Content is different in {file_path}"
+        )
+    # Assert all actual files are expected
+    for actual_file_path in actual_dir.rglob("*"):
+        file_path = actual_file_path.relative_to(actual_dir)
+        expected_file_path = expected_dir / file_path
+        assert expected_file_path.exists(), (
+            f"File {file_path} exist even if not existed"
+        )
 
 
 def _build_docs_for_ffi_check(session: nox.Session) -> None:

pyo3-introspection/Cargo.toml

@@ -14,8 +14,5 @@ goblin = ">=0.9, <0.11"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 
-[dev-dependencies]
-tempfile = "3.12.0"
-
 [lints]
 workspace = true

pyo3-introspection/src/main.rs

@@ -0,0 +1,24 @@
+//! Small CLI entry point to introspect a Python cdylib built using PyO3 and generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html).
+
+use anyhow::{anyhow, Context, Result};
+use pyo3_introspection::{introspect_cdylib, module_stub_files};
+use std::path::Path;
+use std::{env, fs};
+
+fn main() -> Result<()> {
+    let [_, binary_path, output_path] = env::args().collect::<Vec<_>>().try_into().map_err(|_| anyhow!("pyo3-introspection takes two arguments, the path of the binary to introspect and the path of the directory to write the stub to"))?;
+    let module = introspect_cdylib(&binary_path, "pyo3_pytests")
+        .with_context(|| format!("Failed to introspect module {binary_path}"))?;
+    let actual_stubs = module_stub_files(&module);
+    for (path, module) in actual_stubs {
+        let file_path = Path::new(&output_path).join(path);
+        if let Some(parent) = file_path.parent() {
+            fs::create_dir_all(parent).with_context(|| {
+                format!("Failed to create output directory {}", file_path.display())
+            })?;
+        }
+        fs::write(&file_path, module)
+            .with_context(|| format!("Failed to write module {}", file_path.display()))?;
+    }
+    Ok(())
+}