From b7fce309373776c218a01da9797bf95f2fb5fc61 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Sun, 10 May 2026 19:43:56 +0200 Subject: [PATCH 1/9] ci: add ruff isort as deps and check code --- .github/workflows/test_tip.yml | 5 +++ pyproject.toml | 18 ++++++++++ uv.lock | 63 ++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/.github/workflows/test_tip.yml b/.github/workflows/test_tip.yml index a4ca7b8..e2f7fd4 100644 --- a/.github/workflows/test_tip.yml +++ b/.github/workflows/test_tip.yml @@ -27,6 +27,11 @@ jobs: cache-dependency-glob: "pyproject.toml" - name: Sync dependencies run: uv sync --group dev + - name: Lint and import order + run: | + uv run ruff format --check paperscraper + uv run ruff check paperscraper + uv run isort --check-only paperscraper - name: Export AWS secrets env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/pyproject.toml b/pyproject.toml index e3cab00..4bb747b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,25 @@ dev = [ "pytest-cov", "build", "twine", + "ruff>=0.15", + "isort>=6", "mkdocs-material>=9.6", "mkdocstrings[python]>=0.24", "mkdocs-material-extensions", ] + +[tool.ruff] +line-length = 88 +target-version = "py39" + +[tool.ruff.lint] +select = ["E", "F", "I"] +ignore = ["E501"] + +[tool.isort] +line_length = 88 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +known_first_party = ["paperscraper"] diff --git a/uv.lock b/uv.lock index 84a50bc..a0d64d3 100644 --- a/uv.lock +++ b/uv.lock @@ -1427,6 +1427,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "isort" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/82/fa43935523efdfcce6abbae9da7f372b627b27142c3419fcf13bf5b0c397/isort-6.1.0.tar.gz", hash = "sha256:9b8f96a14cfee0677e78e941ff62f03769a06d412aabb9e2a90487b3b7e8d481", size = 824325, upload-time = "2025-10-01T16:26:45.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/cc/9b681a170efab4868a032631dea1e8446d8ec718a7f657b94d49d1a12643/isort-6.1.0-py3-none-any.whl", hash = "sha256:58d8927ecce74e5087aef019f778d4081a3b6c98f15a80ba35782ca8a2097784", size = 94329, upload-time = "2025-10-01T16:26:43.291Z" }, +] + +[[package]] +name = "isort" +version = "8.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/7c/ec4ab396d31b3b395e2e999c8f46dec78c5e29209fac49d1f4dace04041d/isort-8.0.1.tar.gz", hash = "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", size = 769592, upload-time = "2026-02-28T10:08:20.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/95/c7c34aa53c16353c56d0b802fba48d5f5caa2cdee7958acbcb795c830416/isort-8.0.1-py3-none-any.whl", hash = "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75", size = 89733, upload-time = "2026-02-28T10:08:19.466Z" }, +] + [[package]] name = "jaraco-classes" version = "3.4.0" @@ -2754,11 +2787,14 @@ dev = [ { name = "build" }, { name = "coverage", version = "7.10.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "coverage", version = "7.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "isort", version = "6.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "isort", version = "8.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "mkdocs-material" }, { name = "mkdocs-material-extensions" }, { name = "mkdocstrings", version = "0.30.1", source = { registry = "https://pypi.org/simple" }, extra = ["python"], marker = "python_full_version < '3.10'" }, { name = "mkdocstrings", version = "1.0.2", source = { registry = "https://pypi.org/simple" }, extra = ["python"], marker = "python_full_version >= '3.10'" }, { name = "pytest-cov" }, + { name = "ruff" }, { name = "twine" }, ] @@ -2789,10 +2825,12 @@ requires-dist = [ dev = [ { name = "build" }, { name = "coverage" }, + { name = "isort", specifier = ">=6" }, { name = "mkdocs-material", specifier = ">=9.6" }, { name = "mkdocs-material-extensions" }, { name = "mkdocstrings", extras = ["python"], specifier = ">=0.24" }, { name = "pytest-cov" }, + { name = "ruff", specifier = ">=0.15" }, { name = "twine" }, ] @@ -3719,6 +3757,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, ] +[[package]] +name = "ruff" +version = "0.15.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/99/43/3291f1cc9106f4c63bdce7a8d0df5047fe8422a75b091c16b5e9355e0b11/ruff-0.15.12.tar.gz", hash = "sha256:ecea26adb26b4232c0c2ca19ccbc0083a68344180bba2a600605538ce51a40a6", size = 4643852, upload-time = "2026-04-24T18:17:14.305Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/6e/e78ffb61d4686f3d96ba3df2c801161843746dcbcbb17a1e927d4829312b/ruff-0.15.12-py3-none-linux_armv6l.whl", hash = "sha256:f86f176e188e94d6bdbc09f09bfd9dc729059ad93d0e7390b5a73efe19f8861c", size = 10640713, upload-time = "2026-04-24T18:17:22.841Z" }, + { url = "https://files.pythonhosted.org/packages/ae/08/a317bc231fb9e7b93e4ef3089501e51922ff88d6936ce5cf870c4fe55419/ruff-0.15.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e3bcd123364c3770b8e1b7baaf343cc99a35f197c5c6e8af79015c666c423a6c", size = 11069267, upload-time = "2026-04-24T18:17:30.105Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/f828e9718d3dce1f5f11c39c4f65afd32783c8b2aebb2e3d259e492c47bd/ruff-0.15.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fe87510d000220aa1ed530d4448a7c696a0cae1213e5ec30e5874287b66557b5", size = 10397182, upload-time = "2026-04-24T18:17:07.177Z" }, + { url = "https://files.pythonhosted.org/packages/71/e0/3310fc6d1b5e1fdea22bf3b1b807c7e187b581021b0d7d4514cccdb5fb71/ruff-0.15.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84a1630093121375a3e2a95b4a6dc7b59e2b4ee76216e32d81aae550a832d002", size = 10758012, upload-time = "2026-04-24T18:16:55.759Z" }, + { url = "https://files.pythonhosted.org/packages/11/c1/a606911aee04c324ddaa883ae418f3569792fd3c4a10c50e0dd0a2311e1e/ruff-0.15.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb129f40f114f089ebe0ca56c0d251cf2061b17651d464bb6478dc01e69f11f5", size = 10447479, upload-time = "2026-04-24T18:16:51.677Z" }, + { url = "https://files.pythonhosted.org/packages/9d/68/4201e8444f0894f21ab4aeeaee68aa4f10b51613514a20d80bd628d57e88/ruff-0.15.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0c862b172d695db7598426b8af465e7e9ac00a3ea2a3630ee67eb82e366aaa6", size = 11234040, upload-time = "2026-04-24T18:17:16.529Z" }, + { url = "https://files.pythonhosted.org/packages/34/ff/8a6d6cf4ccc23fd67060874e832c18919d1557a0611ebef03fdb01fff11e/ruff-0.15.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2849ea9f3484c3aca43a82f484210370319e7170df4dfe4843395ddf6c57bc33", size = 12087377, upload-time = "2026-04-24T18:17:04.944Z" }, + { url = "https://files.pythonhosted.org/packages/85/f6/c669cf73f5152f623d34e69866a46d5e6185816b19fcd5b6dd8a2d299922/ruff-0.15.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e77c7e51c07fe396826d5969a5b846d9cd4c402535835fb6e21ce8b28fef847", size = 11367784, upload-time = "2026-04-24T18:17:25.409Z" }, + { url = "https://files.pythonhosted.org/packages/e8/39/c61d193b8a1daaa8977f7dea9e8d8ba866e02ea7b65d32f6861693aa4c12/ruff-0.15.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b2f4f2f3b1026b5fb449b467d9264bf22067b600f7b6f41fc5958909f449d0", size = 11344088, upload-time = "2026-04-24T18:17:12.258Z" }, + { url = "https://files.pythonhosted.org/packages/c2/8d/49afab3645e31e12c590acb6d3b5b69d7aab5b81926dbaf7461f9441f37a/ruff-0.15.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9ba3b8f1afd7e2e43d8943e55f249e13f9682fde09711644a6e7290eb4f3e339", size = 11271770, upload-time = "2026-04-24T18:17:02.457Z" }, + { url = "https://files.pythonhosted.org/packages/46/06/33f41fe94403e2b755481cdfb9b7ef3e4e0ed031c4581124658d935d52b4/ruff-0.15.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e852ba9fdc890655e1d78f2df1499efbe0e54126bd405362154a75e2bde159c5", size = 10719355, upload-time = "2026-04-24T18:17:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/0d/59/18aa4e014debbf559670e4048e39260a85c7fcee84acfd761ac01e7b8d35/ruff-0.15.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dd8aed930da53780d22fc70bdf84452c843cf64f8cb4eb38984319c24c5cd5fd", size = 10462758, upload-time = "2026-04-24T18:17:32.347Z" }, + { url = "https://files.pythonhosted.org/packages/25/e7/cc9f16fd0f3b5fddcbd7ec3d6ae30c8f3fde1047f32a4093a98d633c6570/ruff-0.15.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01da3988d225628b709493d7dc67c3b9b12c0210016b08690ef9bd27970b262b", size = 10953498, upload-time = "2026-04-24T18:17:20.674Z" }, + { url = "https://files.pythonhosted.org/packages/72/7a/a9ba7f98c7a575978698f4230c5e8cc54bbc761af34f560818f933dafa0c/ruff-0.15.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9cae0f92bd5700d1213188b31cd3bdd2b315361296d10b96b8e2337d3d11f53e", size = 11447765, upload-time = "2026-04-24T18:17:09.755Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f9/0ae446942c846b8266059ad8a30702a35afae55f5cdc54c5adf8d7afdc27/ruff-0.15.12-py3-none-win32.whl", hash = "sha256:d0185894e038d7043ba8fd6aee7499ece6462dc0ea9f1e260c7451807c714c20", size = 10657277, upload-time = "2026-04-24T18:17:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/33/f1/9614e03e1cdcbf9437570b5400ced8a720b5db22b28d8e0f1bda429f660d/ruff-0.15.12-py3-none-win_amd64.whl", hash = "sha256:c87a162d61ab3adca47c03f7f717c68672edec7d1b5499e652331780fe74950d", size = 11837758, upload-time = "2026-04-24T18:17:00.113Z" }, + { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, +] + [[package]] name = "s3transfer" version = "0.16.0" From eae479a6c0b62c11981defce8d49aafbf5ce7967 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Sun, 10 May 2026 19:46:33 +0200 Subject: [PATCH 2/9] chore: ruff/isort all --- paperscraper/arxiv/utils.py | 11 ++++++++--- paperscraper/async_utils.py | 6 +++++- paperscraper/citations/tests/test_citations.py | 8 ++++---- .../citations/tests/test_self_citations.py | 2 +- .../citations/tests/test_self_references.py | 1 - .../get_dumps/utils/chemrxiv/crossref_api.py | 4 +++- paperscraper/pdf/pdf.py | 4 +++- paperscraper/pubmed/utils.py | 10 ++++++++-- paperscraper/tests/test_dump.py | 10 ++++++++-- paperscraper/tests/test_impactor.py | 18 +++++++++--------- paperscraper/tests/test_pdf.py | 4 +++- paperscraper/xrxiv/xrxiv_api.py | 4 +++- 12 files changed, 55 insertions(+), 27 deletions(-) diff --git a/paperscraper/arxiv/utils.py b/paperscraper/arxiv/utils.py index 6b334f8..bf15877 100644 --- a/paperscraper/arxiv/utils.py +++ b/paperscraper/arxiv/utils.py @@ -5,12 +5,17 @@ from ..utils import get_server_dumps_dir -finalize_disjunction = lambda x: "(" + x[:-4] + ") AND " -finalize_conjunction = lambda x: x[:-5] - EARLIEST_START = "1970-01-01" +def finalize_disjunction(query: str) -> str: + return "(" + query[:-4] + ") AND " + + +def finalize_conjunction(query: str) -> str: + return query[:-5] + + def format_date(date_str: str) -> str: """Converts a date in YYYY-MM-DD format to arXiv's YYYYMMDDTTTT format.""" date_obj = datetime.strptime(date_str, "%Y-%m-%d") diff --git a/paperscraper/async_utils.py b/paperscraper/async_utils.py index 4e49b35..48084c9 100644 --- a/paperscraper/async_utils.py +++ b/paperscraper/async_utils.py @@ -106,7 +106,11 @@ async def wrapper(*args, **kwargs) -> Any: pass delay *= factor - except (httpx.ReadError, httpx.TimeoutException, httpx.TransportError) as e: + except ( + httpx.ReadError, + httpx.TimeoutException, + httpx.TransportError, + ) as e: last_exception = e sleep_for = delay delay *= factor diff --git a/paperscraper/citations/tests/test_citations.py b/paperscraper/citations/tests/test_citations.py index 93bd27f..2e13730 100644 --- a/paperscraper/citations/tests/test_citations.py +++ b/paperscraper/citations/tests/test_citations.py @@ -1,7 +1,7 @@ import logging from paperscraper.citations import get_citations_by_doi -from paperscraper.citations.utils import check_overlap, author_name_to_ssaid +from paperscraper.citations.utils import author_name_to_ssaid, check_overlap logging.disable(logging.INFO) @@ -17,9 +17,9 @@ def test_citations(self): def test_author_name_to_ssid(self): - ssaid, name = author_name_to_ssaid('Fabian H Sinz') - assert ssaid == '50095217' - assert name == 'Fabian H Sinz' + ssaid, name = author_name_to_ssaid("Fabian H Sinz") + assert ssaid == "50095217" + assert name == "Fabian H Sinz" def test_name_overlap(self): assert check_overlap("John Smith", "J. Smith") diff --git a/paperscraper/citations/tests/test_self_citations.py b/paperscraper/citations/tests/test_self_citations.py index 33435bf..66e4f6f 100644 --- a/paperscraper/citations/tests/test_self_citations.py +++ b/paperscraper/citations/tests/test_self_citations.py @@ -64,7 +64,7 @@ def test_multiple_dois(self, dois): f"Synchronous execution time (independent calls): {sync_duration:.2f} seconds" ) - assert async_duration*0.8 <= sync_duration, ( + assert async_duration * 0.8 <= sync_duration, ( f"Async execution ({async_duration:.2f}s) is slower than sync execution " f"({sync_duration:.2f}s)" ) diff --git a/paperscraper/citations/tests/test_self_references.py b/paperscraper/citations/tests/test_self_references.py index c8b14a9..26d75d7 100644 --- a/paperscraper/citations/tests/test_self_references.py +++ b/paperscraper/citations/tests/test_self_references.py @@ -1,5 +1,4 @@ import logging -import time from typing import Dict import pytest diff --git a/paperscraper/get_dumps/utils/chemrxiv/crossref_api.py b/paperscraper/get_dumps/utils/chemrxiv/crossref_api.py index 8a6bb43..a9ec4bf 100644 --- a/paperscraper/get_dumps/utils/chemrxiv/crossref_api.py +++ b/paperscraper/get_dumps/utils/chemrxiv/crossref_api.py @@ -144,7 +144,9 @@ def _request(self, params: Dict) -> Dict: } for attempt in range(self.max_retries): - resp = requests.get(self.base_url, params=params, headers=headers, timeout=30) + resp = requests.get( + self.base_url, params=params, headers=headers, timeout=30 + ) if resp.status_code in transient_status: logger.warning( f"Crossref returned {resp.status_code} (attempt {attempt + 1}/{self.max_retries}); " diff --git a/paperscraper/pdf/pdf.py b/paperscraper/pdf/pdf.py index ce9f276..83c21fe 100644 --- a/paperscraper/pdf/pdf.py +++ b/paperscraper/pdf/pdf.py @@ -29,7 +29,9 @@ CHEMRXIV_API_BASE = "https://www.cambridge.org/engage/coe/public-api/v1/items/doi/" -def _get_chemrxiv_item(doi: str, user_agent: Dict[str, str]) -> Optional[Dict[str, Any]]: +def _get_chemrxiv_item( + doi: str, user_agent: Dict[str, str] +) -> Optional[Dict[str, Any]]: """Fetch ChemRxiv metadata from the Cambridge Open Engage API. Args: diff --git a/paperscraper/pubmed/utils.py b/paperscraper/pubmed/utils.py index ecbf29d..631c39a 100644 --- a/paperscraper/pubmed/utils.py +++ b/paperscraper/pubmed/utils.py @@ -3,11 +3,17 @@ from pymed_paperscraper.article import PubMedArticle -finalize_disjunction = lambda x: "(" + x[:-4] + ") AND " -finalize_conjunction = lambda x: x[:-5] date_root = '("{0}"[Date - Create] : "{1}"[Date - Create])' +def finalize_disjunction(query: str) -> str: + return "(" + query[:-4] + ") AND " + + +def finalize_conjunction(query: str) -> str: + return query[:-5] + + def get_query_from_keywords(keywords: List[Union[str, List]]) -> str: """Receives a list of keywords and returns the query for the pubmed API. diff --git a/paperscraper/tests/test_dump.py b/paperscraper/tests/test_dump.py index aaed47d..aafa799 100644 --- a/paperscraper/tests/test_dump.py +++ b/paperscraper/tests/test_dump.py @@ -43,7 +43,9 @@ def run_with_arxiv_retries(self, func, retries=3, sleep_seconds=2): except arxiv_api.HTTPError as exc: status = getattr(exc, "status", None) if status == 429: - pytest.skip("Skipping arXiv-backed test due to HTTP 429 rate limiting") + pytest.skip( + "Skipping arXiv-backed test due to HTTP 429 rate limiting" + ) if status not in retryable_statuses: raise if attempt == retries: @@ -161,7 +163,11 @@ def test_arxiv_dumping(self): output_filepath="covid19_ai_imaging.jsonl", backend="api", max_results=5, - client_options={"delay_seconds": 6.0, "page_size": 50, "num_retries": 3}, + client_options={ + "delay_seconds": 6.0, + "page_size": 50, + "num_retries": 3, + }, ), retries=5, sleep_seconds=10, diff --git a/paperscraper/tests/test_impactor.py b/paperscraper/tests/test_impactor.py index 5de50d9..ade7091 100644 --- a/paperscraper/tests/test_impactor.py +++ b/paperscraper/tests/test_impactor.py @@ -67,15 +67,15 @@ def test_quantum_information_search(self, impactor): # Ensure that the results match the expected results assert len(results) == len(expected_results), "Number of results does not match" for expected, actual in zip(expected_results, results): - assert ( - expected["journal"] == actual["journal"] - ), f"Journal name does not match for {expected['journal']}" - assert ( - abs(expected["factor"] - actual["factor"]) < 0.001 - ), f"Impact factor does not match for {expected['journal']}" - assert ( - expected["score"] == actual["score"] - ), f"Score does not match for {expected['journal']}" + assert expected["journal"] == actual["journal"], ( + f"Journal name does not match for {expected['journal']}" + ) + assert abs(expected["factor"] - actual["factor"]) < 0.001, ( + f"Impact factor does not match for {expected['journal']}" + ) + assert expected["score"] == actual["score"], ( + f"Score does not match for {expected['journal']}" + ) results = impactor.search( "Quantum information", threshold=90, sort_by="score", min_impact=2 diff --git a/paperscraper/tests/test_pdf.py b/paperscraper/tests/test_pdf.py index 90eb5fe..d58d455 100644 --- a/paperscraper/tests/test_pdf.py +++ b/paperscraper/tests/test_pdf.py @@ -55,7 +55,9 @@ def test_basic_search(self): os.remove("taskload.pdf") # Test S3 fallback with newer DOIs (including year/month/day) - FALLBACKS["s3"](doi="10.1101/2023.10.09.561414", output_path="taskload.pdf", api_keys=keys) + FALLBACKS["s3"]( + doi="10.1101/2023.10.09.561414", output_path="taskload.pdf", api_keys=keys + ) assert os.path.exists("taskload.pdf") os.remove("taskload.pdf") diff --git a/paperscraper/xrxiv/xrxiv_api.py b/paperscraper/xrxiv/xrxiv_api.py index 0e25152..5cddd87 100644 --- a/paperscraper/xrxiv/xrxiv_api.py +++ b/paperscraper/xrxiv/xrxiv_api.py @@ -299,7 +299,9 @@ def dump_papers( raise ValueError(f"max_workers must be >= 1, got {max_workers}") worker_retries = max_retries if max_retries is not None else self.max_retries worker_retries = max(1, int(worker_retries)) - span_days = max(1, int(window_days if window_days is not None else self.window_days)) + span_days = max( + 1, int(window_days if window_days is not None else self.window_days) + ) start_datetime, end_datetime = self._normalize_date_range(start_date, end_date) start_text = start_datetime.strftime("%Y-%m-%d") From 6ec6f71a3af6b5c96dc22f5a7e58294091e2789c Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Sun, 10 May 2026 19:46:51 +0200 Subject: [PATCH 3/9] fix: figpath in plotting utils --- README.md | 6 +++--- paperscraper/plotting.py | 34 ++++++++++++++++++---------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 1a7718e..f4428d8 100644 --- a/README.md +++ b/README.md @@ -369,7 +369,7 @@ sizes_2019 = (55402, 11899, 2563) labels_2020 = ('Medical\nImaging', 'Artificial\nIntelligence', 'COVID-19') labels_2019 = ['Medical Imaging', 'Artificial\nIntelligence'] -plot_venn_two(sizes_2019, labels_2019, title='2019', figname='ai_imaging') +plot_venn_two(sizes_2019, labels_2019, title='2019', figpath='ai_imaging.png') ``` ![2019](https://github.com/jannisborn/paperscraper/blob/main/assets/ai_imaging.png?raw=true "2019") @@ -377,7 +377,7 @@ plot_venn_two(sizes_2019, labels_2019, title='2019', figname='ai_imaging') ```py plot_venn_three( - sizes_2020, labels_2020, title='2020', figname='ai_imaging_covid' + sizes_2020, labels_2020, title='2020', figpath='ai_imaging_covid.png' ) ``` @@ -390,7 +390,7 @@ plot_multiple_venn( [sizes_2019, sizes_2020], [labels_2019, labels_2020], titles=['2019', '2020'], suptitle='Keyword search comparison', gridspec_kw={'width_ratios': [1, 2]}, figsize=(10, 6), - figname='both' + figpath='both.png' ) ``` diff --git a/paperscraper/plotting.py b/paperscraper/plotting.py index ef2ead4..a514adf 100644 --- a/paperscraper/plotting.py +++ b/paperscraper/plotting.py @@ -158,7 +158,9 @@ def plot_comparison( ) plt.gca().add_artist(legend) - get_step_size = lambda x: round(x / 10, -math.floor(math.log10(x)) + 1) + def get_step_size(x): + return round(x / 10, -math.floor(math.log10(x)) + 1) + ymax = plt.gca().get_ylim()[1] step_size = np.clip(get_step_size(ymax), 5, 1000) y_steps = np.arange(0, ymax, step_size) @@ -237,7 +239,6 @@ def plot_single( ind = np.arange(len(arxiv[0])) # the x locations for the groups width = [0.75] * len(ind) # the width of the bars: can also be len(x) sequence - fnc = np.log10 if logscale else np.copy plts = [] legend_plts = [] @@ -324,7 +325,6 @@ def plot_single( ncol=1, ) - get_step_size = lambda x: round(x / 10, -math.floor(math.log10(x)) + 1) ymax = plt.gca().get_ylim()[1] for y_step in plt.yticks()[0]: @@ -337,7 +337,8 @@ def plot_single( plt.show() -get_name = lambda n: " vs. ".join(list(map(lambda x: x.split(" ")[0], n))) +def get_name(names): + return " vs. ".join(name.split(" ")[0] for name in names) def plot_venn_two( @@ -363,7 +364,7 @@ def plot_venn_two( assert len(labels) == 2, "Incorrect type/length of labels" title = get_name(labels) if title == "" else title - figname = title.lower().replace(" vs. ", "_") if figpath == "" else figpath + figpath = f"{title.lower().replace(' vs. ', '_')}.pdf" if figpath == "" else figpath venn2(subsets=sizes, set_labels=labels, alpha=0.6, **kwargs) venn2_circles( subsets=sizes, linestyle="solid", linewidth=0.6, color="grey", **kwargs @@ -374,18 +375,17 @@ def plot_venn_two( kwargs["ax"].set_title(title, fontdict={"fontweight": "bold"}, size=15) else: plt.title(title, fontdict={"fontweight": "bold"}, size=15) - plt.savefig(f"{figname}.pdf") + plt.savefig(figpath) def plot_venn_three( sizes: List[int], labels: List[str], figpath: str = "", title: str = "", **kwargs ) -> None: - """Plot a single Venn Diagram with two terms. + """Plot a single Venn Diagram with three terms. Args: - sizes (List[int]): List of ints of length 3. First two elements correspond to - the labels, third one to the intersection. - labels (List[str]): List of str of length 2, containing names of circles. + sizes (List[int]): List of ints of length 7 with the subset sizes. + labels (List[str]): List of str of length 3, containing names of circles. figpath (str): Name under which figure is saved. Defaults to '', i.e. it is inferred from labels. title (str): Title of the plot. Defaults to '', i.e. it is inferred from @@ -396,7 +396,7 @@ def plot_venn_three( assert len(labels) == 3, "Incorrect type/length of labels" title = get_name(labels) if title == "" else title - figname = title.lower().replace(" vs. ", "_") if figpath == "" else figpath + figpath = f"{title.lower().replace(' vs. ', '_')}.pdf" if figpath == "" else figpath venn3(subsets=sizes, set_labels=labels, alpha=0.6, **kwargs) venn3_circles( @@ -407,14 +407,14 @@ def plot_venn_three( kwargs["ax"].set_title(title, fontdict={"fontweight": "bold"}, size=15) else: plt.title(title, fontdict={"fontweight": "bold"}, size=15) - plt.savefig(f"{figname}.pdf") + plt.savefig(figpath) def plot_multiple_venn( sizes: List[List[int]], labels: List[List[str]], - figname: str, titles: List[str], + figpath: str = "", suptitle: str = "", gridspec_kw: dict = {}, figsize: Iterable = (8, 4.5), @@ -428,7 +428,7 @@ def plot_multiple_venn( (plot_venn_two). labels (List[List[str]]): List of Lists of str containing names of circles. Lengths of lists should be either 2 or 3. - figname (str): Name under which figure is saved. Defaults to '', i.e. it is + figpath (str): Name under which figure is saved. Defaults to '', i.e. it is inferred from labels. titles (List[str]): Titles of subplots. Should have same length like labels and sizes. @@ -449,7 +449,9 @@ def plot_multiple_venn( fig, axes = plt.subplots(1, len(sizes), gridspec_kw=gridspec_kw, figsize=figsize) plt.suptitle(suptitle, size=18, fontweight="bold") - figname = titles[0].lower().replace(" vs. ", "_") if figname == "" else figname + figpath = ( + f"{titles[0].lower().replace(' vs. ', '_')}.pdf" if figpath == "" else figpath + ) for idx, (size, label, title) in enumerate(zip(sizes, labels, titles)): if len(label) == 2: @@ -457,4 +459,4 @@ def plot_multiple_venn( elif len(label) == 3: plot_venn_three(size, label, title=title, ax=axes[idx]) - plt.savefig(f"{figname}.pdf") + plt.savefig(figpath) From a35ab3b95ff0677f997dce0fb208d3a4e373fc5a Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Sun, 10 May 2026 22:59:17 +0200 Subject: [PATCH 4/9] wip --- paperscraper/citations/entity/paper.py | 2 +- paperscraper/tests/test_dump.py | 7 ++++++- pyproject.toml | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/paperscraper/citations/entity/paper.py b/paperscraper/citations/entity/paper.py index 43562ca..bfaaaa2 100644 --- a/paperscraper/citations/entity/paper.py +++ b/paperscraper/citations/entity/paper.py @@ -20,7 +20,7 @@ class PaperResult(ReferenceResult, CitationResult): title: str -ModeType = Literal[tuple(MODES := ("doi", "title", "ss_id", "infer"))] +ModeType = Literal[tuple(MODES := ("doi", "title", "ssid", "infer"))] BASE_URL: str = "https://api.semanticscholar.org/graph/v1/paper/search" diff --git a/paperscraper/tests/test_dump.py b/paperscraper/tests/test_dump.py index aafa799..307f415 100644 --- a/paperscraper/tests/test_dump.py +++ b/paperscraper/tests/test_dump.py @@ -90,7 +90,12 @@ def run_function_with_timeout(self, func, timeout): process.join() if not was_alive and not queue.empty(): - raise queue.get() + result = queue.get() + if result is True: + return True + if isinstance(result, BaseException): + raise result + return False elif not was_alive: return False else: diff --git a/pyproject.toml b/pyproject.toml index 4bb747b..014f3b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,3 +106,8 @@ include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true known_first_party = ["paperscraper"] + +[tool.pytest.ini_options] +markers = [ + "timeout(seconds): mark tests that enforce a wall-clock timeout", +] From 09730797d79fc611a3193415f54b726952f5dc61 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Mon, 11 May 2026 00:36:37 +0200 Subject: [PATCH 5/9] refactor: more robust no-api-key handling --- paperscraper/citations/citations.py | 30 +++++-- paperscraper/citations/entity/researcher.py | 14 ++-- paperscraper/citations/self_citations.py | 6 +- paperscraper/citations/self_references.py | 6 +- .../citations/tests/test_citations.py | 31 ++++++- paperscraper/citations/utils.py | 83 +++++++++++++++++-- paperscraper/pdf/fallbacks.py | 45 ++++++++-- 7 files changed, 176 insertions(+), 39 deletions(-) diff --git a/paperscraper/citations/citations.py b/paperscraper/citations/citations.py index 2b1f372..6f0818a 100644 --- a/paperscraper/citations/citations.py +++ b/paperscraper/citations/citations.py @@ -1,14 +1,14 @@ import logging -import os import sys from time import sleep from scholarly import scholarly -from semanticscholar import SemanticScholar, SemanticScholarException +from semanticscholar import SemanticScholarException + +from .utils import PAPER_URL, semantic_scholar_requests_get logging.basicConfig(stream=sys.stdout, level=logging.INFO) logger = logging.getLogger(__name__) -sch = SemanticScholar(api_key=os.getenv("SS_API_KEY")) def get_citations_by_doi(doi: str) -> int: @@ -23,17 +23,29 @@ def get_citations_by_doi(doi: str) -> int: """ try: - paper = sch.get_paper(doi) - citations = len(paper["citations"]) + response = semantic_scholar_requests_get( + f"{PAPER_URL}DOI:{doi}", + params={"fields": "citationCount"}, + timeout=20, + ) + if response.status_code == 404: + logger.warning(f"Could not find paper {doi}, assuming 0 citation.") + return 0 + response.raise_for_status() + return response.json()["citationCount"] except SemanticScholarException.ObjectNotFoundException: logger.warning(f"Could not find paper {doi}, assuming 0 citation.") - citations = 0 + return 0 except ConnectionRefusedError as e: logger.warning(f"Waiting for 10 sec since {doi} gave: {e}") sleep(10) - citations = len(sch.get_paper(doi)["citations"]) - finally: - return citations + response = semantic_scholar_requests_get( + f"{PAPER_URL}DOI:{doi}", + params={"fields": "citationCount"}, + timeout=20, + ) + response.raise_for_status() + return response.json()["citationCount"] def get_citations_from_title(title: str) -> int: diff --git a/paperscraper/citations/entity/researcher.py b/paperscraper/citations/entity/researcher.py index d5d623e..b4bfbb3 100644 --- a/paperscraper/citations/entity/researcher.py +++ b/paperscraper/citations/entity/researcher.py @@ -1,13 +1,14 @@ -import os from typing import Any, List, Literal, Optional, Tuple -from semanticscholar import SemanticScholar - from ...async_utils import run_sync from ..orcid import orcid_to_author_name from ..self_citations import CitationResult, self_citations_paper from ..self_references import ReferenceResult, self_references_paper -from ..utils import author_name_to_ssaid, get_papers_for_author +from ..utils import ( + author_name_to_ssaid, + get_author_name_from_ssaid, + get_papers_for_author, +) from .core import Entity, EntityResult @@ -40,9 +41,6 @@ def __str__(self) -> str: ModeType = Literal[tuple(MODES := ("name", "orcid", "ssaid", "infer"))] -sch = SemanticScholar(api_key=os.getenv("SS_API_KEY")) - - class Researcher(Entity): name: str ssaid: int @@ -77,7 +75,7 @@ def __init__(self, input: str, mode: ModeType = "infer"): else: mode = "name" if mode == "ssaid": - self.name = sch.get_author(input)._name + self.name = get_author_name_from_ssaid(input) self.ssaid = input elif mode == "orcid": orcid_name = orcid_to_author_name(input) diff --git a/paperscraper/citations/self_citations.py b/paperscraper/citations/self_citations.py index 2edd918..18394a7 100644 --- a/paperscraper/citations/self_citations.py +++ b/paperscraper/citations/self_citations.py @@ -12,11 +12,11 @@ from ..async_utils import optional_async, retry_with_exponential_backoff from .utils import ( DOI_PATTERN, - HEADERS, HTTPX_LIMITS, REQUEST_SEMAPHORE, REQUEST_TIMEOUT_SECONDS, find_matching, + semantic_scholar_get, wait_for_request_slot, ) @@ -50,10 +50,10 @@ async def _fetch_citation_data( """ await wait_for_request_slot() - response = await client.get( + response = await semantic_scholar_get( + client, f"https://api.semanticscholar.org/graph/v1/paper/{suffix}", params={"fields": "title,authors,citations.authors"}, - headers=HEADERS, ) response.raise_for_status() return response.json() diff --git a/paperscraper/citations/self_references.py b/paperscraper/citations/self_references.py index 419f953..34ed4bd 100644 --- a/paperscraper/citations/self_references.py +++ b/paperscraper/citations/self_references.py @@ -12,11 +12,11 @@ from ..async_utils import optional_async, retry_with_exponential_backoff from .utils import ( DOI_PATTERN, - HEADERS, HTTPX_LIMITS, REQUEST_SEMAPHORE, REQUEST_TIMEOUT_SECONDS, find_matching, + semantic_scholar_get, wait_for_request_slot, ) @@ -51,10 +51,10 @@ async def _fetch_paper_with_references( """ await wait_for_request_slot() - response = await client.get( + response = await semantic_scholar_get( + client, f"https://api.semanticscholar.org/graph/v1/paper/{suffix}", params={"fields": "title,authors,references.authors"}, - headers=HEADERS, ) response.raise_for_status() return response.json() diff --git a/paperscraper/citations/tests/test_citations.py b/paperscraper/citations/tests/test_citations.py index 2e13730..2b65b26 100644 --- a/paperscraper/citations/tests/test_citations.py +++ b/paperscraper/citations/tests/test_citations.py @@ -1,6 +1,6 @@ import logging -from paperscraper.citations import get_citations_by_doi +from paperscraper.citations import get_citations_by_doi, utils from paperscraper.citations.utils import author_name_to_ssaid, check_overlap logging.disable(logging.INFO) @@ -21,6 +21,35 @@ def test_author_name_to_ssid(self): assert ssaid == "50095217" assert name == "Fabian H Sinz" + def test_semantic_scholar_403_disables_api_key(self, monkeypatch): + class Response: + def __init__(self, status_code): + self.status_code = status_code + + original_headers = utils.HEADERS.copy() + original_disabled = utils.semantic_scholar_key_disabled() + utils.HEADERS.clear() + utils.HEADERS["x-api-key"] = "bad-key" + utils._SEMANTIC_SCHOLAR_KEY_DISABLED = False + + calls = [] + + def mock_get(url, headers=None, **kwargs): + calls.append(dict(headers or {})) + return Response(403 if len(calls) == 1 else 200) + + monkeypatch.setattr(utils.requests, "get", mock_get) + try: + response = utils.semantic_scholar_requests_get("https://example.test") + assert response.status_code == 200 + assert calls == [{"x-api-key": "bad-key"}, {}] + assert utils.HEADERS == {} + assert utils.semantic_scholar_key_disabled() + finally: + utils.HEADERS.clear() + utils.HEADERS.update(original_headers) + utils._SEMANTIC_SCHOLAR_KEY_DISABLED = original_disabled + def test_name_overlap(self): assert check_overlap("John Smith", "J. Smith") assert check_overlap("J. Smith", "John Smith") diff --git a/paperscraper/citations/utils.py b/paperscraper/citations/utils.py index a8dda17..5f6db6a 100644 --- a/paperscraper/citations/utils.py +++ b/paperscraper/citations/utils.py @@ -31,6 +31,7 @@ HEADERS: Dict[str, str] = {} if SS_API_KEY: HEADERS["x-api-key"] = SS_API_KEY +_SEMANTIC_SCHOLAR_KEY_DISABLED = False HTTPX_LIMITS = httpx.Limits( max_connections=CONCURRENCY_LIMIT, max_keepalive_connections=CONCURRENCY_LIMIT @@ -40,6 +41,58 @@ _NEXT_REQUEST_TIME = 0.0 +def disable_semantic_scholar_api_key() -> None: + """ + Disable the configured Semantic Scholar API key after the API rejects it. + """ + global _SEMANTIC_SCHOLAR_KEY_DISABLED + if "x-api-key" not in HEADERS: + return + + HEADERS.clear() + _SEMANTIC_SCHOLAR_KEY_DISABLED = True + logger.error( + "Semantic Scholar rejected SS_API_KEY with HTTP 403 Forbidden. " + "Continuing without the API key for subsequent requests." + ) + + +def semantic_scholar_key_disabled() -> bool: + """ + Return whether the configured Semantic Scholar API key was disabled. + """ + return _SEMANTIC_SCHOLAR_KEY_DISABLED + + +def _should_retry_without_key(status_code: int) -> bool: + if status_code != 403 or "x-api-key" not in HEADERS: + return False + disable_semantic_scholar_api_key() + return True + + +async def semantic_scholar_get( + client: httpx.AsyncClient, url: str, **kwargs +) -> httpx.Response: + """ + Perform a Semantic Scholar GET request and retry without a rejected API key. + """ + response = await client.get(url, headers=HEADERS, **kwargs) + if _should_retry_without_key(response.status_code): + response = await client.get(url, headers=HEADERS, **kwargs) + return response + + +def semantic_scholar_requests_get(url: str, **kwargs) -> requests.Response: + """ + Perform a synchronous Semantic Scholar GET request and retry without a rejected API key. + """ + response = requests.get(url, headers=HEADERS, **kwargs) + if _should_retry_without_key(response.status_code): + response = requests.get(url, headers=HEADERS, **kwargs) + return response + + async def wait_for_request_slot() -> None: """ Enforces global pacing between Semantic Scholar requests. @@ -67,10 +120,9 @@ def get_doi_from_title(title: str) -> Optional[str]: Returns: DOI according to semantic scholar API """ - response = requests.get( + response = semantic_scholar_requests_get( PAPER_URL + "search", params={"query": title, "fields": "externalIds", "limit": 1}, - headers=HEADERS, ) data = response.json() @@ -82,6 +134,20 @@ def get_doi_from_title(title: str) -> Optional[str]: logger.warning(f"Did not find DOI for title={title}") +def get_author_name_from_ssaid(ss_author_id: str) -> Optional[str]: + """ + Given a Semantic Scholar author ID, return the author's name. + """ + response = semantic_scholar_requests_get( + f"https://api.semanticscholar.org/graph/v1/author/{ss_author_id}", + params={"fields": "name"}, + timeout=REQUEST_TIMEOUT_SECONDS, + ) + response.raise_for_status() + data = response.json() + return data.get("name") + + @optional_async async def get_doi_from_ssid(ssid: str, max_retries: int = 10) -> Optional[str]: """ @@ -104,10 +170,10 @@ async def get_doi_from_ssid(ssid: str, max_retries: int = 10) -> Optional[str]: range(1, max_retries + 1), desc=f"Fetching DOI for {ssid}", unit="attempt" ): # Make the GET request to Semantic Scholar. - response = await client.get( + response = await semantic_scholar_get( + client, f"{PAPER_URL}{ssid}", params={"fields": "externalIds", "limit": 1}, - headers=HEADERS, ) # If successful, try to extract and return the DOI. @@ -136,7 +202,7 @@ async def get_title_and_id_from_doi(doi: str) -> Dict[str, str] | None: timeout=httpx.Timeout(REQUEST_TIMEOUT_SECONDS), limits=HTTPX_LIMITS ) as client: # Send the GET request to Semantic Scholar - response = await client.get(f"{PAPER_URL}DOI:{doi}", headers=HEADERS) + response = await semantic_scholar_get(client, f"{PAPER_URL}DOI:{doi}") if response.status_code == 200: data = response.json() return {"title": data.get("title"), "ssid": data.get("paperId")} @@ -164,10 +230,10 @@ async def author_name_to_ssaid(author_name: str) -> Tuple[str, str]: ) as client: await wait_for_request_slot() - response = await client.get( + response = await semantic_scholar_get( + client, AUTHOR_URL, params={"query": author_name, "fields": "name", "limit": 1}, - headers=HEADERS, ) response.raise_for_status() data = response.json() @@ -224,7 +290,8 @@ async def get_papers_for_author(ss_author_id: str) -> List[str]: timeout=httpx.Timeout(REQUEST_TIMEOUT_SECONDS), limits=HTTPX_LIMITS ) as client: while True: - response = await client.get( + response = await semantic_scholar_get( + client, f"https://api.semanticscholar.org/graph/v1/author/{ss_author_id}/papers", params={"fields": "paperId", "offset": offset, "limit": limit}, ) diff --git a/paperscraper/pdf/fallbacks.py b/paperscraper/pdf/fallbacks.py index c4ec59f..c98811b 100644 --- a/paperscraper/pdf/fallbacks.py +++ b/paperscraper/pdf/fallbacks.py @@ -236,11 +236,16 @@ def fallback_elife_xml(doi: str, output_path: Path) -> bool: article_num = parts[1].strip() index = get_elife_xml_index() - if article_num not in index: - logger.warning(f"No eLife XML found for DOI {doi}.") - return False - candidate_files = index[article_num] - latest_version, latest_download_url = max(candidate_files, key=lambda x: x[0]) + if article_num in index: + candidate_files = index[article_num] + latest_version, latest_download_url = max(candidate_files, key=lambda x: x[0]) + else: + candidate_files = _direct_elife_xml_candidates(article_num) + if not candidate_files: + logger.warning(f"No eLife XML found for DOI {doi}.") + return False + latest_version, latest_download_url = candidate_files[0] + try: r = requests.get(latest_download_url, timeout=60) r.raise_for_status() @@ -258,6 +263,23 @@ def fallback_elife_xml(doi: str, output_path: Path) -> bool: return True +def _direct_elife_xml_candidates(article_num: str) -> list: + """ + Find versioned eLife XML files directly when the GitHub tree API is unavailable. + """ + base_url = "https://raw.githubusercontent.com/elifesciences/elife-article-xml/master" + candidates = [] + for version in range(1, 8): + download_url = f"{base_url}/articles/elife-{article_num}-v{version}.xml" + try: + response = requests.head(download_url, allow_redirects=True, timeout=20) + if response.status_code == 200: + candidates.append((version, download_url)) + except requests.RequestException as e: + logger.debug(f"Could not check eLife XML candidate {download_url}: {e}") + return sorted(candidates, key=lambda x: x[0], reverse=True) + + def get_elife_xml_index() -> dict: """ Fetch the eLife XML index from GitHub and return it as a dictionary. @@ -277,8 +299,17 @@ def get_elife_xml_index() -> dict: ELIFE_XML_INDEX = {} # Use the git tree API to get the full repository tree. base_tree_url = "https://api.github.com/repos/elifesciences/elife-article-xml/git/trees/master?recursive=1" - r = requests.get(base_tree_url, timeout=60) - r.raise_for_status() + for attempt in range(1, 4): + try: + r = requests.get(base_tree_url, timeout=60) + r.raise_for_status() + break + except requests.RequestException as e: + if attempt == 3: + logger.error(f"Could not fetch eLife XML index from GitHub: {e}") + return ELIFE_XML_INDEX + logger.info("Retrying eLife XML index fetch from GitHub...") + time.sleep(2 * attempt) tree_data = r.json() items = tree_data.get("tree", []) # Look for files in the 'articles' directory matching the pattern. From 8f80d571a385596252625c3276eb429a1cf227fe Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Mon, 11 May 2026 09:19:21 +0200 Subject: [PATCH 6/9] ci: pre-commit hook and formatting --- .pre-commit-config.yaml | 14 ++++++++++++++ paperscraper/citations/entity/researcher.py | 1 + paperscraper/pdf/fallbacks.py | 4 +++- 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0ff9196 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +# Note that one can skip some hooks e.g. with `SKIP=detect-secrets git commit ...` +repos: + - repo: local + hooks: + - id: Linting + name: Style checks (ruff, isort) + entry: sh -exc + language: system + always_run: true + pass_filenames: false + args: + - | + uv run --frozen uv run isort --check paperscraper + uv run --frozen uv run ruff format --check paperscraper diff --git a/paperscraper/citations/entity/researcher.py b/paperscraper/citations/entity/researcher.py index b4bfbb3..bafcb87 100644 --- a/paperscraper/citations/entity/researcher.py +++ b/paperscraper/citations/entity/researcher.py @@ -41,6 +41,7 @@ def __str__(self) -> str: ModeType = Literal[tuple(MODES := ("name", "orcid", "ssaid", "infer"))] + class Researcher(Entity): name: str ssaid: int diff --git a/paperscraper/pdf/fallbacks.py b/paperscraper/pdf/fallbacks.py index c98811b..aae8345 100644 --- a/paperscraper/pdf/fallbacks.py +++ b/paperscraper/pdf/fallbacks.py @@ -267,7 +267,9 @@ def _direct_elife_xml_candidates(article_num: str) -> list: """ Find versioned eLife XML files directly when the GitHub tree API is unavailable. """ - base_url = "https://raw.githubusercontent.com/elifesciences/elife-article-xml/master" + base_url = ( + "https://raw.githubusercontent.com/elifesciences/elife-article-xml/master" + ) candidates = [] for version in range(1, 8): download_url = f"{base_url}/articles/elife-{article_num}-v{version}.xml" From dbede52028d6e26ac9754502663d531e054e61b3 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Mon, 11 May 2026 14:30:45 +0200 Subject: [PATCH 7/9] test --- paperscraper/citations/tests/test_self_citations.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/paperscraper/citations/tests/test_self_citations.py b/paperscraper/citations/tests/test_self_citations.py index 66e4f6f..effa7cf 100644 --- a/paperscraper/citations/tests/test_self_citations.py +++ b/paperscraper/citations/tests/test_self_citations.py @@ -64,11 +64,6 @@ def test_multiple_dois(self, dois): f"Synchronous execution time (independent calls): {sync_duration:.2f} seconds" ) - assert async_duration * 0.8 <= sync_duration, ( - f"Async execution ({async_duration:.2f}s) is slower than sync execution " - f"({sync_duration:.2f}s)" - ) - for a, s in zip( sorted(result, key=lambda r: r.ssid), sorted(sync_result, key=lambda r: r.ssid), From 8a641340fe9e7f86c89cab099287deb57681a4bd Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Tue, 12 May 2026 11:19:05 +0200 Subject: [PATCH 8/9] robustify --- paperscraper/async_utils.py | 24 ++++--- .../citations/tests/test_self_citations.py | 2 +- paperscraper/citations/utils.py | 63 ++++++++++++++++++- 3 files changed, 78 insertions(+), 11 deletions(-) diff --git a/paperscraper/async_utils.py b/paperscraper/async_utils.py index 48084c9..1feae59 100644 --- a/paperscraper/async_utils.py +++ b/paperscraper/async_utils.py @@ -2,6 +2,7 @@ import logging import sys import threading +import random from functools import wraps from typing import Any, Awaitable, Callable, TypeVar, Union @@ -64,15 +65,19 @@ def retry_with_exponential_backoff( base_delay: float = 1.0, factor: float = 1.3, constant_delay: float = 0.2, + max_delay: float = 60.0, + jitter_ratio: float = 0.1, ) -> Callable[[F], F]: """ - Decorator factory that retries an `async def` on HTTP 429, with exponential backoff. + Decorator factory that retries an `async def` on transient HTTP/network errors, with exponential backoff. Args: max_retries: how many times to retry before giving up. base_delay: initial delay in seconds; next delays will be multiplied by `factor`. factor: multiplier for delay after each retry. constant_delay: fixed delay before each attempt. + max_delay: maximum backoff delay between attempts. + jitter_ratio: add +/- jitter_ratio * delay seconds of jitter. Usage: @@ -93,18 +98,19 @@ async def wrapper(*args, **kwargs) -> Any: return await func(*args, **kwargs) except httpx.HTTPStatusError as e: status = e.response.status_code if e.response is not None else None - if status != 429: + retryable = status == 429 or (status is not None and (status == 408 or 500 <= status <= 599)) + if not retryable: raise last_exception = e - sleep_for = delay + sleep_for = min(delay, max_delay) if e.response is not None: ra = e.response.headers.get("Retry-After") if ra is not None: try: - sleep_for = float(ra) + sleep_for = min(float(ra), max_delay) except ValueError: pass - delay *= factor + delay = min(delay * factor, max_delay) except ( httpx.ReadError, @@ -112,8 +118,8 @@ async def wrapper(*args, **kwargs) -> Any: httpx.TransportError, ) as e: last_exception = e - sleep_for = delay - delay *= factor + sleep_for = min(delay, max_delay) + delay = min(delay * factor, max_delay) if attempt == max_retries: msg = ( @@ -122,6 +128,10 @@ async def wrapper(*args, **kwargs) -> Any: ) raise RuntimeError(msg) from last_exception + if jitter_ratio > 0: + jitter = sleep_for * jitter_ratio + sleep_for = max(0.0, sleep_for + random.uniform(-jitter, jitter)) + await asyncio.sleep(sleep_for) return wrapper diff --git a/paperscraper/citations/tests/test_self_citations.py b/paperscraper/citations/tests/test_self_citations.py index effa7cf..7ed6edb 100644 --- a/paperscraper/citations/tests/test_self_citations.py +++ b/paperscraper/citations/tests/test_self_citations.py @@ -129,7 +129,7 @@ def test_whole_researcher(self): assert result.num_citations > 0 assert isinstance(result.self_citations, Dict) assert isinstance(result.self_references, Dict) - assert len(result.self_citations) > 5 + assert len(result.self_citations) >= 5 assert len(result.self_references) >= 3 for title, ratio in result.self_citations.items(): assert isinstance(title, str) diff --git a/paperscraper/citations/utils.py b/paperscraper/citations/utils.py index 5f6db6a..e0eb0a5 100644 --- a/paperscraper/citations/utils.py +++ b/paperscraper/citations/utils.py @@ -4,6 +4,7 @@ import re import sys import time +import random from typing import Dict, List, Literal, Optional, Tuple import httpx @@ -93,6 +94,64 @@ def semantic_scholar_requests_get(url: str, **kwargs) -> requests.Response: return response +def _semantic_scholar_requests_get_with_backoff( + url: str, + *, + max_retries: int = 10, + base_delay: float = 1.0, + factor: float = 1.3, + max_delay: float = 60.0, + jitter_ratio: float = 0.1, + **kwargs, +) -> requests.Response: + """ + Synchronous GET with backoff for transient Semantic Scholar errors. + + Retries 429 / 408 / 5xx and respects Retry-After when present. + """ + delay = base_delay + last_exc: BaseException | None = None + for attempt in range(1, max_retries + 1): + try: + # Keep a minimum pacing between outbound requests. + if RATE_LIMIT_DELAY > 0: + time.sleep(RATE_LIMIT_DELAY) + resp = semantic_scholar_requests_get(url, timeout=REQUEST_TIMEOUT_SECONDS, **kwargs) + except requests.exceptions.RequestException as exc: + last_exc = exc + sleep_for = min(delay, max_delay) + else: + if resp.status_code in (200, 201, 204): + return resp + + retryable = resp.status_code == 429 or resp.status_code == 408 or 500 <= resp.status_code <= 599 + if not retryable: + resp.raise_for_status() + return resp + + sleep_for = min(delay, max_delay) + ra = resp.headers.get("Retry-After") + if ra is not None: + try: + sleep_for = min(float(ra), max_delay) + except ValueError: + pass + + if attempt == max_retries: + raise RuntimeError( + f"_semantic_scholar_requests_get_with_backoff failed after {attempt} attempts " + f"with last delay {sleep_for:.2f}s" + ) from last_exc + + delay = min(delay * factor, max_delay) + if jitter_ratio > 0: + jitter = sleep_for * jitter_ratio + sleep_for = max(0.0, sleep_for + random.uniform(-jitter, jitter)) + time.sleep(sleep_for) + + raise RuntimeError("_semantic_scholar_requests_get_with_backoff: unreachable") + + async def wait_for_request_slot() -> None: """ Enforces global pacing between Semantic Scholar requests. @@ -138,12 +197,10 @@ def get_author_name_from_ssaid(ss_author_id: str) -> Optional[str]: """ Given a Semantic Scholar author ID, return the author's name. """ - response = semantic_scholar_requests_get( + response = _semantic_scholar_requests_get_with_backoff( f"https://api.semanticscholar.org/graph/v1/author/{ss_author_id}", params={"fields": "name"}, - timeout=REQUEST_TIMEOUT_SECONDS, ) - response.raise_for_status() data = response.json() return data.get("name") From a28ceb8d80489d9acb2a95adc2f18fd1e8a68131 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Tue, 12 May 2026 21:03:58 +0200 Subject: [PATCH 9/9] chore --- .pre-commit-config.yaml | 4 +- paperscraper/async_utils.py | 6 +- paperscraper/citations/utils.py | 12 ++- pyproject.toml | 1 + uv.lock | 166 +++++++++++++++++++++++++++++++- 5 files changed, 178 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0ff9196..de1d3f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,5 +10,5 @@ repos: pass_filenames: false args: - | - uv run --frozen uv run isort --check paperscraper - uv run --frozen uv run ruff format --check paperscraper + uv run --frozen isort --check paperscraper + uv run --frozen ruff format --check paperscraper diff --git a/paperscraper/async_utils.py b/paperscraper/async_utils.py index 1feae59..d04d6b2 100644 --- a/paperscraper/async_utils.py +++ b/paperscraper/async_utils.py @@ -1,8 +1,8 @@ import asyncio import logging +import random import sys import threading -import random from functools import wraps from typing import Any, Awaitable, Callable, TypeVar, Union @@ -98,7 +98,9 @@ async def wrapper(*args, **kwargs) -> Any: return await func(*args, **kwargs) except httpx.HTTPStatusError as e: status = e.response.status_code if e.response is not None else None - retryable = status == 429 or (status is not None and (status == 408 or 500 <= status <= 599)) + retryable = status == 429 or ( + status is not None and (status == 408 or 500 <= status <= 599) + ) if not retryable: raise last_exception = e diff --git a/paperscraper/citations/utils.py b/paperscraper/citations/utils.py index e0eb0a5..19be04c 100644 --- a/paperscraper/citations/utils.py +++ b/paperscraper/citations/utils.py @@ -1,10 +1,10 @@ import asyncio import logging import os +import random import re import sys import time -import random from typing import Dict, List, Literal, Optional, Tuple import httpx @@ -116,7 +116,9 @@ def _semantic_scholar_requests_get_with_backoff( # Keep a minimum pacing between outbound requests. if RATE_LIMIT_DELAY > 0: time.sleep(RATE_LIMIT_DELAY) - resp = semantic_scholar_requests_get(url, timeout=REQUEST_TIMEOUT_SECONDS, **kwargs) + resp = semantic_scholar_requests_get( + url, timeout=REQUEST_TIMEOUT_SECONDS, **kwargs + ) except requests.exceptions.RequestException as exc: last_exc = exc sleep_for = min(delay, max_delay) @@ -124,7 +126,11 @@ def _semantic_scholar_requests_get_with_backoff( if resp.status_code in (200, 201, 204): return resp - retryable = resp.status_code == 429 or resp.status_code == 408 or 500 <= resp.status_code <= 599 + retryable = ( + resp.status_code == 429 + or resp.status_code == 408 + or 500 <= resp.status_code <= 599 + ) if not retryable: resp.raise_for_status() return resp diff --git a/pyproject.toml b/pyproject.toml index 014f3b7..727e147 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ dev = [ "mkdocs-material>=9.6", "mkdocstrings[python]>=0.24", "mkdocs-material-extensions", + "pre-commit>=4.3.0", ] [tool.ruff] diff --git a/uv.lock b/uv.lock index a0d64d3..4caabf1 100644 --- a/uv.lock +++ b/uv.lock @@ -284,6 +284,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/8f/a1e836f82d8e32a97e6b29cc8f641779181ac7363734f12df27db803ebda/cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9", size = 182794, upload-time = "2025-09-08T23:24:02.943Z" }, ] +[[package]] +name = "cfgv" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/11/74/539e56497d9bd1d484fd863dd69cbbfa653cd2aa27abfe35653494d85e94/cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560", size = 7114, upload-time = "2023-08-12T20:38:17.776Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249, upload-time = "2023-08-12T20:38:16.269Z" }, +] + +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + [[package]] name = "chardet" version = "5.2.0" @@ -890,6 +920,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + [[package]] name = "docutils" version = "0.21.2" @@ -990,7 +1029,7 @@ wheels = [ [[package]] name = "filelock" -version = "3.20.3" +version = "3.29.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'win32'", @@ -1001,9 +1040,9 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, + { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, ] [[package]] @@ -1337,6 +1376,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/cb/18326d2d89ad3b0dd143da971e77afd1e6ca6674f1b1c3df4b6bec6279fc/id-1.5.0-py3-none-any.whl", hash = "sha256:f1434e1cef91f2cbb8a4ec64663d5a23b9ed43ef44c4c957d02583d61714c658", size = 13611, upload-time = "2024-12-04T19:53:03.02Z" }, ] +[[package]] +name = "identify" +version = "2.6.15" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/e7/685de97986c916a6d93b3876139e00eef26ad5bbbd61925d670ae8013449/identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf", size = 99311, upload-time = "2025-10-02T17:43:40.631Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757", size = 99183, upload-time = "2025-10-02T17:43:39.137Z" }, +] + +[[package]] +name = "identify" +version = "2.6.19" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -2413,6 +2482,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/7e/a96255f63b7aef032cbee8fc4d6e37def72e3aaedc1f72759235e8f13cb1/nh3-0.3.2-cp38-abi3-win_arm64.whl", hash = "sha256:cf5964d54edd405e68583114a7cba929468bcd7db5e676ae38ee954de1cfc104", size = 584162, upload-time = "2025-10-30T11:17:44.96Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "numpy" version = "2.0.2" @@ -2793,6 +2871,8 @@ dev = [ { name = "mkdocs-material-extensions" }, { name = "mkdocstrings", version = "0.30.1", source = { registry = "https://pypi.org/simple" }, extra = ["python"], marker = "python_full_version < '3.10'" }, { name = "mkdocstrings", version = "1.0.2", source = { registry = "https://pypi.org/simple" }, extra = ["python"], marker = "python_full_version >= '3.10'" }, + { name = "pre-commit", version = "4.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pre-commit", version = "4.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pytest-cov" }, { name = "ruff" }, { name = "twine" }, @@ -2829,6 +2909,7 @@ dev = [ { name = "mkdocs-material", specifier = ">=9.6" }, { name = "mkdocs-material-extensions" }, { name = "mkdocstrings", extras = ["python"], specifier = ">=0.24" }, + { name = "pre-commit", specifier = ">=4.3.0" }, { name = "pytest-cov" }, { name = "ruff", specifier = ">=0.15" }, { name = "twine" }, @@ -3058,6 +3139,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pre-commit" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "cfgv", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "identify", version = "2.6.15", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "nodeenv", marker = "python_full_version < '3.10'" }, + { name = "pyyaml", marker = "python_full_version < '3.10'" }, + { name = "virtualenv", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/29/7cf5bbc236333876e4b41f56e06857a87937ce4bf91e117a6991a2dbb02a/pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16", size = 193792, upload-time = "2025-08-09T18:56:14.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965, upload-time = "2025-08-09T18:56:13.192Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "cfgv", version = "3.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "identify", version = "2.6.19", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "nodeenv", marker = "python_full_version >= '3.10'" }, + { name = "pyyaml", marker = "python_full_version >= '3.10'" }, + { name = "virtualenv", marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/2de9408ac81acbb8a7d05d4cc064a152ccf33b3d480ebe0cd292153db239/pre_commit-4.6.0.tar.gz", hash = "sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9", size = 198525, upload-time = "2026-04-21T20:31:41.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/6e/4b28b62ecb6aae56769c34a8ff1d661473ec1e9519e2d5f8b2c150086b26/pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b", size = 226472, upload-time = "2026-04-21T20:31:40.092Z" }, +] + [[package]] name = "prettytable" version = "3.16.0" @@ -3390,6 +3515,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-discovery" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "filelock", version = "3.29.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/e0/cc5a8653e9a24f6cf84768f05064aa8ed5a83dcefd5e2a043db14a1c5f44/python_discovery-1.3.0.tar.gz", hash = "sha256:d098f1e86be5d45fe4d14bf1029294aabbd332f4321179dec85e76cddce834b0", size = 63925, upload-time = "2026-05-05T14:38:39.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/d4/24d543ab8b8158b7f5a97113c831205f5c900c92c8762b1e7f44b7ea0405/python_discovery-1.3.0-py3-none-any.whl", hash = "sha256:441d9ced3dfce36e113beb35ca302c71c7ef06f3c0f9c227a0b9bb3bd49b9e9f", size = 33124, upload-time = "2026-05-05T14:38:38.539Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.1" @@ -4491,7 +4631,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "filelock", version = "3.20.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "filelock", version = "3.29.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "idna", marker = "python_full_version >= '3.10'" }, { name = "requests", marker = "python_full_version >= '3.10'" }, { name = "requests-file", marker = "python_full_version >= '3.10'" }, @@ -4750,6 +4890,24 @@ socks = [ { name = "pysocks", marker = "python_full_version >= '3.10'" }, ] +[[package]] +name = "virtualenv" +version = "21.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "filelock", version = "3.29.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "python-discovery" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/e1/665267cea4767debd19f584667a9197c2098b5e7f67a502da9f3a086ab37/virtualenv-21.3.2.tar.gz", hash = "sha256:3ecda97894a6fc1c53106356f488690e5c86278c1f693f3fc0805ac85a513686", size = 7613810, upload-time = "2026-05-12T14:44:18.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/5b/885f479093f6627669d39b57bc3d4e674da532e1a4b247d473a61d8d2118/virtualenv-21.3.2-py3-none-any.whl", hash = "sha256:c58ea748fa50bb2a4367da5ba3d30b02458ed40b4ea888faad94021f3309f764", size = 7594558, upload-time = "2026-05-12T14:44:15.193Z" }, +] + [[package]] name = "watchdog" version = "6.0.0"