From efa0679fed4208005fe83663301faec786717dc6 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Wed, 27 May 2026 22:14:54 +0200 Subject: [PATCH 1/3] doc: Draft for full documentation and prepare stable release --- .github/workflows/docs.yml | 5 - README.md | 442 ++++++---------------- docs/api_reference.md | 8 - docs/examples/paper-keyword-analysis.md | 207 ++++++++++ docs/examples/pdf-retrieval.md | 85 +++++ docs/examples/scholar-metrics-analysis.md | 170 +++++++++ docs/examples/self-citation-analysis.md | 101 +++++ mkdocs.yml | 6 + paperscraper/__init__.py | 2 +- paperscraper/arxiv/arxiv.py | 7 +- paperscraper/plotting.py | 67 ++-- paperscraper/pubmed/pubmed.py | 4 +- paperscraper/tests/test_dump.py | 5 +- paperscraper/tests/test_pdf.py | 10 +- paperscraper/xrxiv/tests/test_xrxiv.py | 8 +- pyproject.toml | 9 +- 16 files changed, 758 insertions(+), 378 deletions(-) delete mode 100644 docs/api_reference.md create mode 100644 docs/examples/paper-keyword-analysis.md create mode 100644 docs/examples/pdf-retrieval.md create mode 100644 docs/examples/scholar-metrics-analysis.md create mode 100644 docs/examples/self-citation-analysis.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4532d63..71c28d7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,11 +30,6 @@ jobs: run: | uv sync --group dev - - name: Copy README assets for MkDocs - run: | - mkdir -p docs/assets - cp assets/ai_quantum_*.png docs/assets/ - - name: Build docs run: uv run mkdocs build diff --git a/README.md b/README.md index 2e758c2..944441a 100644 --- a/README.md +++ b/README.md @@ -8,28 +8,28 @@ MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.or [![codecov](https://codecov.io/github/jannisborn/paperscraper/branch/main/graph/badge.svg?token=Clwi0pu61a)](https://codecov.io/github/jannisborn/paperscraper) # paperscraper -`paperscraper` is a `python` package for scraping publication metadata or full text files (PDF or XML) from -**PubMed** or preprint servers such as **arXiv**, **medRxiv**, **bioRxiv** and **chemRxiv**. -It provides a streamlined interface to scrape metadata, allows to retrieve citation counts -from Google Scholar, impact factors from journals and comes with simple postprocessing functions -and plotting routines for meta-analysis. +`paperscraper` is a `python` package for scraping publication metadata or full text files +(PDF or XML) from +[PubMed](https://pubmed.ncbi.nlm.nih.gov/) or preprint servers such as +[arXiv](https://arxiv.org/), [medRxiv](https://www.medrxiv.org/), +[bioRxiv](https://www.biorxiv.org/), and [chemRxiv](https://chemrxiv.org/). +It provides a streamlined interface to scrape metadata, retrieve citation counts +from [Google Scholar](https://scholar.google.com/), query journal impact factors, +and run simple postprocessing and plotting routines for meta-analysis. ## Table of Contents 1. [Getting Started](#getting-started) - - [Download X-rxiv Dumps](#download-x-rxiv-dumps) - - [Arxiv Local Dump](#arxiv-local-dump) + - [Download xRxiv Dumps](#download-xrxiv-dumps) + - [arXiv Local Dump](#arxiv-local-dump) 2. [Examples](#examples) - - [Publication Keyword Search](#publication-keyword-search) - - [Full-Text Retrieval (PDFs & XMLs)](#full-text-retrieval-pdfs--xmls) - - [Citation Search](#citation-search) - - [Journal Impact Factor](#journal-impact-factor) -3. [Plotting](#plotting) - - [Barplots](#barplots) - - [Venn Diagrams](#venn-diagrams) -4. [Citation](#citation) -5. [Contributions](#contributions) + - [Paper Keyword Analysis](#paper-keyword-analysis) + - [PDF Retrieval](#pdf-retrieval) + - [Self-Citation Analysis](#self-citation-analysis) + - [Scholar Metrics Analysis](#scholar-metrics-analysis) +3. [Citation](#citation) +4. [Contributors](#contributors) ## Getting started @@ -37,7 +37,14 @@ and plotting routines for meta-analysis. pip install paperscraper ``` -This is enough to query PubMed, arXiv or Google Scholar. +or, with [uv](https://docs.astral.sh/uv/): + +```console +uv add paperscraper +``` + +This is enough to query [PubMed](https://pubmed.ncbi.nlm.nih.gov/), +[arXiv](https://arxiv.org/) or [Google Scholar](https://scholar.google.com/). ### Local development @@ -51,9 +58,12 @@ This installs the project and dev tooling into `.venv`. Use `uv run` to execute uv run python -c "import paperscraper" ``` -#### Download X-rxiv Dumps +#### Download xRxiv Dumps -However, to scrape publication data from the preprint servers [biorxiv](https://www.biorxiv.org), [medrxiv](https://www.medrxiv.org) and [chemrxiv](https://www.chemrxiv.org), the setup is different. The entire history of papers is downloaded and stored in the `server_dumps` folder in a `.jsonl` format (one paper per line). This takes a while, as of November 2025: +However, to scrape publication data from the preprint servers [bioRxiv](https://www.biorxiv.org), +[medRxiv](https://www.medrxiv.org/) and [chemRxiv](https://chemrxiv.org/), the setup is +different. The entire history of papers is downloaded and stored in the `server_dumps` +folder in JSONL format (one paper per line). This takes a while, as of November 2025: ```py from paperscraper.get_dumps import biorxiv, medrxiv, chemrxiv @@ -74,14 +84,18 @@ biorxiv( ) ``` -Since v0.2.5 `paperscraper` also allows to scrape {med/bio/chem}rxiv for specific dates. +`paperscraper` also allows scraping {med/bio/chem}rxiv for specific dates. ```py medrxiv(start_date="2023-04-01", end_date="2023-04-08") ``` But watch out. The resulting `.jsonl` file will be labelled according to the current date and all your subsequent searches will be based on this file **only**. If you use this option you might want to keep an eye on the source files (`paperscraper/server_dumps/*jsonl`) to ensure they contain the paper metadata for all papers you're interested in. -#### Arxiv local dump -Local search can be faster than using the arxiv API especially if you plan many queries. Paperscraper provides two backends to bulk-download arxiv, `kaggle` and `arxiv`. The default is `kaggle` since it is much faster. Before using it, authenticate with your Kaggle account: +#### arXiv local dump +Local search can be faster than using the [arXiv API](https://info.arxiv.org/help/api/index.html), +especially if you plan many queries. Paperscraper provides two backends to bulk-download +arXiv, [Kaggle](https://www.kaggle.com/) and the +[`arxiv`](https://pypi.org/project/arxiv/) package. The default is `kaggle` since it is +much faster. Before using it, authenticate with your Kaggle account: ```sh kaggle auth login @@ -91,14 +105,16 @@ kaggle auth login from paperscraper.get_dumps import arxiv arxiv(start_date='2019-01-01', end_date='2026-12-31') ``` -NOTE: The disadvantage of `kaggle` backend is that it bulk-downloads **all** of arXiv. For small API-backed dumps, better use the `arxiv` PyPI package backend: +NOTE: The disadvantage of the `kaggle` backend is that it bulk-downloads **all** of +[arXiv](https://arxiv.org/). For small API-backed dumps, better use the +[`arxiv`](https://pypi.org/project/arxiv/) PyPI package backend: ```py from paperscraper.get_dumps import arxiv arxiv(start_date='2024-01-01',end_date='2024-01-04',backend='api') ``` -Afterwards you can search the local arxiv dump just like the other x-rxiv dumps. +Afterwards you can search the local arXiv dump just like the other x-rxiv dumps. The direct endpoint is `paperscraper.arxiv.get_arxiv_papers_local`. You can also specify the backend directly in the `get_and_dump_arxiv_papers` function: ```py @@ -108,336 +124,124 @@ get_and_dump_arxiv_papers(..., backend='local') ## Examples -`paperscraper` is build on top of the packages [arxiv](https://pypi.org/project/arxiv/), [pymed](https://pypi.org/project/pymed-paperscraper/), and [scholarly](https://pypi.org/project/scholarly/). +`paperscraper` is built on top of [`arxiv`](https://pypi.org/project/arxiv/), +[`pymed`](https://pypi.org/project/pymed-paperscraper/), and +[`scholarly`](https://pypi.org/project/scholarly/). + +The README keeps examples short. The +[documentation site](https://jannisborn.github.io/paperscraper/) has fuller walkthroughs: -### Publication keyword search +- [Paper Keyword Analysis](https://jannisborn.github.io/paperscraper/examples/paper-keyword-analysis/) +- [PDF Retrieval](https://jannisborn.github.io/paperscraper/examples/pdf-retrieval/) +- [Self-Citation Analysis](https://jannisborn.github.io/paperscraper/examples/self-citation-analysis/) +- [Scholar Metrics Analysis](https://jannisborn.github.io/paperscraper/examples/scholar-metrics-analysis/) -Consider you want to perform a publication keyword search with the query: -`Artificial Intelligence` **AND** `Quantum Computing` **AND** `Chemistry`. +### Paper keyword analysis -* Scrape papers from PubMed: +Nested keyword lists encode Boolean logic: outer lists are `AND`, inner lists are +synonyms with `OR`. ```py from paperscraper.pubmed import get_and_dump_pubmed_papers -ai = ['Artificial intelligence', 'Machine learning'] -qc = [ - 'Quantum computing', 'Quantum computer', 'Quantum information', - 'Quantum algorithm', 'Quantum circuit', 'Quantum simulation', - 'Quantum machine learning', 'Qubit', 'Quantum annealing' -] -chemistry = ['Chemistry', 'Chemical', 'Molecule', 'Molecular', 'Materials science'] -query = [ai, qc, chemistry] - -get_and_dump_pubmed_papers(query, output_filepath='ai_quantum_chemistry.jsonl') -``` -* Scrape papers from arXiv: +ai = ["Artificial intelligence", "Machine learning"] +qc = ["Quantum computing", "Quantum information", "Quantum algorithm"] +chemistry = ["Chemistry", "Chemical", "Molecule", "Materials science"] -```py -from paperscraper.arxiv import get_and_dump_arxiv_papers - -get_and_dump_arxiv_papers(query, output_filepath='ai_quantum_chemistry.jsonl') +get_and_dump_pubmed_papers([ai, qc, chemistry], "ai_quantum_chemistry.jsonl") ``` -* Scrape papers from bioRiv, medRxiv or chemRxiv: - -```py -from paperscraper.xrxiv.xrxiv_query import XRXivQuery - -querier = XRXivQuery('server_dumps/chemrxiv_2020-11-10.jsonl') -querier.search_keywords(query, output_filepath='ai_quantum_chemistry.jsonl') -``` +This writes matching [PubMed](https://pubmed.ncbi.nlm.nih.gov/) records to +`ai_quantum_chemistry.jsonl`. -You can also use `dump_queries` to iterate over a bunch of queries for all available databases. +For local [bioRxiv](https://www.biorxiv.org/), [medRxiv](https://www.medrxiv.org/), or +[chemRxiv](https://chemrxiv.org/) search, download the dumps once and use +`dump_queries` to query all available backends: ```py from paperscraper import dump_queries +from paperscraper.get_dumps import biorxiv, medrxiv, chemrxiv -physics = [ - 'Physics', 'Physical', 'Particle', 'Condensed matter', - 'Many-body', 'Fermion', 'Hamiltonian', 'Spin' -] -biology = ['Biology', 'Biological', 'Cellular', 'Genomics', 'Gene', 'Protein'] -queries = [[ai, qc, chemistry], [ai, qc, physics], [ai, qc, biology]] -dump_queries(queries, '.') -``` - -Or use the harmonized interface of `QUERY_FN_DICT` to query multiple databases of your choice: -```py -from paperscraper.load_dumps import QUERY_FN_DICT -print(QUERY_FN_DICT.keys()) - -QUERY_FN_DICT['biorxiv'](query, output_filepath='biorxiv_ai_quantum_chemistry.jsonl') -QUERY_FN_DICT['chemrxiv'](query, output_filepath='chemrxiv_ai_quantum_chemistry.jsonl') -``` - -* Scrape papers from Google Scholar: - -Thanks to [scholarly](https://pypi.org/project/scholarly/), there is an endpoint for Google Scholar too. -It does not understand Boolean expressions like the others, but should be used just like -the [Google Scholar search fields](https://scholar.google.com). +biorxiv() +medrxiv() +chemrxiv() -```py -from paperscraper.scholar import get_and_dump_scholar_papers -topic = 'Machine Learning' -get_and_dump_scholar_papers(topic) +# Restart Python after downloading dumps, then rerun or import your query lists. +dump_queries([[ai, qc, chemistry]], ".") ``` -*NOTE*: The scholar endpoint does not require authentication but since it regularly prompts with captchas, it's difficult to apply large scale. - -### Full-Text Retrieval (PDFs & XMLs) -`paperscraper` allows you to download full text of publications using DOIs. The basic functionality works reliably for preprint servers (arXiv, bioRxiv, medRxiv, chemRxiv), but retrieving papers from PubMed dumps is more challenging due to publisher restrictions and paywalls. +See the [paper keyword analysis example](https://jannisborn.github.io/paperscraper/examples/paper-keyword-analysis/) +for [arXiv](https://arxiv.org/), [Google Scholar](https://scholar.google.com/), +multi-database querying, and plotting. -#### Standard Usage +### PDF Retrieval -The main download functions work for all paper types with automatic fallbacks: +Download a PDF or XML by DOI: ```py from paperscraper.pdf import save_pdf -paper_data = {'doi': "10.48550/arXiv.2207.03928"} -save_pdf(paper_data, filepath='gt4sd_paper.pdf') -``` - -To batch download full texts from your metadata search results: - -```py -from paperscraper.pdf import save_pdf_from_dump - -# Save PDFs/XMLs in current folder and name the files by their DOI -save_pdf_from_dump('ai_quantum_chemistry.jsonl', pdf_path='.', key_to_save='doi') -``` - -#### Automatic Fallback Mechanisms - -When the standard text retrieval fails, `paperscraper` automatically tries these fallbacks: - -- **BioC-PMC**: For biomedical papers in [PubMed Central](https://pmc.ncbi.nlm.nih.gov/) (open-access repository), it retrieves open-access full-text XML from the [BioC-PMC API](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PMC/). -- **eLife Papers**: For [eLife](https://elifesciences.org/) journal papers, it fetches XML files from eLife's open [GitHub repository](https://github.com/elifesciences/elife-article-xml). - -These fallbacks are tried automatically without requiring any additional configuration. - -#### Enhanced Retrieval with Publisher APIs - -For more comprehensive access to papers from major publishers, you can provide API keys for: - -- **Wiley TDM API**: Enables access to [Wiley](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) publications (2,000+ journals). -- **Elsevier TDM API**: Enables access to [Elsevier](https://www.elsevier.com/about/policies-and-standards/text-and-data-mining) publications (The Lancet, Cell, ...). -- **bioRxiv TDM API** Enable access to [bioRxiv](https://www.biorxiv.org/tdm) publications (since May 2025 bioRxiv is protected with Cloudflare) - -To use publisher APIs: - -1. Create a file with your API keys: -``` -WILEY_TDM_API_TOKEN=your_wiley_token_here -ELSEVIER_TDM_API_KEY=your_elsevier_key_here -AWS_ACCESS_KEY_ID=your_aws_access_key_here -AWS_SECRET_ACCESS_KEY=your_aws_secret_key_here -``` -NOTE: The AWS keys can be created in your AWS/IAM account. When creating the key, make sure you tick the `AmazonS3ReadOnlyAccess` permission policy. -NOTE: If you name the file `.env` it will be loaded automatically (if it is in the cwd or anywhere above the tree to home). -2. Pass the file path when calling retrieval functions: - -```py -from paperscraper.pdf import save_pdf_from_dump - -save_pdf_from_dump( - 'pubmed_query_results.jsonl', - pdf_path='./papers', - key_to_save='doi', - api_keys='path/to/your/api_keys.txt' -) +save_pdf({"doi": "10.48550/arXiv.2207.03928"}, filepath="gt4sd_paper.pdf") ``` -For obtaining API keys: -- Wiley TDM API: Visit [Wiley Text and Data Mining](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) (free for academic users with institutional subscription) -- Elsevier TDM API: Visit [Elsevier's Text and Data Mining](https://www.elsevier.com/about/policies-and-standards/text-and-data-mining) (free for academic users with institutional subscription) - -*NOTE*: While these fallback mechanisms improve retrieval success rates, they cannot guarantee access to all papers due to various access restrictions. - - -### Citation search - -You can fetch the number of citations of a paper from its title or DOI - -```py -from paperscraper.citations import get_citations_from_title, get_citations_by_doi -title = 'Über formal unentscheidbare Sätze der Principia Mathematica und verwandter Systeme I.' -print(get_citations_from_title(title)) - -doi = '10.1021/acs.jcim.3c00132' -get_citations_by_doi(doi) -``` +Output: `True` when the file was saved. -NOTE: This uses the [Semantic Scholar API](https://www.semanticscholar.org/product/api/tutorial) which is bandwidth-limited. If you have an API Key set it via: -```sh -export SS_API_KEY=YOUR_API_KEY -``` -This will increase your throughput for using `paperscraper.citations` based on the rate limits of your key. +See the [PDF retrieval example](https://jannisborn.github.io/paperscraper/examples/pdf-retrieval/) +for batch downloads, fallbacks, publisher API keys, and downstream PDF analysis. +### Self-citation analysis -### Journal impact factor +Estimate paper-level self-citations and self-references: -You can also retrieve the impact factor for all journals: -```py ->>>from paperscraper.impact import Impactor ->>>i = Impactor() ->>>i.search("Nat Comms", threshold=85, sort_by='impact') -[ - {'journal': 'Nature Communications', 'factor': 17.694, 'score': 94}, - {'journal': 'Natural Computing', 'factor': 1.504, 'score': 88} -] -``` -This performs a fuzzy search with a threshold of 85. `threshold` defaults to 100 in which case an exact search -is performed. You can also search by journal abbreviation, [E-ISSN](https://portal.issn.org) or [NLM ID](https://portal.issn.org). ```py -i.search("Nat Rev Earth Environ") # [{'journal': 'Nature Reviews Earth & Environment', 'factor': 37.214, 'score': 100}] -i.search("101771060") # [{'journal': 'Nature Reviews Earth & Environment', 'factor': 37.214, 'score': 100}] -i.search('2662-138X') # [{'journal': 'Nature Reviews Earth & Environment', 'factor': 37.214, 'score': 100}] - -# Filter results by impact factor -i.search("Neural network", threshold=85, min_impact=1.5, max_impact=20) -# [ -# {'journal': 'IEEE Transactions on Neural Networks and Learning Systems', 'factor': 14.255, 'score': 93}, -# {'journal': 'NEURAL NETWORKS', 'factor': 9.657, 'score': 91}, -# {'journal': 'WORK-A Journal of Prevention Assessment & Rehabilitation', 'factor': 1.803, 'score': 86}, -# {'journal': 'NETWORK-COMPUTATION IN NEURAL SYSTEMS', 'factor': 1.5, 'score': 92} -# ] - -# Show all fields -i.search("quantum information", threshold=90, return_all=True) -# [ -# {'factor': 10.758, 'jcr': 'Q1', 'journal_abbr': 'npj Quantum Inf', 'eissn': '2056-6387', 'journal': 'npj Quantum Information', 'nlm_id': '101722857', 'issn': '', 'score': 92}, -# {'factor': 1.577, 'jcr': 'Q3', 'journal_abbr': 'Nation', 'eissn': '0027-8378', 'journal': 'NATION', 'nlm_id': '9877123', 'issn': '0027-8378', 'score': 91} -# ] -``` - - -## Plotting - -When multiple query searches are performed, two types of plots can be generated -automatically: Venn diagrams and bar plots. +from paperscraper.citations import self_citations_paper, self_references_paper -### Barplots - -Compare the temporal evolution of different queries across different servers. - -```py -import os - -from paperscraper import QUERY_FN_DICT -from paperscraper.postprocessing import aggregate_paper -from paperscraper.utils import get_filename_from_query, load_jsonl - -# Define search terms and their synonyms -ai = ['Artificial intelligence', 'Machine learning'] -qc = [ - 'Quantum computing', 'Quantum computer', 'Quantum information', - 'Quantum algorithm', 'Quantum circuit', 'Quantum simulation', - 'Quantum machine learning', 'Qubit', 'Quantum annealing' -] -chemistry = ['Chemistry', 'Chemical', 'Molecule', 'Molecular', 'Materials science'] -physics = [ - 'Physics', 'Physical', 'Particle', 'Condensed matter', - 'Many-body', 'Fermion', 'Hamiltonian', 'Spin' -] -biology = ['Biology', 'Biological', 'Cellular', 'Genomics', 'Gene', 'Protein'] -medicine = ['Medicine', 'Medical', 'Clinical', 'Disease', 'Patient', 'Health'] - -# Define queries -queries = [ - [ai, qc, chemistry], - [ai, qc, physics], - [ai, qc, biology], - [ai, qc, medicine], -] - -root = '../keyword_dumps' - -data_dict = dict() -for query in queries: - filename = get_filename_from_query(query) - data_dict[filename] = dict() - for db,_ in QUERY_FN_DICT.items(): - # Assuming the keyword search has been performed already - data = load_jsonl(os.path.join(root, db, filename)) - - # Unstructured matches are aggregated into 8 bins, 1 per year - # from 2019 to 2026. Sanity check is performed by having - # `filtering=True`, removing papers that don't contain all of - # the keywords in query. - data_dict[filename][db], filtered = aggregate_paper( - data, 2019, bins_per_year=1, filtering=True, - filter_keys=query, return_filtered=True, last_year=2026 - ) - -# Plotting is now very simple -from paperscraper.plotting import plot_comparison - -data_keys = [ - 'artificialintelligence_quantumcomputing_chemistry.jsonl', - 'artificialintelligence_quantumcomputing_physics.jsonl', - 'artificialintelligence_quantumcomputing_biology.jsonl', - 'artificialintelligence_quantumcomputing_medicine.jsonl' -] -plot_comparison( - data_dict, - data_keys, - x_ticks=[str(year) for year in range(2019, 2027)], - title_text="'Artificial intelligence' AND 'Quantum computing' AND X", - keyword_text=['Chemistry', 'Physics', 'Biology', 'Medicine'], - figpath='assets/ai_quantum_fields.png' -) +doi = "10.1038/s41586-023-06600-9" +self_citations_paper(doi).citation_score +self_references_paper(doi).reference_score ``` -![Artificial intelligence and quantum computing by field](assets/ai_quantum_fields.png) +Output: `3.192` and `5.05`, the mean self-citation and self-reference percentages +across paper authors. +See the [self-citation analysis example](https://jannisborn.github.io/paperscraper/examples/self-citation-analysis/) +for paper- and author-level workflows using [Semantic Scholar](https://www.semanticscholar.org/). -### Venn Diagrams +### Scholar metrics analysis -The Venn diagrams below use the local arXiv, bioRxiv, ChemRxiv and medRxiv dumps. +Get paper citation counts and journal metrics: ```py -from paperscraper.plotting import ( - plot_venn_two, plot_venn_three, plot_multiple_venn -) - -sizes_2024 = (18762, 6324, 624) -sizes_2025 = (18774, 6966, 775, 28652, 1965, 597, 77) -labels_2024 = ['Artificial\nIntelligence', 'Quantum\nComputing'] -labels_2025 = ('Artificial\nIntelligence', 'Quantum\nComputing', 'Chemistry') - -plot_venn_two( - sizes_2024, labels_2024, title='2024', figpath='assets/ai_quantum_venn_2024.png' -) -``` - -![2024 Venn diagram](assets/ai_quantum_venn_2024.png) +from paperscraper.citations import get_citations_by_doi +from paperscraper.impact import Impactor - -```py -plot_venn_three( - sizes_2025, labels_2025, title='2025', - figpath='assets/ai_quantum_chemistry_venn_2025.png' -) +get_citations_by_doi("10.1021/acs.jcim.3c00132") +Impactor().search("Nat Comms", threshold=85, sort_by="impact") ``` -![2025 Venn diagram](assets/ai_quantum_chemistry_venn_2025.png) +Outputs: `12` citations, then matching journal records such as +`Nature Communications` with impact factor `15.7`. -Or plot both together: +Author-level [Semantic Scholar](https://www.semanticscholar.org/) metrics can be retrieved by +Semantic Scholar ID, name, or [ORCID](https://orcid.org/): ```py -plot_multiple_venn( - [sizes_2024, sizes_2025], [labels_2024, labels_2025], - titles=['2024', '2025'], suptitle='Keyword search comparison', - gridspec_kw={'width_ratios': [1, 2]}, figsize=(10, 6), - figpath='assets/ai_quantum_venn_both.png' -) +from paperscraper.citations.orcid import orcid_to_author_name +from paperscraper.citations.utils import author_name_to_ssaid, semantic_scholar_requests_get + +ssaid, name = author_name_to_ssaid(orcid_to_author_name("0000-0001-8307-5670")) +metrics = semantic_scholar_requests_get( + f"https://api.semanticscholar.org/graph/v1/author/{ssaid}", + params={"fields": "paperCount,citationCount,hIndex"}, +).json() ``` -![Venn diagram comparison](assets/ai_quantum_venn_both.png) - +Output: a JSON object with `paperCount`, `citationCount`, and `hIndex` +(for example, `63`, `1910`, and `21` for the ORCID above). +See the [scholar metrics analysis example](https://jannisborn.github.io/paperscraper/examples/scholar-metrics-analysis/) +for [Google Scholar](https://scholar.google.com/), [Semantic Scholar](https://www.semanticscholar.org/), +researcher metrics, and journal impact factors. ## Citation If you use `paperscraper`, please cite a paper that motivated our development of this tool. @@ -455,18 +259,18 @@ If you use `paperscraper`, please cite a paper that motivated our development of } ``` -## Contributions +## Contributors Thanks to the following contributors: -- [@mathinic](https://github.com/mathinic): Since `v0.3.0` improved PubMed full text retrieval with additional fallback mechanisms (BioC-PMC, eLife and optional Wiley/Elsevier APIs). - -- [@memray](https://github.com/memray): Since `v0.2.12` there are automatic retries when downloading the {med/bio/chem}rxiv dumps. - -- [@achouhan93](https://github.com/achouhan93): Since `v0.2.5` {med/bio/chem}rxiv can be scraped for specific dates! - -- [@daenuprobst](https://github.com/daenuprobst): Since `v0.2.4` PDF files can be scraped directly (`paperscraper.pdf.save_pdf`) - -- [@oppih](https://github.com/oppih): Since `v0.2.3` chemRxiv API also provides DOI and URL if available - -- [@lukasschwab](https://github.com/lukasschwab): Enabled support for `arxiv` >`1.4.2` in paperscraper `v0.1.0`. -- [@juliusbierk](https://github.com/juliusbierk): Bugfixes +- [@mathinic](https://github.com/mathinic): improved PubMed full text retrieval with + additional fallback mechanisms ([BioC-PMC](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PMC/), + [eLife](https://elifesciences.org/) and optional Wiley/Elsevier APIs). +- [@memray](https://github.com/memray): added automatic retries when downloading the + {med/bio/chem}rxiv dumps. +- [@achouhan93](https://github.com/achouhan93): added date-bounded scraping for + {med/bio/chem}rxiv. +- [@daenuprobst](https://github.com/daenuprobst): added direct PDF scraping via + `paperscraper.pdf.save_pdf`. +- [@oppih](https://github.com/oppih): added chemRxiv DOI and URL metadata where available. +- [@lukasschwab](https://github.com/lukasschwab): enabled support for `arxiv` > `1.4.2`. +- [@juliusbierk](https://github.com/juliusbierk): bug fixes. diff --git a/docs/api_reference.md b/docs/api_reference.md deleted file mode 100644 index 225e126..0000000 --- a/docs/api_reference.md +++ /dev/null @@ -1,8 +0,0 @@ -# API Reference - -::: paperscraper - options: - show_if_no_docstring: false - show_submodules: true - filters: - - "!^_[^_]" diff --git a/docs/examples/paper-keyword-analysis.md b/docs/examples/paper-keyword-analysis.md new file mode 100644 index 0000000..be7b855 --- /dev/null +++ b/docs/examples/paper-keyword-analysis.md @@ -0,0 +1,207 @@ +# Paper Keyword Analysis + +Search terms are represented as nested lists. The outer list is interpreted as +`AND`; inner lists are interpreted as synonyms with `OR`. + +The examples below use the query: +`Artificial Intelligence` **AND** `Quantum Computing` **AND** `Chemistry`. + +## PubMed + +```py +from paperscraper.pubmed import get_and_dump_pubmed_papers + +ai = ["Artificial intelligence", "Machine learning"] +qc = [ + "Quantum computing", "Quantum computer", "Quantum information", + "Quantum algorithm", "Quantum circuit", "Quantum simulation", + "Quantum machine learning", "Qubit", "Quantum annealing", +] +chemistry = ["Chemistry", "Chemical", "Molecule", "Molecular", "Materials science"] +query = [ai, qc, chemistry] + +get_and_dump_pubmed_papers(query, output_filepath="ai_quantum_chemistry.jsonl") +``` + +## arXiv + +```py +from paperscraper.arxiv import get_and_dump_arxiv_papers + +get_and_dump_arxiv_papers(query, output_filepath="ai_quantum_chemistry.jsonl") +``` + +## bioRxiv, medRxiv, and chemRxiv + +Download local xRxiv dumps once: + +```py +from paperscraper.get_dumps import biorxiv, medrxiv, chemrxiv + +chemrxiv() +medrxiv() +biorxiv() +``` + +Restart Python after downloading dumps so `paperscraper.load_dumps` can discover +the new files. + +For local xRxiv dumps, use `XRXivQuery` directly: + +```py +from paperscraper.xrxiv.xrxiv_query import XRXivQuery + +querier = XRXivQuery("server_dumps/chemrxiv_2020-11-10.jsonl") +querier.search_keywords(query, output_filepath="ai_quantum_chemistry.jsonl") +``` + +`QUERY_FN_DICT` is mostly internal. It can be useful if you want to query only +one specific preprint server after local dumps have been downloaded: + +```py +from paperscraper.load_dumps import QUERY_FN_DICT + +QUERY_FN_DICT["biorxiv"](query, output_filepath="biorxiv_ai_quantum_chemistry.jsonl") +QUERY_FN_DICT["chemrxiv"](query, output_filepath="chemrxiv_ai_quantum_chemistry.jsonl") +``` + +To run several queries across all available backends: + +```py +from paperscraper import dump_queries + +physics = [ + "Physics", "Physical", "Particle", "Condensed matter", + "Many-body", "Fermion", "Hamiltonian", "Spin", +] +biology = ["Biology", "Biological", "Cellular", "Genomics", "Gene", "Protein"] +queries = [[ai, qc, chemistry], [ai, qc, physics], [ai, qc, biology]] + +dump_queries(queries, ".") +``` + +## Google Scholar + +Google Scholar uses a plain text query: + +```py +from paperscraper.scholar import get_and_dump_scholar_papers + +get_and_dump_scholar_papers("Machine Learning") +``` + +Google Scholar does not use the nested Boolean query syntax. It follows the +search behavior of the Google Scholar search box and may prompt captchas during +large automated runs. + +## Plotting + +After aggregating keyword-analysis results, you can visualize temporal trends +and overlaps. + +### Bar Plots + +Use `aggregate_paper` to bin matched papers by year, then pass the aggregated +counts to `plot_comparison`. This compact workflow loops over every available +backend via `QUERY_FN_DICT`. + +```py +import os + +from paperscraper import QUERY_FN_DICT +from paperscraper.postprocessing import aggregate_paper +from paperscraper.utils import get_filename_from_query, load_jsonl + +medicine = ["Medicine", "Medical", "Clinical", "Disease", "Patient", "Health"] +queries = [ + [ai, qc, chemistry], + [ai, qc, physics], + [ai, qc, biology], + [ai, qc, medicine], +] + +root = "../keyword_dumps" +data_dict = {} + +for query in queries: + filename = get_filename_from_query(query) + data_dict[filename] = {} + + for db, _ in QUERY_FN_DICT.items(): + data = load_jsonl(os.path.join(root, db, filename)) + data_dict[filename][db], _filtered = aggregate_paper( + data, + 2019, + bins_per_year=1, + filtering=True, + filter_keys=query, + return_filtered=True, + last_year=2026, + ) +``` + +```py +from paperscraper.plotting import plot_comparison + +data_keys = [ + "artificialintelligence_quantumcomputing_chemistry.jsonl", + "artificialintelligence_quantumcomputing_physics.jsonl", + "artificialintelligence_quantumcomputing_biology.jsonl", + "artificialintelligence_quantumcomputing_medicine.jsonl", +] + +plot_comparison( + data_dict, + data_keys, + x_ticks=[str(year) for year in range(2019, 2027)], + title_text="'Artificial intelligence' AND 'Quantum computing' AND X", + keyword_text=["Chemistry", "Physics", "Biology", "Medicine"], + figpath="assets/ai_quantum_fields.png", +) +``` + +![Artificial intelligence and quantum computing by field](https://raw.githubusercontent.com/jannisborn/paperscraper/main/assets/ai_quantum_fields.png) + +### Venn Diagrams + +The Venn diagrams below use the local arXiv, bioRxiv, chemRxiv, and medRxiv +dumps. + +```py +from paperscraper.plotting import plot_venn_two, plot_venn_three, plot_multiple_venn + +sizes_2024 = (18762, 6324, 624) +sizes_2025 = (18774, 6966, 775, 28652, 1965, 597, 77) +labels_2024 = ["Artificial\nIntelligence", "Quantum\nComputing"] +labels_2025 = ("Artificial\nIntelligence", "Quantum\nComputing", "Chemistry") + +plot_venn_two( + sizes_2024, + labels_2024, + title="2024", + figpath="assets/ai_quantum_venn_2024.png", +) + +plot_venn_three( + sizes_2025, + labels_2025, + title="2025", + figpath="assets/ai_quantum_chemistry_venn_2025.png", +) + +plot_multiple_venn( + [sizes_2024, sizes_2025], + [labels_2024, labels_2025], + titles=["2024", "2025"], + suptitle="Keyword search comparison", + gridspec_kw={"width_ratios": [1, 2]}, + figsize=(10, 6), + figpath="assets/ai_quantum_venn_both.png", +) +``` + +![2024 Venn diagram](https://raw.githubusercontent.com/jannisborn/paperscraper/main/assets/ai_quantum_venn_2024.png) + +![2025 Venn diagram](https://raw.githubusercontent.com/jannisborn/paperscraper/main/assets/ai_quantum_chemistry_venn_2025.png) + +![Venn diagram comparison](https://raw.githubusercontent.com/jannisborn/paperscraper/main/assets/ai_quantum_venn_both.png) diff --git a/docs/examples/pdf-retrieval.md b/docs/examples/pdf-retrieval.md new file mode 100644 index 0000000..3f645df --- /dev/null +++ b/docs/examples/pdf-retrieval.md @@ -0,0 +1,85 @@ +# PDF Retrieval + +`paperscraper` downloads full text from DOI metadata. Preprint servers are the +most reliable targets; journal papers can depend on open-access status, +publisher pages, and institutional/API access. + +## Single DOI + +Download a single paper by DOI: + +```pycon +>>> from paperscraper.pdf import save_pdf +>>> paper = {"doi": "10.48550/arXiv.2207.03928"} +>>> save_pdf(paper, filepath="gt4sd_paper.pdf") +True +``` + +Pass `save_metadata=True` to store paper metadata next to the downloaded file: + +```pycon +>>> save_pdf(paper, filepath="gt4sd_paper.pdf", save_metadata=True) +True +``` + +## Batch Downloads + +Download PDFs or XMLs from a metadata dump: + +```py +from paperscraper.pdf import save_pdf_from_dump + +save_pdf_from_dump( + "ai_quantum_chemistry.jsonl", + pdf_path="papers", + key_to_save="doi", +) +``` + +`key_to_save` can be `"doi"`, `"title"`, or `"date"`. + +## Fallbacks + +When direct PDF retrieval fails, `paperscraper` tries supported fallbacks: + +- BioC-PMC XML for open-access papers in PubMed Central. +- eLife XML from the eLife article XML repository. +- Publisher APIs when matching credentials are available. +- bioRxiv S3 access when AWS requester-pays credentials are provided. + +These improve retrieval success, but they cannot bypass publisher restrictions +or paywalls. + +## Publisher API Keys + +Publisher API keys can be supplied via a file or loaded from `.env`: + +```txt +WILEY_TDM_API_TOKEN=your_wiley_token_here +ELSEVIER_TDM_API_KEY=your_elsevier_key_here +AWS_ACCESS_KEY_ID=your_aws_access_key_here +AWS_SECRET_ACCESS_KEY=your_aws_secret_key_here +``` + +Then pass the path when downloading from a dump: + +```py +save_pdf_from_dump( + "pubmed_query_results.jsonl", + pdf_path="papers", + key_to_save="doi", + api_keys="api_keys.txt", +) +``` + +Wiley and Elsevier TDM APIs are generally free for eligible academic users with +institutional access. For bioRxiv S3 access, use an AWS IAM key with +`AmazonS3ReadOnlyAccess`. + +## Downstream Analysis + +Retrieved PDFs can be passed to document conversion and analysis tools. For +example, [Docling](https://github.com/docling-project/docling) can convert PDFs +into structured text/Markdown for downstream extraction, indexing, or RAG +pipelines. See the [Docling technical report](https://arxiv.org/abs/2408.09869) +for details. diff --git a/docs/examples/scholar-metrics-analysis.md b/docs/examples/scholar-metrics-analysis.md new file mode 100644 index 0000000..fc4069c --- /dev/null +++ b/docs/examples/scholar-metrics-analysis.md @@ -0,0 +1,170 @@ +# Scholar Metrics Analysis + +This page collects examples for paper citation counts, researcher-level +Semantic Scholar metrics, and journal impact factors. + +## Paper Citation Counts + +Fetch citation counts by DOI using Semantic Scholar: + +```pycon +>>> from paperscraper.citations import get_citations_by_doi +>>> get_citations_by_doi("10.1021/acs.jcim.3c00132") +12 # Semantic Scholar citation count. +``` + +You can also query Google Scholar by title: + +```pycon +>>> from paperscraper.citations import get_citations_from_title +>>> title = "GT4SD: Generative Toolkit for Scientific Discovery" +>>> get_citations_from_title(title) +9 # Google Scholar citation count. +``` + +Citation counts can differ between Semantic Scholar and Google Scholar. Google +Scholar queries may trigger captchas during automated use. Semantic Scholar +throughput can be increased with `SS_API_KEY`: + +```sh +export SS_API_KEY=YOUR_API_KEY +``` + +## Researcher Metrics + +Semantic Scholar author pages expose `paperCount`, `citationCount`, and `hIndex`. +You can query them by Semantic Scholar Author ID: + +```py +from paperscraper.citations.utils import semantic_scholar_requests_get + +ssaid = "2062641025" +metrics = semantic_scholar_requests_get( + f"https://api.semanticscholar.org/graph/v1/author/{ssaid}", + params={"fields": "name,paperCount,citationCount,hIndex"}, +).json() +``` + +```text +{ + "authorId": "2062641025", + "name": "Jannis Born", + "paperCount": 63, + "citationCount": 1910, + "hIndex": 21, +} +``` + +Resolve the same author by name: + +```pycon +>>> from paperscraper.citations.utils import author_name_to_ssaid +>>> author_name_to_ssaid("Jannis Born") +("2062641025", "Jannis Born") +``` + +Or resolve through ORCID first: + +```pycon +>>> from paperscraper.citations.orcid import orcid_to_author_name +>>> from paperscraper.citations.utils import author_name_to_ssaid +>>> name = orcid_to_author_name("0000-0001-8307-5670") +>>> author_name_to_ssaid(name) +("2062641025", "Jannis Born") +``` + +Citation and paper counts can change as Semantic Scholar updates author records. + +## Journal Impact Factors + +Use `Impactor` to search journal names, abbreviations, E-ISSNs, or NLM IDs. + +```pycon +>>> from paperscraper.impact import Impactor +>>> impactor = Impactor() +>>> impactor.search("Nat Comms", threshold=85, sort_by="impact") +[ + {"journal": "Nature Computational Science", "factor": 18.3, "score": 88}, + {"journal": "Nature Communications", "factor": 15.7, "score": 94}, + {"journal": "Natural Computing", "factor": 1.6, "score": 88}, +] +``` + +`threshold` defaults to `100`, which behaves like an exact search. Lower values +allow fuzzier matches. `sort_by` can be `"impact"`, `"journal"`, or `"score"`. + +Search by abbreviation, NLM ID, or E-ISSN: + +```pycon +>>> impactor.search("Nat Rev Earth Environ") +[{"journal": "Nature Reviews Earth & Environment", "factor": 71.5, "score": 100}] +>>> impactor.search("101771060") +[{"journal": "Nature Reviews Earth & Environment", "factor": 71.5, "score": 100}] +>>> impactor.search("2662-138X") +[{"journal": "Nature Reviews Earth & Environment", "factor": 71.5, "score": 100}] +``` + +Filter by impact factor range: + +```pycon +>>> impactor.search("Neural network", threshold=85, min_impact=1.5, max_impact=20) +[ + {"journal": "IEEE Transactions on Neural Networks and Learning Systems", "factor": 8.9, "score": 93}, + {"journal": "NEURAL NETWORKS", "factor": 6.3, "score": 91}, + {"journal": "Network", "factor": 3.1, "score": 92}, + {"journal": "NETWORK-COMPUTATION IN NEURAL SYSTEMS", "factor": 1.6, "score": 92}, + {"journal": "WORK-A Journal of Prevention Assessment & Rehabilitation", "factor": 1.5, "score": 86}, +] +``` + +Return all available fields: + +```pycon +>>> impactor.search("quantum information", threshold=90, return_all=True) +[ + { + "factor": 8.3, + "jcr": "Q1", + "nlm_id": "101722857", + "journal": "npj Quantum Information", + "issn": ".", + "zky": ".", + "journal_abbr": "npj Quantum Inf", + "eissn": "2056-6387", + "score": 92, + }, + { + "factor": 2.9, + "jcr": "Q2", + "nlm_id": "101703749", + "journal": "Information", + "issn": ".", + "zky": ".", + "journal_abbr": "Information (Basel)", + "eissn": "2078-2489", + "score": 95, + }, + { + "factor": 1.3, + "jcr": "Q2", + "nlm_id": "9877123", + "journal": "NATION", + "issn": "0027-8378", + "zky": ".", + "journal_abbr": "Nation", + "eissn": "0027-8378", + "score": 91, + }, + { + "factor": 1.1, + "jcr": ".", + "nlm_id": "138060", + "journal": "Reformation", + "issn": "1357-4175", + "zky": ".", + "journal_abbr": "Reformation", + "eissn": "1752-0738", + "score": 90, + }, +] +``` diff --git a/docs/examples/self-citation-analysis.md b/docs/examples/self-citation-analysis.md new file mode 100644 index 0000000..8b72a6b --- /dev/null +++ b/docs/examples/self-citation-analysis.md @@ -0,0 +1,101 @@ +# Self-Citation Analysis + +The citations submodule uses Semantic Scholar paper, citation, and reference +metadata to estimate how often authors cite or reference their own work. Set +`SS_API_KEY` to increase throughput: + +```sh +export SS_API_KEY=YOUR_API_KEY +``` + +Citation and reference counts can change as Semantic Scholar updates its records. + +## Paper-Level Self-Links + +Estimate self-citations and self-references for a paper: + +```pycon +>>> from paperscraper.citations import self_citations_paper, self_references_paper +>>> doi = "10.1038/s41586-023-06600-9" +>>> self_citations = self_citations_paper(doi) +>>> self_citations.num_citations +141 # Total citations. +>>> self_citations.citation_score +3.192 # Mean self-citation percentage across paper authors. +>>> self_references = self_references_paper(doi) +>>> self_references.num_references +33 # Total references. +>>> self_references.reference_score +5.05 # Mean self-reference percentage across paper authors. +``` + +Both functions accept either one DOI/Semantic Scholar paper ID or a list. A +single input returns one result object; a list returns a list of result objects. + +## Author Breakdown + +Print the per-author self-link percentages: + +```pycon +>>> self_citations.self_citations +{ + "Abhishek Sharma": 3.55, + "Dániel Czégel": 0.71, + "Michael Lachmann": 1.42, + "C. Kempes": 3.55, + "S. I. Walker": 4.96, + "Leroy Cronin": 4.96, +} # Percentage of citations that include each paper author. +>>> self_references.self_references +{ + "Abhishek Sharma": 3.03, + "Dániel Czégel": 0.0, + "Michael Lachmann": 0.0, + "C. Kempes": 0.0, + "S. I. Walker": 6.06, + "Leroy Cronin": 21.21, +} # Percentage of references that include each paper author. +``` + +## Author-Level Summary + +For an author-level summary, use `Researcher`. Full author analyses can take +longer for large publication lists, so this example limits the run to one paper. + +```pycon +>>> from paperscraper.citations.entity import Researcher +>>> researcher = Researcher("2289839817") +>>> researcher.ssids = ["2c1edb95c07643a834c9d4f8f2acedfecfe894de"] +>>> _ = researcher.self_citations() +>>> result = researcher.self_references() +>>> result.name +"K. Wijk" +>>> result.self_citation_ratio +0.0 # Mean self-citation percentage across the selected papers. +>>> result.self_reference_ratio +4.65 # Mean self-reference percentage across the selected papers. +>>> result.num_citations +10 +>>> result.num_references +43 +>>> result.self_references +{"Diff-SPORT: Diffusion-based Sensor Placement Optimization and Reconstruction of Turbulent flows in urban environments": 4.65} +``` + +## Unified Paper Interface + +Use `SelfLinkClient` when you want self-citations and self-references through one +paper-level object: + +```pycon +>>> from paperscraper.citations import SelfLinkClient +>>> client = SelfLinkClient("10.1038/s41586-023-06600-9", mode="paper") +>>> client.extract() +>>> result = client.get_result() +>>> result.title +"Assembly theory explains and quantifies selection and evolution" +>>> result.citation_score +3.192 # Mean self-citation percentage across paper authors. +>>> result.reference_score +5.05 # Mean self-reference percentage across paper authors. +``` diff --git a/mkdocs.yml b/mkdocs.yml index 35b36e1..44ced1c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -16,6 +16,7 @@ theme: favicon: favicon.png features: - content.code.annotate + - content.code.copy # - content.tabs.link - content.tooltips # - header.autohide @@ -60,6 +61,11 @@ nav: - paperscraper.pubmed: api/pubmed.md - paperscraper.scholar: api/scholar.md - paperscraper.xrxiv: api/xrxiv.md + - Examples: + - Paper Keyword Analysis: examples/paper-keyword-analysis.md + - PDF Retrieval: examples/pdf-retrieval.md + - Self-Citation Analysis: examples/self-citation-analysis.md + - Scholar Metrics Analysis: examples/scholar-metrics-analysis.md # Copyright copyright: MIT License diff --git a/paperscraper/__init__.py b/paperscraper/__init__.py index 9298b10..d910f4b 100644 --- a/paperscraper/__init__.py +++ b/paperscraper/__init__.py @@ -1,7 +1,7 @@ """Initialize the module.""" __name__ = "paperscraper" -__version__ = "0.3.6" +__version__ = "1.0.0" import logging import os diff --git a/paperscraper/arxiv/arxiv.py b/paperscraper/arxiv/arxiv.py index 98f42e0..1b00913 100644 --- a/paperscraper/arxiv/arxiv.py +++ b/paperscraper/arxiv/arxiv.py @@ -137,8 +137,8 @@ def get_and_dump_arxiv_papers( start_date: str = "None", end_date: str = "None", backend: Literal["api", "local", "infer"] = "api", - *args, - **kwargs, + *args: object, + **kwargs: object, ): """ Combines get_arxiv_papers and dump_papers. @@ -159,7 +159,8 @@ def get_and_dump_arxiv_papers( is queried (has to be downloaded before). If `infer` the local dump will be used if exists, otherwise API will be queried. Defaults to `api` since it is faster. - *args, **kwargs are additional arguments for `get_arxiv_papers`. + *args: Additional positional arguments for `get_arxiv_papers`. + **kwargs: Additional keyword arguments for `get_arxiv_papers`. """ # Translate keywords into query. query = get_query_from_keywords(keywords, start_date=start_date, end_date=end_date) diff --git a/paperscraper/plotting.py b/paperscraper/plotting.py index 3856e42..fef6fbd 100644 --- a/paperscraper/plotting.py +++ b/paperscraper/plotting.py @@ -1,6 +1,6 @@ import logging import math -from typing import Iterable, List, Optional +from typing import Any, Iterable, List, Optional import matplotlib.pyplot as plt import numpy as np @@ -15,7 +15,16 @@ def plot_comparison( data_dict: dict, keys: List[str], - x_ticks: List[str] = ["2015", "2016", "2017", "2018", "2019", "2020"], + x_ticks: List[str] = [ + "2019", + "2020", + "2021", + "2022", + "2023", + "2024", + "2025", + "2026", + ], show_preprint: bool = False, title_text: str = "", keyword_text: Optional[List[str]] = None, @@ -27,20 +36,21 @@ def plot_comparison( data_dict: A dictionary with keywords as keys. Each value should be a dictionary itself, with keys for the different APIs. For example data_dict = { - 'covid_19.jsonl': { - 'pubmed': [0, 0, 0, 12345], - 'arxiv': [0, 0, 0, 1234], + 'artificialintelligence_quantumcomputing_chemistry.jsonl': { + 'pubmed': [0, 0, 1, 3, 7, 15, 31, 42], + 'arxiv': [2, 5, 12, 24, 40, 63, 91, 120], ... } - 'coronavirus.jsonl': - 'pubmed': [234, 345, 456, 12345], - 'arxiv': [123, 234, 345, 1234], + 'artificialintelligence_quantumcomputing_physics.jsonl': { + 'pubmed': [1, 1, 2, 4, 9, 16, 27, 38], + 'arxiv': [12, 25, 44, 72, 110, 155, 210, 260], ... } } keys: List of keys which should be plotted. This has to be a subset of data_dict.keys(). x_ticks: List of strings to be used for the x-ticks. Should have same length as - data_dict[key][database]. Defaults to ['2015', '2016', '2017', '2018', '2019', '2020'], + each data series. Defaults to ['2019', '2020', '2021', '2022', '2023', + '2024', '2025', '2026'], meaning that papers are aggregated per year. show_preprint: Whether preprint servers are aggregated or not. Defaults to False. @@ -180,7 +190,16 @@ def get_step_size(x): def plot_single( data_dict: dict, keys: str, - x_ticks: List[str] = ["2015", "2016", "2017", "2018", "2019", "2020"], + x_ticks: List[str] = [ + "2019", + "2020", + "2021", + "2022", + "2023", + "2024", + "2025", + "2026", + ], show_preprint: bool = False, title_text: str = "", figpath: str = "comparison_plot.pdf", @@ -192,22 +211,22 @@ def plot_single( data_dict: A dictionary with keywords as keys. Each value should be a dictionary itself, with keys for the different APIs. For example data_dict = { - 'covid_19.jsonl': { - 'pubmed': [0, 0, 0, 12345], - 'arxiv': [0, 0, 0, 1234], + 'artificialintelligence_quantumcomputing_chemistry.jsonl': { + 'pubmed': [0, 0, 1, 3, 7, 15, 31, 42], + 'arxiv': [2, 5, 12, 24, 40, 63, 91, 120], ... } - 'coronavirus.jsonl': - 'pubmed': [234, 345, 456, 12345], - 'arxiv': [123, 234, 345, 1234], + 'artificialintelligence_quantumcomputing_physics.jsonl': { + 'pubmed': [1, 1, 2, 4, 9, 16, 27, 38], + 'arxiv': [12, 25, 44, 72, 110, 155, 210, 260], ... } } keys: A key which should be plotted. This has to be a subset of data_dict.keys(). x_ticks (List[str]): List of strings to be used for the x-ticks. Should have - same length as data_dict[key][database]. Defaults to ['2015', '2016', - '2017', '2018', '2019', '2020'], meaning that papers are aggregated per - year. + the same length as each data series. Defaults to ['2019', '2020', + '2021', '2022', '2023', '2024', '2025', '2026'], meaning that papers + are aggregated per year. show_preprint: Whether preprint servers are aggregated or not. Defaults to False. title_text: Title for the produced figure. Defaults to ''. @@ -348,7 +367,7 @@ def plot_venn_two( labels: List[str], figpath: str = "venn_two.pdf", title: str = "", - **kwargs, + **kwargs: Any, ) -> None: """Plot a single Venn Diagram with two terms. @@ -379,7 +398,11 @@ def plot_venn_two( def plot_venn_three( - sizes: List[int], labels: List[str], figpath: str = "", title: str = "", **kwargs + sizes: List[int], + labels: List[str], + figpath: str = "", + title: str = "", + **kwargs: Any, ) -> None: """Plot a single Venn Diagram with three terms. @@ -418,7 +441,7 @@ def plot_multiple_venn( suptitle: str = "", gridspec_kw: dict = {}, figsize: Iterable = (8, 4.5), - **kwargs, + **kwargs: Any, ) -> None: """Plots multiple Venn Diagrams next to each other diff --git a/paperscraper/pubmed/pubmed.py b/paperscraper/pubmed/pubmed.py index 95fa386..6f96486 100644 --- a/paperscraper/pubmed/pubmed.py +++ b/paperscraper/pubmed/pubmed.py @@ -35,8 +35,8 @@ def get_pubmed_papers( query: str, fields: List = ["title", "authors", "date", "abstract", "journal", "doi"], max_results: int = 9998, - *args, - **kwargs, + *args: object, + **kwargs: object, ) -> pd.DataFrame: """ Performs PubMed API request of a query and returns list of papers with diff --git a/paperscraper/tests/test_dump.py b/paperscraper/tests/test_dump.py index 7e54df6..8bea78a 100644 --- a/paperscraper/tests/test_dump.py +++ b/paperscraper/tests/test_dump.py @@ -21,7 +21,6 @@ logging.disable(logging.INFO) -covid19 = ["COVID-19", "SARS-CoV-2"] ai = ["Artificial intelligence", "Deep learning", "Machine learning"] mi = ["Medical imaging"] @@ -226,7 +225,7 @@ def test_arxiv_dumping(self): self.run_with_arxiv_retries( lambda: get_and_dump_arxiv_papers( ["MPEGO"], - output_filepath="covid19_ai_imaging.jsonl", + output_filepath="mpego_ai_imaging.jsonl", backend="api", max_results=5, client_options={ @@ -238,7 +237,7 @@ def test_arxiv_dumping(self): retries=5, sleep_seconds=10, ) - assert os.path.exists("covid19_ai_imaging.jsonl") + assert os.path.exists("mpego_ai_imaging.jsonl") def test_get_arxiv_date(self): def run_once(): diff --git a/paperscraper/tests/test_pdf.py b/paperscraper/tests/test_pdf.py index d58d455..b449026 100644 --- a/paperscraper/tests/test_pdf.py +++ b/paperscraper/tests/test_pdf.py @@ -63,11 +63,11 @@ def test_basic_search(self): # medrxiv now also seems cloudflare-controlled. skipping test # paper_data = {"doi": "10.1101/2020.09.02.20187096"} - # save_pdf(paper_data, filepath="covid_review.pdf", save_metadata=True) - # assert os.path.exists("covid_review.pdf") - # assert os.path.exists("covid_review.json") - # os.remove("covid_review.pdf") - # os.remove("covid_review.json") + # save_pdf(paper_data, filepath="medrxiv_review.pdf", save_metadata=True) + # assert os.path.exists("medrxiv_review.pdf") + # assert os.path.exists("medrxiv_review.json") + # os.remove("medrxiv_review.pdf") + # os.remove("medrxiv_review.json") # journal with OA paper paper_data = {"doi": "10.1038/s42256-023-00639-z"} diff --git a/paperscraper/xrxiv/tests/test_xrxiv.py b/paperscraper/xrxiv/tests/test_xrxiv.py index 9f13580..5aab1ce 100644 --- a/paperscraper/xrxiv/tests/test_xrxiv.py +++ b/paperscraper/xrxiv/tests/test_xrxiv.py @@ -3,8 +3,8 @@ from paperscraper.get_dumps import medrxiv from paperscraper.xrxiv.xrxiv_query import XRXivQuery -covid19 = ["COVID-19", "SARS-CoV-2"] ai = ["Artificial intelligence", "Deep learning", "Machine learning"] +qc = ["Quantum computing", "Quantum information", "Quantum algorithm"] mi = ["Medical imaging"] @@ -18,6 +18,6 @@ def test_get_medrxiv(self): def test_xriv_querier(self): querier = XRXivQuery("medriv_tmp_dump.jsonl") - query = [covid19, ai, mi] - querier.search_keywords(query, output_filepath="covid19_ai_imaging.jsonl") - assert os.path.exists("covid19_ai_imaging.jsonl") + query = [ai, qc, mi] + querier.search_keywords(query, output_filepath="ai_quantum_imaging.jsonl") + assert os.path.exists("ai_quantum_imaging.jsonl") diff --git a/pyproject.toml b/pyproject.toml index c8f9781..b2e558b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "paperscraper: Package to scrape papers." readme = "README.md" requires-python = ">=3.9,<3.14" -license = {text = "MIT"} +license = "MIT" authors = [ {name = "Jannis Born", email = "jannis.born@gmx.de"}, {name = "Matteo Manica", email = "drugilsberg@gmail.com"}, @@ -26,10 +26,9 @@ keywords = [ "Google Scholar", ] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -68,6 +67,7 @@ Repository = "https://github.com/jannisborn/paperscraper" [tool.setuptools] zip-safe = false +include-package-data = false [tool.setuptools.dynamic] version = {attr = "paperscraper.__version__"} @@ -76,9 +76,6 @@ version = {attr = "paperscraper.__version__"} where = ["."] include = ["paperscraper*"] -[tool.setuptools.package-data] -"paperscraper.server_dumps" = ["*"] - [dependency-groups] dev = [ "coverage", From d0a61b45aee764c0505230c681aafed5b4022c36 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Thu, 28 May 2026 00:01:04 +0200 Subject: [PATCH 2/3] doc --- README.md | 20 +++--- docs/examples/paper-keyword-analysis.md | 81 ++++++++++++++++++++--- docs/examples/pdf-retrieval.md | 16 +++++ docs/examples/scholar-metrics-analysis.md | 15 +++++ docs/examples/self-citation-analysis.md | 7 +- 5 files changed, 119 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 944441a..9833b5d 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,17 @@ biorxiv( ``` `paperscraper` also allows scraping {med/bio/chem}rxiv for specific dates. + ```py medrxiv(start_date="2023-04-01", end_date="2023-04-08") ``` -But watch out. The resulting `.jsonl` file will be labelled according to the current date and all your subsequent searches will be based on this file **only**. If you use this option you might want to keep an eye on the source files (`paperscraper/server_dumps/*jsonl`) to ensure they contain the paper metadata for all papers you're interested in. + +But watch out. The resulting `.jsonl` file will be labelled according to the +current date and all your subsequent searches will be based on this file **only**. +If you use this option you might want to keep an eye on the source files +(`paperscraper/server_dumps/*jsonl`) to ensure they contain the paper metadata +for all papers you're interested in. +Use `paperscraper.utils.get_server_dumps_dir()` to inspect the active dump directory. #### arXiv local dump Local search can be faster than using the [arXiv API](https://info.arxiv.org/help/api/index.html), @@ -155,18 +162,13 @@ This writes matching [PubMed](https://pubmed.ncbi.nlm.nih.gov/) records to `ai_quantum_chemistry.jsonl`. For local [bioRxiv](https://www.biorxiv.org/), [medRxiv](https://www.medrxiv.org/), or -[chemRxiv](https://chemrxiv.org/) search, download the dumps once and use -`dump_queries` to query all available backends: +[chemRxiv](https://chemrxiv.org/) search, download the dumps once as described in +[Download xRxiv Dumps](#download-xrxiv-dumps), restart Python, then use `dump_queries` +to query all available backends: ```py from paperscraper import dump_queries -from paperscraper.get_dumps import biorxiv, medrxiv, chemrxiv - -biorxiv() -medrxiv() -chemrxiv() -# Restart Python after downloading dumps, then rerun or import your query lists. dump_queries([[ai, qc, chemistry]], ".") ``` diff --git a/docs/examples/paper-keyword-analysis.md b/docs/examples/paper-keyword-analysis.md index be7b855..a147d17 100644 --- a/docs/examples/paper-keyword-analysis.md +++ b/docs/examples/paper-keyword-analysis.md @@ -23,6 +23,32 @@ query = [ai, qc, chemistry] get_and_dump_pubmed_papers(query, output_filepath="ai_quantum_chemistry.jsonl") ``` +Use `get_pubmed_papers` when you want a DataFrame in memory instead of a JSONL +file. PubMed can also return author emails when `"emails"` is included in +`fields`. + +```py +from paperscraper.pubmed import get_pubmed_papers + +papers = get_pubmed_papers( + "(machine learning) AND (zoology)", + fields=["title", "doi", "emails"], + max_results=50, +) +``` + +Date bounds and custom fields are available on the dump helper: + +```py +get_and_dump_pubmed_papers( + query, + output_filepath="pubmed_ai_quantum_chemistry_2024.jsonl", + fields=["title", "authors", "date", "doi", "emails"], + start_date="2024/01/01", + end_date="2024/12/31", +) +``` + ## arXiv ```py @@ -31,20 +57,32 @@ from paperscraper.arxiv import get_and_dump_arxiv_papers get_and_dump_arxiv_papers(query, output_filepath="ai_quantum_chemistry.jsonl") ``` -## bioRxiv, medRxiv, and chemRxiv +Use date bounds and `backend="infer"` when you want `paperscraper` to use a local +arXiv dump if one exists, otherwise fall back to the API: + +```py +get_and_dump_arxiv_papers( + query, + output_filepath="arxiv_ai_quantum_chemistry_2024.jsonl", + start_date="2024-01-01", + end_date="2024-12-31", + backend="infer", +) +``` -Download local xRxiv dumps once: +Use `get_arxiv_papers_api` when you want arXiv API results as a DataFrame: ```py -from paperscraper.get_dumps import biorxiv, medrxiv, chemrxiv +from paperscraper.arxiv import get_arxiv_papers_api -chemrxiv() -medrxiv() -biorxiv() +papers = get_arxiv_papers_api('all:"quantum machine learning"', max_results=25) ``` -Restart Python after downloading dumps so `paperscraper.load_dumps` can discover -the new files. +## bioRxiv, medRxiv, and chemRxiv + +Download local xRxiv dumps once using the +[setup instructions](../README.md#download-xrxiv-dumps), then restart Python so +`paperscraper.load_dumps` can discover the new files. For local xRxiv dumps, use `XRXivQuery` directly: @@ -90,6 +128,14 @@ from paperscraper.scholar import get_and_dump_scholar_papers get_and_dump_scholar_papers("Machine Learning") ``` +Use `get_scholar_papers` when you want the search results as a DataFrame: + +```py +from paperscraper.scholar import get_scholar_papers + +papers = get_scholar_papers("GT4SD") +``` + Google Scholar does not use the nested Boolean query syntax. It follows the search behavior of the Google Scholar search box and may prompt captchas during large automated runs. @@ -102,8 +148,9 @@ and overlaps. ### Bar Plots Use `aggregate_paper` to bin matched papers by year, then pass the aggregated -counts to `plot_comparison`. This compact workflow loops over every available -backend via `QUERY_FN_DICT`. +counts to `plot_comparison`. `aggregate_paper` can also remove false positives +with `unwanted_keys` or restrict filtering to titles with `filter_abstract=False`. +This compact workflow loops over every available backend via `QUERY_FN_DICT`. ```py import os @@ -160,6 +207,20 @@ plot_comparison( ) ``` +For one query at a time, use `plot_single`: + +```py +from paperscraper.plotting import plot_single + +plot_single( + data_dict, + [data_keys[0]], + x_ticks=[str(year) for year in range(2019, 2027)], + title_text="'Artificial intelligence' AND 'Quantum computing' AND Chemistry", + figpath="assets/ai_quantum_chemistry_single.png", +) +``` + ![Artificial intelligence and quantum computing by field](https://raw.githubusercontent.com/jannisborn/paperscraper/main/assets/ai_quantum_fields.png) ### Venn Diagrams diff --git a/docs/examples/pdf-retrieval.md b/docs/examples/pdf-retrieval.md index 3f645df..8dbe9a1 100644 --- a/docs/examples/pdf-retrieval.md +++ b/docs/examples/pdf-retrieval.md @@ -15,6 +15,9 @@ Download a single paper by DOI: True ``` +`filepath` can be provided with or without `.pdf`. XML fallbacks write an `.xml` +file next to the requested path when XML full text is the available format. + Pass `save_metadata=True` to store paper metadata next to the downloaded file: ```pycon @@ -72,6 +75,19 @@ save_pdf_from_dump( ) ``` +Or load the keys once and reuse them across calls: + +```py +from paperscraper.pdf import load_api_keys, save_pdf + +api_keys = load_api_keys("api_keys.txt") +save_pdf( + {"doi": "10.1101/786871"}, + filepath="taskload.pdf", + api_keys=api_keys, +) +``` + Wiley and Elsevier TDM APIs are generally free for eligible academic users with institutional access. For bioRxiv S3 access, use an AWS IAM key with `AmazonS3ReadOnlyAccess`. diff --git a/docs/examples/scholar-metrics-analysis.md b/docs/examples/scholar-metrics-analysis.md index fc4069c..b14dae2 100644 --- a/docs/examples/scholar-metrics-analysis.md +++ b/docs/examples/scholar-metrics-analysis.md @@ -30,6 +30,9 @@ throughput can be increased with `SS_API_KEY`: export SS_API_KEY=YOUR_API_KEY ``` +For larger author-level runs, `SS_REQUEST_TIMEOUT`, `SS_CONCURRENCY_LIMIT`, and +`SS_RATE_LIMIT_DELAY` can be tuned through environment variables. + ## Researcher Metrics Semantic Scholar author pages expose `paperCount`, `citationCount`, and `hIndex`. @@ -73,6 +76,18 @@ Or resolve through ORCID first: ("2062641025", "Jannis Born") ``` +If you need the actual Semantic Scholar paper IDs for an author, use +`get_papers_for_author`: + +```pycon +>>> from paperscraper.citations.utils import get_papers_for_author +>>> paper_ids = get_papers_for_author("2062641025") +>>> len(paper_ids) +63 # Number of papers linked to this Semantic Scholar author record. +>>> paper_ids[0] +'6c245545fcb88df49cf921ba0871b40818665b92' +``` + Citation and paper counts can change as Semantic Scholar updates author records. ## Journal Impact Factors diff --git a/docs/examples/self-citation-analysis.md b/docs/examples/self-citation-analysis.md index 8b72a6b..ddd8dae 100644 --- a/docs/examples/self-citation-analysis.md +++ b/docs/examples/self-citation-analysis.md @@ -85,7 +85,8 @@ longer for large publication lists, so this example limits the run to one paper. ## Unified Paper Interface Use `SelfLinkClient` when you want self-citations and self-references through one -paper-level object: +paper-level object. Paper inputs can be DOIs, Semantic Scholar paper IDs, or +titles; use `mode` when you want to disambiguate. ```pycon >>> from paperscraper.citations import SelfLinkClient @@ -99,3 +100,7 @@ paper-level object: >>> result.reference_score 5.05 # Mean self-reference percentage across paper authors. ``` + +`SelfLinkClient(..., mode="author")` is also available for author-level runs. +Use `Researcher` directly when you want to limit the paper list before running, +as shown in the author-level example above. From 148c48e27ca9f3c2aa230dc28f7a9e8767e4eca4 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Thu, 28 May 2026 00:03:27 +0200 Subject: [PATCH 3/3] feat: backoff when ss citation search --- paperscraper/citations/citations.py | 18 ++++-------------- paperscraper/citations/utils.py | 4 +++- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/paperscraper/citations/citations.py b/paperscraper/citations/citations.py index 6f0818a..4d3a5de 100644 --- a/paperscraper/citations/citations.py +++ b/paperscraper/citations/citations.py @@ -1,11 +1,10 @@ import logging import sys -from time import sleep from scholarly import scholarly from semanticscholar import SemanticScholarException -from .utils import PAPER_URL, semantic_scholar_requests_get +from .utils import PAPER_URL, _semantic_scholar_requests_get_with_backoff logging.basicConfig(stream=sys.stdout, level=logging.INFO) logger = logging.getLogger(__name__) @@ -23,10 +22,11 @@ def get_citations_by_doi(doi: str) -> int: """ try: - response = semantic_scholar_requests_get( + response = _semantic_scholar_requests_get_with_backoff( f"{PAPER_URL}DOI:{doi}", params={"fields": "citationCount"}, - timeout=20, + max_retries=14, + raise_for_status=False, ) if response.status_code == 404: logger.warning(f"Could not find paper {doi}, assuming 0 citation.") @@ -36,16 +36,6 @@ def get_citations_by_doi(doi: str) -> int: except SemanticScholarException.ObjectNotFoundException: logger.warning(f"Could not find paper {doi}, assuming 0 citation.") return 0 - except ConnectionRefusedError as e: - logger.warning(f"Waiting for 10 sec since {doi} gave: {e}") - sleep(10) - response = semantic_scholar_requests_get( - f"{PAPER_URL}DOI:{doi}", - params={"fields": "citationCount"}, - timeout=20, - ) - response.raise_for_status() - return response.json()["citationCount"] def get_citations_from_title(title: str) -> int: diff --git a/paperscraper/citations/utils.py b/paperscraper/citations/utils.py index 19be04c..fc8e79e 100644 --- a/paperscraper/citations/utils.py +++ b/paperscraper/citations/utils.py @@ -102,6 +102,7 @@ def _semantic_scholar_requests_get_with_backoff( factor: float = 1.3, max_delay: float = 60.0, jitter_ratio: float = 0.1, + raise_for_status: bool = True, **kwargs, ) -> requests.Response: """ @@ -132,7 +133,8 @@ def _semantic_scholar_requests_get_with_backoff( or 500 <= resp.status_code <= 599 ) if not retryable: - resp.raise_for_status() + if raise_for_status: + resp.raise_for_status() return resp sleep_for = min(delay, max_delay)