From 95d279c82e535372feab0bc57870d0299a060ae0 Mon Sep 17 00:00:00 2001 From: Jannis Born Date: Tue, 28 Jul 2026 19:41:13 +0200 Subject: [PATCH] refactor: homogenize usage of auth-header --- .github/workflows/test_tip.yml | 3 +++ paperscraper/get_dumps/utils/chemrxiv/crossref_api.py | 7 +++++-- paperscraper/pdf/fallbacks.py | 3 ++- paperscraper/pdf/pdf.py | 2 +- paperscraper/pubmed/pubmed.py | 5 ++++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_tip.yml b/.github/workflows/test_tip.yml index e2f7fd4..4aabcf6 100644 --- a/.github/workflows/test_tip.yml +++ b/.github/workflows/test_tip.yml @@ -2,6 +2,9 @@ name: Source on: [push, release] +env: + PAPERSCRAPER_EMAIL: jannis.born@gmx.de + jobs: test-source-install: runs-on: ubuntu-latest diff --git a/paperscraper/get_dumps/utils/chemrxiv/crossref_api.py b/paperscraper/get_dumps/utils/chemrxiv/crossref_api.py index a9ec4bf..9c33c91 100644 --- a/paperscraper/get_dumps/utils/chemrxiv/crossref_api.py +++ b/paperscraper/get_dumps/utils/chemrxiv/crossref_api.py @@ -10,6 +10,7 @@ """ import logging +import os import sys from time import sleep from typing import Dict, Generator, List, Optional @@ -58,7 +59,9 @@ def __init__( self.end_date = end_date self.page_size = min(max(1, page_size), 1000) self.max_retries = max_retries - self.mailto = mailto + self.mailto = mailto or os.getenv( + "PAPERSCRAPER_EMAIL", "your_email@example.com" + ) self.request_delay_seconds = max(0.0, request_delay_seconds) def iter_items(self) -> Generator[Dict, None, None]: @@ -140,7 +143,7 @@ def _request(self, params: Dict) -> Dict: headers = { "Accept": "application/json", - "User-Agent": "paperscraper (Crossref fallback)", + "User-Agent": "paperscraper", } for attempt in range(self.max_retries): diff --git a/paperscraper/pdf/fallbacks.py b/paperscraper/pdf/fallbacks.py index 1818fc4..c52f7a1 100644 --- a/paperscraper/pdf/fallbacks.py +++ b/paperscraper/pdf/fallbacks.py @@ -4,6 +4,7 @@ import datetime import io import logging +import os import re import sys import threading @@ -123,7 +124,7 @@ def fallback_bioc_pmc( bool: True if the XML file was successfully downloaded, False otherwise. """ ncbi_tool = "paperscraper" - ncbi_email = "your_email@example.com" + ncbi_email = os.getenv("PAPERSCRAPER_EMAIL", "your_email@example.com") converter_url = "https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/" params = { diff --git a/paperscraper/pdf/pdf.py b/paperscraper/pdf/pdf.py index 83c21fe..232a982 100644 --- a/paperscraper/pdf/pdf.py +++ b/paperscraper/pdf/pdf.py @@ -151,7 +151,7 @@ def save_pdf( api_keys = load_api_keys(api_keys) doi = paper_metadata["doi"] url = f"https://doi.org/{doi}" - user_agent = {"User-Agent": "paperscraper/1.0 (+https)"} + user_agent = {"User-Agent": "paperscraper"} success = False metadata_written = False diff --git a/paperscraper/pubmed/pubmed.py b/paperscraper/pubmed/pubmed.py index 6f96486..23f8752 100644 --- a/paperscraper/pubmed/pubmed.py +++ b/paperscraper/pubmed/pubmed.py @@ -12,7 +12,10 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -PUBMED = PubMed(tool=os.getenv("NCBI_TOOL", "paperscraper"), email="abc@def.gh") +PUBMED = PubMed( + tool="paperscraper", + email=os.getenv("PAPERSCRAPER_EMAIL", "your_email@example.com"), +) pubmed_field_mapper = {"publication_date": "date"}