diff --git a/paperscraper/pdf/__init__.py b/paperscraper/pdf/__init__.py index 555cbec..7f647ec 100644 --- a/paperscraper/pdf/__init__.py +++ b/paperscraper/pdf/__init__.py @@ -1 +1,2 @@ -from .pdf import load_api_keys, save_pdf, save_pdf_from_dump # noqa +from .pdf import load_api_keys, save_pdf, save_pdf_from_dump, debug_save_pdf, debug_save_pdf_from_dump # noqa + diff --git a/paperscraper/pdf/fallbacks.py b/paperscraper/pdf/fallbacks.py index 1818fc4..83a14fb 100644 --- a/paperscraper/pdf/fallbacks.py +++ b/paperscraper/pdf/fallbacks.py @@ -12,6 +12,8 @@ from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from pathlib import Path from typing import Any, Callable, Dict, Union +import threading +from collections import deque import boto3 import requests @@ -26,8 +28,70 @@ logger = logging.getLogger(__name__) -class NCBIRateLimitError(RuntimeError): - """Raised when NCBI returns a rate-limit response.""" +class WileyRateLimiter: + """ + Smart rate limiter for Wiley API that handles both: + - 3 articles per second + - 60 requests per 10 minutes + + Uses a token bucket approach for efficient rate limiting. + """ + + def __init__(self): + self._lock = threading.Lock() + # Token bucket for per-second limit (3 tokens, refill 3 per second) + self._per_second_tokens = 3.0 + self._per_second_capacity = 3.0 + self._per_second_refill_rate = 3.0 # tokens per second + self._last_refill = time.time() + + # Sliding window for 10-minute limit (60 requests per 600 seconds) + self._request_times = deque() + self._ten_minute_limit = 60 + self._ten_minute_window = 600 # seconds + + def acquire(self) -> float: + """ + Acquire permission to make a request. + Returns the time to wait before making the request (0 if immediate). + """ + with self._lock: + now = time.time() + + # Refill per-second tokens based on elapsed time + elapsed = now - self._last_refill + self._per_second_tokens = min( + self._per_second_capacity, + self._per_second_tokens + elapsed * self._per_second_refill_rate + ) + self._last_refill = now + + # Clean old requests from 10-minute window + cutoff = now - self._ten_minute_window + while self._request_times and self._request_times[0] < cutoff: + self._request_times.popleft() + + # Check 10-minute limit + if len(self._request_times) >= self._ten_minute_limit: + # Calculate how long to wait for oldest request to expire + wait_time = self._request_times[0] + self._ten_minute_window - now + return max(0, wait_time) + + # Check per-second limit + if self._per_second_tokens < 1.0: + # Calculate how long to wait for next token + wait_time = (1.0 - self._per_second_tokens) / self._per_second_refill_rate + return wait_time + + # Consume tokens and record request + self._per_second_tokens -= 1.0 + self._request_times.append(now) + + return 0.0 + + +# Global rate limiter instance +_wiley_rate_limiter = WileyRateLimiter() def fallback_wiley_api( @@ -37,11 +101,13 @@ def fallback_wiley_api( max_attempts: int = 2, ) -> bool: """ - Attempt to download the PDF via the Wiley TDM API (popular publisher which blocks standard scraping attempts; API access free for academic users). + Attempt to download the PDF via the Wiley TDM API with smart rate limiting. - This function uses the WILEY_TDM_API_TOKEN environment variable to authenticate - with the Wiley TDM API and attempts to download the PDF for the given paper. - See https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining for a description on how to get your WILEY_TDM_API_TOKEN. + Implements proper rate limiting for: + - up to 3 articles per second + - up to 60 requests per 10 minutes + + Uses token bucket algorithm for efficient handling of rate limits. Args: paper_metadata (dict): Dictionary containing paper metadata. Must include the 'doi' key. @@ -54,6 +120,10 @@ def fallback_wiley_api( """ WILEY_TDM_API_TOKEN = api_keys.get("WILEY_TDM_API_TOKEN") + if not WILEY_TDM_API_TOKEN: + logger.info("No Wiley API token found, skipping Wiley fallback.") + return False + encoded_doi = paper_metadata["doi"].replace("/", "%2F") api_url = f"https://api.wiley.com/onlinelibrary/tdm/v1/articles/{encoded_doi}" headers = {"Wiley-TDM-Client-Token": WILEY_TDM_API_TOKEN} @@ -63,13 +133,20 @@ def fallback_wiley_api( while attempt < max_attempts: try: + # Smart rate limiting - wait if necessary + wait_time = _wiley_rate_limiter.acquire() + if wait_time > 0: + logger.info(f"Wiley API rate limit: waiting {wait_time:.1f} seconds...") + time.sleep(wait_time) + api_response = requests.get( api_url, headers=headers, allow_redirects=True, timeout=60 ) api_response.raise_for_status() + if api_response.content[:4] != b"%PDF": logger.warning( - f"API returned content that is not a valid PDF for {paper_metadata['doi']}." + f"Wiley API returned content that is not a valid PDF for {paper_metadata['doi']}." ) else: with open(output_path.with_suffix(".pdf"), "wb+") as f: @@ -79,30 +156,28 @@ def fallback_wiley_api( ) success = True break - except Exception as e2: + + except requests.exceptions.HTTPError as e: + if e.response.status_code == 429: # Rate limit exceeded + # If we hit rate limit despite our limiter, wait longer + retry_after = int(e.response.headers.get('Retry-After', 30)) + logger.warning(f"Wiley API rate limit hit, waiting {retry_after} seconds...") + time.sleep(retry_after) + else: + logger.error(f"Wiley API HTTP error (attempt {attempt + 1}/{max_attempts}): {e}") + if attempt < max_attempts - 1: + time.sleep(5) # Brief pause before retry + except Exception as e: + logger.error(f"Wiley API error (attempt {attempt + 1}/{max_attempts}): {e}") if attempt < max_attempts - 1: - logger.info("Waiting 20 seconds before retrying...") - time.sleep(20) - logger.error( - f"Could not download via Wiley API (attempt {attempt + 1}/{max_attempts}): {e2}" - ) + time.sleep(5) # Brief pause before retry attempt += 1 - # **Mandatory delay of 10 seconds to comply with Wiley API rate limits** - logger.info( - "Waiting 10 seconds before next request to comply with Wiley API rate limits..." - ) - time.sleep(10) return success -def fallback_bioc_pmc( - doi: str, - output_path: Path, - max_attempts: int = 3, - retry_sleep: int = 10, -) -> bool: +def fallback_bioc_pmc(doi: str, output_path: Path, ncbi_email="your_email@example.com") -> bool: """ Attempt to download the XML via the BioC-PMC fallback. @@ -123,8 +198,9 @@ def fallback_bioc_pmc( bool: True if the XML file was successfully downloaded, False otherwise. """ ncbi_tool = "paperscraper" - ncbi_email = "your_email@example.com" - + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + } converter_url = "https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/" params = { "tool": ncbi_tool, @@ -133,31 +209,14 @@ def fallback_bioc_pmc( "idtype": "doi", "format": "json", } - for attempt in range(1, max_attempts + 1): - try: - conv_response = requests.get(converter_url, params=params, timeout=60) - if conv_response.status_code == 429: - raise NCBIRateLimitError( - f"NCBI rate-limited DOI to PMCID conversion for {doi}" - ) - conv_response.raise_for_status() - data = conv_response.json() - records = data.get("records", []) - if not records or "pmcid" not in records[0]: - logger.warning( - f"No PMCID available for DOI {doi}. Fallback via PMC therefore not possible." - ) - return False - pmcid = records[0]["pmcid"] - logger.info(f"Converted DOI {doi} to PMCID {pmcid}.") - break - except NCBIRateLimitError as conv_err: - if attempt == max_attempts: - logger.error(f"Error during DOI to PMCID conversion: {conv_err}") - return False - logger.info( - f"NCBI rate limit hit during DOI to PMCID conversion " - f"(attempt {attempt}/{max_attempts}); retrying" + try: + conv_response = requests.get(converter_url, params=params, headers=headers, timeout=60) + conv_response.raise_for_status() + data = conv_response.json() + records = data.get("records", []) + if not records or "pmcid" not in records[0]: + logger.warning( + f"No PMCID available for DOI {doi}. Fallback via PMC therefore not possible." ) time.sleep(retry_sleep * attempt) except Exception as conv_err: @@ -205,7 +264,10 @@ def fallback_bioc_pmc( def fallback_elsevier_api( - paper_metadata: Dict[str, Any], output_path: Path, api_keys: Dict[str, str] + paper_metadata: Dict[str, Any], + output_path: Path, + api_keys: Dict[str, str], + preferred_type: str = "xml", ) -> bool: """ Attempt to download the full text via the Elsevier TDM API. @@ -215,49 +277,80 @@ def fallback_elsevier_api( Args: paper_metadata (Dict[str, Any]): Dictionary containing paper metadata. Must include the 'doi' key. - output_path (Path): A pathlib.Path object representing the path where the XML file will be saved. + output_path (Path): A pathlib.Path object representing the path where the file will be saved. api_keys (Dict[str, str]): A dictionary containing API keys. Must include the key "ELSEVIER_TDM_API_KEY". + preferred_type (str): The preferred file type to download, either "xml" or "pdf". Defaults to "xml". Returns: - bool: True if the XML file was successfully downloaded, False otherwise. + bool: True if the file was successfully downloaded, False otherwise. """ elsevier_api_key = api_keys.get("ELSEVIER_TDM_API_KEY") + if not elsevier_api_key: + logger.info("No Elsevier API key found, skipping Elsevier fallback.") + return False + + if preferred_type not in ["xml", "pdf"]: + logger.warning( + f"Invalid preferred_type '{preferred_type}'. Defaulting to 'xml'." + ) + preferred_type = "xml" + doi = paper_metadata["doi"] - api_url = f"https://api.elsevier.com/content/article/doi/{doi}?apiKey={elsevier_api_key}&httpAccept=text%2Fxml" - logger.info(f"Attempting download via Elsevier API (XML) for {doi}: {api_url}") - headers = {"Accept": "application/xml"} + api_url = f"https://api.elsevier.com/content/article/doi/{doi}" + accept_header = ( + "application/xml" if preferred_type == "xml" else "application/pdf" + ) + headers = {"Accept": accept_header, "X-ELS-APIKey": elsevier_api_key} + + logger.info( + f"Attempting download via Elsevier API ({preferred_type.upper()}) for {doi}" + ) + try: response = requests.get(api_url, headers=headers, timeout=60) - # Check for 401 error and look for APIKEY_INVALID in the response - if response.status_code == 401: + if response.status_code in [401, 403]: error_text = response.text if "APIKEY_INVALID" in error_text: - logger.error("Invalid API key. Couldn't download via Elsevier XML API") + logger.error( + "Invalid API key. Couldn't download via Elsevier API." + ) else: - logger.error("401 Unauthorized. Couldn't download via Elsevier XML API") + logger.error( + f"{response.status_code} Unauthorized/Forbidden. Couldn't download via Elsevier API." + ) return False response.raise_for_status() - # Attempt to parse it with lxml to confirm it's valid XML - try: - etree.fromstring(response.content) - except etree.XMLSyntaxError as e: - logger.warning(f"Elsevier API returned invalid XML for {doi}: {e}") - return False + content = response.content + file_path = output_path.with_suffix(f".{preferred_type}") + + if preferred_type == "xml": + try: + etree.fromstring(content) + except etree.XMLSyntaxError as e: + logger.warning( + f"Elsevier API returned invalid XML for {doi}: {e}" + ) + return False + elif preferred_type == "pdf": + if not content.startswith(b"%PDF"): + logger.warning( + f"Elsevier API did not return a valid PDF for {doi}." + ) + return False - xml_path = output_path.with_suffix(".xml") - with open(xml_path, "wb") as f: - f.write(response.content) + with open(file_path, "wb") as f: + f.write(content) logger.info( - f"Successfully used Elsevier API to downloaded XML for {doi} to {xml_path}" + f"Successfully downloaded {preferred_type.upper()} via Elsevier API for {doi} to {file_path}" ) return True - except Exception as e: - logger.error(f"Could not download via Elsevier XML API: {e}") - return False + except requests.exceptions.RequestException as e: + logger.error(f"Could not download via Elsevier API for {doi}: {e}") + return False def fallback_elife_xml(doi: str, output_path: Path) -> bool: """ @@ -616,10 +709,484 @@ def job(k): return True +def fallback_unpaywall(doi: str, output_path: Union[str,Path], mail: str, final_url: str) -> bool: + """ + Attempt to download the PDF via Unpaywall. + Unpaywall is a service that finds open access versions of paywalled articles. + Args: + doi (str): The DOI of the paper to retrieve. + output_path (Path): A pathlib.Path object representing the path where the PDF will be saved. + mail (str): Email address to use for Unpaywall API requests. + final_url (str): The redirected URL of the DOI + Returns: + bool: True if the PDF file was successfully downloaded, False otherwise. + """ + if type(output_path) is str: + output_path = Path(output_path) + unpaywall_url = f"https://api.unpaywall.org/v2/{doi}?email={mail}" + try: + response = requests.get(unpaywall_url, timeout=60) + response.raise_for_status() + data = response.json() + if not data.get("is_oa", False): + logger.info(f"No open access version found for {doi} on Unpaywall.") + return False + pdf_url = data.get("best_oa_location", {}).get("url_for_pdf", None) + if final_url== pdf_url: + logger.info(f"Unpaywall returned the same URL as the redirected URL for {doi}") + return False + + if pdf_url: + pdf_response = requests.get(pdf_url, timeout=60) + pdf_response.raise_for_status() + if pdf_response.content[:4] == b"%PDF": + with open(output_path.with_suffix(".pdf"), "wb+") as f: + f.write(pdf_response.content) + logger.info(f"Successfully downloaded PDF via Unpaywall for {doi}.") + return True + else: + logger.warning(f"Unpaywall URL for {doi} did not return a valid PDF.") + return False + else: + logger.info(f"No open access PDF found on Unpaywall for {doi}.") + return False + except Exception as e: + logger.warning(f"Error during Unpaywall fallback for {doi}: {e}") + return False + +def fallback_springer_api( + paper_metadata: Dict[str, Any], + output_path: Path, + api_keys: Dict[str, str], +) -> bool: + """ + Attempt to download the PDF via the Springer Nature API. + This function uses the SPRINGER_API_KEY environment variable to authenticate. + See https://dev.springernature.com/ for details on how to get an API key. + Args: + paper_metadata (dict): Dictionary containing paper metadata. Must include the 'doi' key. + output_path (Path): A pathlib.Path object representing the path where the PDF will be saved. + api_keys (dict): Preloaded API keys. + Returns: + bool: True if the PDF file was successfully downloaded, False otherwise. + """ + springer_api_key = api_keys.get("SPRINGER_API_KEY") + if not springer_api_key: + logger.info("No Springer API key found, skipping Springer fallback.") + return False + + doi = paper_metadata["doi"] + # Try open access endpoint first + api_url = f"https://api.springernature.com/openaccess/v2/json?q=doi:{doi}&api_key={springer_api_key}" + try: + response = requests.get(api_url, timeout=60) + response.raise_for_status() + data = response.json() + if data.get("records"): + pdf_url = data["records"][0].get("url") + if pdf_url: + # The URL is often a list, take the first one which is usually the PDF + if isinstance(pdf_url, list): + pdf_url = pdf_url[0]["url"] + + pdf_response = requests.get(pdf_url, timeout=60) + pdf_response.raise_for_status() + if pdf_response.content[:4] == b"%PDF": + with open(output_path.with_suffix(".pdf"), "wb+") as f: + f.write(pdf_response.content) + logger.info(f"Successfully downloaded PDF via Springer Open Access API for {doi}.") + return True + + except Exception as e: + logger.info(f"Springer Open Access API failed for {doi}: {e}. Trying metadata API.") + + # Fallback to metadata API (TDM) + api_url = f"https://api.springernature.com/metadata/v2/json?q=doi:{doi}&api_key={springer_api_key}" + try: + response = requests.get(api_url, timeout=60) + response.raise_for_status() + data = response.json() + if data.get("records"): + pdf_url = data["records"][0].get("url") + if pdf_url: + if isinstance(pdf_url, list): + pdf_url = pdf_url[0]["url"] + + pdf_response = requests.get(pdf_url, timeout=60) + pdf_response.raise_for_status() + if pdf_response.content[:4] == b"%PDF": + with open(output_path.with_suffix(".pdf"), "wb+") as f: + f.write(pdf_response.content) + logger.info(f"Successfully downloaded PDF via Springer Metadata API for {doi}.") + return True + except Exception as e: + logger.error(f"Could not download via Springer API for {doi}: {e}") + + return False + + +def fallback_plos_api(doi: str, output_path: Path) -> bool: + """ + Attempt to download the PDF from PLOS journals. + PLOS articles are open access and their PDFs can often be downloaded directly. + Args: + doi (str): The DOI of the paper to retrieve. + output_path (Path): A pathlib.Path object representing the path where the PDF will be saved. + Returns: + bool: True if the PDF file was successfully downloaded, False otherwise. + """ + if "plos" not in doi.lower(): + return False + try: + # Construct the URL based on common PLOS URL patterns + # e.g., https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0000001&type=printable + journal_match = re.search(r'journal\.(\w+)', doi) + if not journal_match: + logger.warning(f"Could not determine PLOS journal from DOI: {doi}") + return False + journal_short_name = journal_match.group(1) + # 'pone' is a special case, it maps to 'plosone' in the URL + if journal_short_name == 'pone': + journal_name = 'plosone' + else: + journal_name = f'plos{journal_short_name}' + + pdf_url = f"https://journals.plos.org/{journal_name}/article/file?id={doi}&type=printable" + + pdf_response = requests.get(pdf_url, timeout=60) + pdf_response.raise_for_status() + if pdf_response.content[:4] == b"%PDF": + with open(output_path.with_suffix(".pdf"), "wb+") as f: + f.write(pdf_response.content) + logger.info(f"Successfully downloaded PDF from PLOS for {doi}.") + return True + else: + logger.warning(f"PLOS URL for {doi} did not return a valid PDF.") + return False + except Exception as e: + logger.error(f"Error during PLOS fallback for {doi}: {e}") + return False + + + +def fallback_europepmc(doi: str, output_path: Path) -> bool: + """ + Attempt to download the XML via Europe PMC. + + This function first converts a given DOI to a PMCID using the Europe PMC REST API. + If a PMCID is found, it attempts to download the full-text XML from Europe PMC. + + Europe PMC is a repository of biomedical and life sciences literature that provides + free access to abstracts and full-text articles. + + Args: + doi (str): The DOI of the paper to retrieve. + output_path (Path): A pathlib.Path object representing the path where the XML file will be saved. + + Returns: + bool: True if the XML file was successfully downloaded, False otherwise. + """ + # First, search for the article using DOI to get PMCID + search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search" + search_params = { + "query": f'DOI:"{doi}"', + "format": "json", + "resultType": "core" + } + + try: + search_response = requests.get(search_url, params=search_params, timeout=60) + search_response.raise_for_status() + search_data = search_response.json() + + results = search_data.get("resultList", {}).get("result", []) + if not results: + logger.warning(f"No results found for DOI {doi} in Europe PMC.") + return False + + # Search through all results to find one with a PMCID + pmcid = None + for result in results: + candidate_pmcid = result.get("pmcid") + if candidate_pmcid: + pmcid = candidate_pmcid + logger.info(f"Found PMCID {pmcid} for DOI {doi} in Europe PMC (result {results.index(result) + 1} of {len(results)}).") + break + + if not pmcid: + logger.warning(f"No PMCID available for DOI {doi} in Europe PMC (searched {len(results)} results).") + return False + + except Exception as search_err: + logger.error(f"Error searching Europe PMC for DOI {doi}: {search_err}") + return False + + # Download full-text XML using PMCID + xml_url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/fullTextXML" + + try: + xml_response = requests.get(xml_url, timeout=60) + xml_response.raise_for_status() + + # Check if we got valid XML content + xml_content = xml_response.content + if xml_content.startswith(b" bool: + """ + Use OpenAlex to locate an OA PDF for a DOI. + https://api.openalex.org/works/doi:{doi} + """ + try: + url = f"https://api.openalex.org/works/doi:{quote(doi)}" + r = requests.get(url, timeout=60) + if r.status_code == 404: + logger.info(f"OpenAlex: no record for {doi}") + return False + r.raise_for_status() + data = r.json() + + best = data.get("best_oa_location") or {} + pdf_url = best.get("pdf_url") + if not pdf_url: + # Fallbacks: try other locations OpenAlex exposes + primary = data.get("primary_location") or {} + pdf_url = primary.get("pdf_url") or (best.get("landing_page_url") if best.get("is_oa") else None) + + if not pdf_url: + logger.info(f"OpenAlex: no OA PDF for {doi}") + return False + + pdf = requests.get(pdf_url, timeout=60) + pdf.raise_for_status() + if not pdf.content.startswith(b"%PDF"): + logger.warning(f"OpenAlex PDF URL did not return a PDF for {doi}") + return False + + with open(output_path.with_suffix(".pdf"), "wb") as f: + f.write(pdf.content) + logger.info(f"Successfully downloaded PDF via OpenAlex for {doi}.") + return True + except Exception as e: + logger.error(f"OpenAlex fallback failed for {doi}: {e}") + return False + + +def fallback_crossref_links(doi: str, output_path: Path, contact_email: str = "your_email@example.com") -> bool: + """ + Use Crossref /works to find publisher-provided text-mining PDF links. + Prefers links with intended-application='text-mining' and content-type='application/pdf'. + """ + try: + url = f"https://api.crossref.org/works/{quote(doi)}" + headers = {"User-Agent": f"paperscraper (mailto:{contact_email})"} + r = requests.get(url, headers=headers, timeout=60) + r.raise_for_status() + msg = r.json().get("message", {}) + links = msg.get("link", []) or [] + + # Prioritize text-mining PDF links, then any PDF links + def score(link: dict) -> tuple: + return ( + 0 if link.get("intended-application") == "text-mining" else 1, + 0 if link.get("content-type") == "application/pdf" else 1, + ) + + links = sorted(links, key=score) + for link in links: + if link.get("content-type") != "application/pdf": + continue + pdf_url = link.get("URL") + if not pdf_url: + continue + try: + pdf = requests.get(pdf_url, headers=headers, timeout=60) + pdf.raise_for_status() + if pdf.content.startswith(b"%PDF"): + with open(output_path.with_suffix(".pdf"), "wb") as f: + f.write(pdf.content) + logger.info(f"Successfully downloaded PDF via Crossref link for {doi}.") + return True + except Exception as sub_e: + logger.info(f"Crossref link failed for {doi}: {sub_e}") + logger.info(f"Crossref: no usable PDF links for {doi}.") + return False + except Exception as e: + logger.error(f"Crossref fallback failed for {doi}: {e}") + return False + + +def fallback_arxiv(doi: str, output_path: Path) -> bool: + """ + If an arXiv preprint is associated with the DOI, fetch the arXiv PDF. + """ + try: + # arXiv Atom API supports DOI query + q = quote(f'doi:"{doi}"') + url = f"http://export.arxiv.org/api/query?search_query={q}&max_results=1" + r = requests.get(url, timeout=60) + r.raise_for_status() + root = etree.fromstring(r.content) + ns = {"a": "http://www.w3.org/2005/Atom"} + entry_id = root.find(".//a:entry/a:id", namespaces=ns) + if entry_id is None or not entry_id.text: + logger.info(f"arXiv: no entry for DOI {doi}") + return False + abs_url = entry_id.text.strip() + if "/abs/" not in abs_url: + logger.info(f"arXiv: unexpected entry URL for {doi}: {abs_url}") + return False + pdf_url = abs_url.replace("/abs/", "/pdf/") + ".pdf" + pdf = requests.get(pdf_url, timeout=60) + pdf.raise_for_status() + if not pdf.content.startswith(b"%PDF"): + logger.warning(f"arXiv URL did not return a PDF for {doi}") + return False + with open(output_path.with_suffix(".pdf"), "wb") as f: + f.write(pdf.content) + logger.info(f"Successfully downloaded PDF via arXiv for {doi}.") + return True + except Exception as e: + logger.error(f"arXiv fallback failed for {doi}: {e}") + return False + + +def month_folder_medrxiv(doi: str) -> str: + """ + Get medRxiv posting month folder, rolling over last-day postings to next month. + """ + url = f"https://api.medrxiv.org/details/medrxiv/{doi}/na/json" + resp = requests.get(url, timeout=30) + resp.raise_for_status() + date_str = resp.json()["collection"][0]["date"] + date = datetime.date.fromisoformat(date_str) + last_day = calendar.monthrange(date.year, date.month)[1] + if date.day == last_day: + date = date + datetime.timedelta(days=1) + return date.strftime("%B_%Y") + + +def fallback_medrxiv_s3( + doi: str, output_path: Union[str, Path], api_keys: dict, workers: int = 32 +) -> bool: + """ + Download a medRxiv PDF via the requester-pays S3 bucket using range requests. + """ + s3 = boto3.client( + "s3", + aws_access_key_id=api_keys.get("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=api_keys.get("AWS_SECRET_ACCESS_KEY"), + region_name="us-east-1", + ) + bucket = "medrxiv-src-monthly" + try: + prefix = f"Current_Content/{month_folder_medrxiv(doi)}/" + except Exception as e: + logger.error(f"Could not resolve medRxiv month folder for {doi}: {e}") + return False + + meca_keys = list_meca_keys(s3, bucket, prefix) + if not meca_keys: + logger.info(f"No MECA archives in {bucket}/{prefix} for {doi}") + return False + + token = doi.split("/")[-1].lower() + executor = ThreadPoolExecutor(max_workers=workers) + futures = {executor.submit(find_meca_for_doi, s3, bucket, key, token): key for key in meca_keys} + pbar = tqdm(total=len(futures), desc=f"Scanning in medrxiv with {workers} workers for {doi}…") + target = None + for fut in as_completed(futures): + key = futures[fut] + try: + if fut.result(): + target = key + pbar.set_description(f"Success! Found target {doi} in {key}") + for other in futures: + other.cancel() + break + except Exception: + pass + finally: + pbar.update(1) + executor.shutdown(wait=False) + if target is None: + logger.error(f"Could not find {doi} on medrxiv") + return False + + data = s3.get_object(Bucket=bucket, Key=target, RequestPayer="requester")["Body"].read() + output_path = Path(output_path) + with zipfile.ZipFile(io.BytesIO(data)) as z: + for name in z.namelist(): + if name.lower().endswith(".pdf"): + z.extract(name, path=output_path.parent) + (output_path.parent / name).rename(output_path.with_suffix(".pdf")) + return True + return False + + +def fallback_doaj(doi: str, output_path: Path) -> bool: + """ + Use DOAJ API to find fulltext links for OA articles. + """ + try: + url = f"https://doaj.org/api/v2/search/articles/doi:{quote(doi)}" + r = requests.get(url, timeout=60) + r.raise_for_status() + results = r.json().get("results", []) or [] + for res in results: + links = (res.get("bibjson", {}) or {}).get("link", []) or [] + for ln in links: + if ln.get("type") != "fulltext": + continue + pdf_url = ln.get("url") + if not pdf_url: + continue + try: + pdf = requests.get(pdf_url, timeout=60) + pdf.raise_for_status() + if not pdf.content.startswith(b"%PDF"): + continue + with open(output_path.with_suffix(".pdf"), "wb") as f: + f.write(pdf.content) + logger.info(f"Successfully downloaded PDF via DOAJ for {doi}.") + return True + except Exception: + continue + logger.info(f"DOAJ: no usable fulltext PDF for {doi}.") + return False + except Exception as e: + logger.error(f"DOAJ fallback failed for {doi}: {e}") + return False + + + FALLBACKS: Dict[str, Callable] = { "bioc_pmc": fallback_bioc_pmc, "elife": fallback_elife_xml, "elsevier": fallback_elsevier_api, + "europepmc": fallback_europepmc, "s3": fallback_s3, "wiley": fallback_wiley_api, + "unpaywall": fallback_unpaywall, + "springer": fallback_springer_api, + "plos": fallback_plos_api, + "openalex": fallback_openalex, + "crossref": fallback_crossref_links, + "arxiv": fallback_arxiv, + "medrxiv_s3": fallback_medrxiv_s3, + "doaj": fallback_doaj, } diff --git a/paperscraper/pdf/pdf.py b/paperscraper/pdf/pdf.py index 83c21fe..451d5fb 100644 --- a/paperscraper/pdf/pdf.py +++ b/paperscraper/pdf/pdf.py @@ -114,13 +114,89 @@ def _write_metadata(metadata: Dict[str, Any], output_path: Path) -> bool: logger.error(f"Failed to save metadata to {str(output_path)}: {exc}") return False +# python +def _get_abstract_pubmed(pmid: str, timeout: int = 20) -> Optional[str]: + """ + Query NCBI EFetch for PubMed and return the abstract text or None. + Uses the XML retmode and extracts all AbstractText nodes. + """ + try: + url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" + params = {"db": "pubmed", "id": pmid, "retmode": "xml"} + resp = requests.get(url, params=params, timeout=timeout) + resp.raise_for_status() + soup_xml = BeautifulSoup(resp.text, "xml") + abstract_texts = soup_xml.find_all("abstracttext") + if not abstract_texts: + return None + parts = [] + for node in abstract_texts: + parts.append(node.get_text("\n").strip()) + return "\n".join([p for p in parts if p]) + except Exception as e: + logger.warning(f"PubMed fetch failed for PMID={pmid}: {e}") + return None + +def _get_abstract_crossref(doi: str, timeout: int = 20) -> Optional[str]: + """ + Query Crossref works API and return the abstract (HTML cleaned) or None. + """ + try: + url = f"https://api.crossref.org/works/{doi}" + resp = requests.get(url, timeout=timeout) + resp.raise_for_status() + data = resp.json().get("message", {}) + raw = data.get("abstract") + if not raw: + return None + return BeautifulSoup(raw, "html.parser").get_text("\n").strip() + except Exception as e: + logger.warning(f"Crossref fetch failed for DOI={doi}: {e}") + return None + +# python +def _get_abstract_europepmc(doi: str, timeout: int = 20) -> Optional[str]: + """ + Query Europe PMC REST API for DOI and return the abstract (prefer `abstractText`) or None. + Uses resultType=core&format=json as requested. + """ + try: + url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search" + params = {"query": f"DOI:{doi}", "resultType": "core", "format": "json"} + resp = requests.get(url, params=params, timeout=timeout) + resp.raise_for_status() + data = resp.json() + results = data.get("resultList", {}).get("result", []) + if not results: + return None + first = results[0] + # Try keys likely returned: 'abstractText', 'abstract' (case-insensitive fallback) + abstract = first.get("abstractText") or first.get("abstract") + if not abstract: + # case-insensitive fallback + for k, v in first.items(): + if k.lower() == "abstracttext" or k.lower() == "abstract": + abstract = v + break + if not abstract: + return None + return BeautifulSoup(abstract, "html.parser").get_text("\n").strip() + except Exception as e: + logger.warning(f"EuropePMC fetch failed for DOI={doi}: {e}") + return None + +# --- Replace abstract retrieval section in save_pdf with the following block --- + + def save_pdf( paper_metadata: Dict[str, Any], filepath: Union[str, Path], save_metadata: bool = False, api_keys: Optional[Union[str, Dict[str, str]]] = None, -) -> bool: + preferred_type: str = "pdf", + mail: Optional[str] = None +) -> Dict[str, Any]: """ Save a PDF file of a paper. @@ -130,21 +206,21 @@ def save_pdf( save_metadata: A boolean indicating whether to save paper metadata as a separate json. api_keys: Either a dictionary containing API keys (if already loaded) or a string (path to API keys file). If None, will try to load from `.env` file and if unsuccessful, skip API-based fallbacks. - + preferred_type: Preferred file type to download, 'pdf' or 'xml'. Defaults to 'pdf'. Returns: - Whether the PDF was saved successfully + A dict summary: {success: bool, method: str|None, filetype: 'pdf'|'xml'|None} """ if not isinstance(paper_metadata, Dict): raise TypeError(f"paper_metadata must be a dict, not {type(paper_metadata)}.") if "doi" not in paper_metadata.keys(): raise KeyError("paper_metadata must contain the key 'doi'.") - if not isinstance(filepath, str): - raise TypeError(f"filepath must be a string, not {type(filepath)}.") + if not isinstance(filepath, (str, Path)): + raise TypeError(f"filepath must be a string or Path, not {type(filepath)}.") output_path = Path(filepath) - if not Path(output_path).parent.exists(): - raise ValueError(f"The folder: {output_path} seems to not exist.") + if not output_path.parent.exists(): + raise ValueError(f"The folder: {output_path.parent} seems to not exist.") # load API keys from file if not already loaded via in save_pdf_from_dump (dict) if not isinstance(api_keys, dict): @@ -153,113 +229,43 @@ def save_pdf( url = f"https://doi.org/{doi}" user_agent = {"User-Agent": "paperscraper/1.0 (+https)"} success = False - metadata_written = False + used_method: Optional[str] = None + used_filetype: Optional[str] = None + soup = None + final_url = None - # Forward to publisher URL - resolved_url = url try: - r_resolve = requests.head( - url, timeout=60, headers=user_agent, allow_redirects=True - ) - if r_resolve.url: - resolved_url = r_resolve.url - except Exception: - try: - r_resolve = requests.get( - url, timeout=60, headers=user_agent, allow_redirects=True - ) - if r_resolve.url: - resolved_url = r_resolve.url - except Exception: - pass + response = requests.get(url, timeout=60) + soup = BeautifulSoup(response.text, features="lxml") + response.raise_for_status() + final_url = response.url + soup = BeautifulSoup(response.text, features="lxml") + meta_pdf = soup.find("meta", {"name": "citation_pdf_url"}) + if meta_pdf and meta_pdf.get("content"): + pdf_url = meta_pdf.get("content") + pdf_response = requests.get(pdf_url, timeout=60) + pdf_response.raise_for_status() - # Arxiv PDFs can be downloaded directly - if "arxiv" in doi.lower(): - soup = None - try: - match = re.search( - r"arxiv\.([0-9]{4}\.[0-9]{4,5}(?:v\d+)?)", doi, re.IGNORECASE - ) - arxiv_id = match.group(1) - pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf" - r = requests.get(pdf_url, timeout=60, headers=user_agent) - r.raise_for_status() - if r.content[:4] == b"%PDF": + if pdf_response.content[:4] == b"%PDF": with open(output_path.with_suffix(".pdf"), "wb+") as f: - f.write(r.content) + f.write(pdf_response.content) success = True - # If metadata requested, fetch the landing page now to extract it - if save_metadata: - try: - resp_landing = requests.get(url, timeout=60, headers=user_agent) - resp_landing.raise_for_status() - soup = BeautifulSoup(resp_landing.text, features="lxml") - except Exception as _: - soup = None - else: - return True + used_method = "direct" + used_filetype = "pdf" else: logger.warning( - f"Direct arXiv fetch returned non-PDF for {doi}. Falling back." + f"The file from {pdf_url} does not appear to be a valid PDF." ) - except Exception as e: - logger.warning( - f"Direct arXiv PDF fetch failed for {doi}: {e}. Falling back." - ) - - if "chemrxiv" in doi.lower(): - item = _get_chemrxiv_item(doi, user_agent) - if item: - if save_metadata: - metadata_written = _write_metadata( - _chemrxiv_metadata_from_item(item, doi), output_path - ) - pdf_url = _chemrxiv_pdf_url(item) - if pdf_url: - try: - if download_pdf_to_path(pdf_url, output_path, user_agent): - return True - logger.warning( - f"ChemRxiv Open Engage PDF endpoint did not return a PDF: {pdf_url}" - ) - except Exception as e: - logger.warning( - f"ChemRxiv Open Engage PDF download failed for {doi}: {e}" - ) - else: - logger.warning(f"ChemRxiv API response missing PDF URL for {doi}") - # Try to load biorxiv PDF but may be blocked by Cloudflare - if is_biorxiv := "biorxiv" in resolved_url.lower(): - # Try manual download - response = requests.get(url, timeout=60) - - pdf_url = f"https://www.biorxiv.org/content/{doi}.full.pdf" - try: - if download_pdf_to_path(pdf_url, output_path, user_agent): - if not save_metadata: - return True - success = True - else: - logger.info( - f"Direct bioRxiv PDF endpoint did not return a PDF: {pdf_url}" - ) - except Exception as e: - logger.info(f"Direct bioRxiv PDF download failed: {pdf_url} ({e})") - - try: - response = requests.get(url, timeout=60) - soup = BeautifulSoup(response.text, features="lxml") - response.raise_for_status() - error = "" except Exception as e: - error = str(e) - logger.warning(f"Could not download from: {url} - {e}. ") - soup = None + logger.warning(f"Could not download from: {final_url} - {e}. Trying fallbacks.") + + if success: + if not save_metadata: + return {"success": True, "method": used_method, "filetype": used_filetype} - # Try to save the metadata - if soup is not None and save_metadata and not metadata_written: metadata = {} + # Extract title title_tag = soup.find("meta", {"name": "citation_title"}) metadata["title"] = title_tag.get("content") if title_tag else "Title not found" @@ -271,7 +277,7 @@ def save_pdf( metadata["authors"] = authors if authors else ["Author information not found"] # Extract abstract - domain = tldextract.extract(resolved_url).domain + domain = tldextract.extract(url).domain abstract_keys = ABSTRACT_ATTRIBUTE.get(domain, DEFAULT_ATTRIBUTES) for key in abstract_keys: @@ -279,7 +285,7 @@ def save_pdf( if abstract_tag: raw_abstract = BeautifulSoup( abstract_tag.get("content", "None"), "html.parser" - ).get_text(separator="\n") + ).get_text("\n") if raw_abstract.strip().startswith("Abstract"): raw_abstract = raw_abstract.strip()[8:] metadata["abstract"] = raw_abstract.strip() @@ -292,125 +298,115 @@ def save_pdf( logger.warning(f"Abstract truncated from {url}") # Save metadata to JSON - _write_metadata(metadata, output_path) + try: + with open(output_path.with_suffix(".json"), "w", encoding="utf-8") as f: + json.dump(metadata, f, ensure_ascii=False, indent=4) + except Exception as e: + logger.error(f"Failed to save metadata to {str(output_path)}: {e}") + return {"success": True, "method": used_method, "filetype": used_filetype} - if success: - return True + # If primary download failed, try fallbacks + logger.info(f"Primary download failed for {doi}. Attempting fallbacks.") - if is_biorxiv: - if ( - api_keys.get("AWS_ACCESS_KEY_ID") is None - or api_keys.get("AWS_SECRET_ACCESS_KEY") is None - ): - logger.info( - "BiorXiv PDFs can be downloaded from a S3 bucket with a requester-pay option. " - "Consider setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to use this option. " - "Pricing is a few cent per GB, thus each request costs < 0.1 cents. " - "For details see: https://www.biorxiv.org/tdm" - ) - else: - success = FALLBACKS["s3"](doi, output_path, api_keys) - if success: - return True + # Order of fallbacks tries to maximize OA coverage first + if mail and FALLBACKS["unpaywall"](doi, output_path, mail, final_url): + return {"success": True, "method": "unpaywall", "filetype": "pdf"} - # always first try fallback to BioC-PMC (open access papers on PubMed Central) - success = FALLBACKS["bioc_pmc"](doi, output_path) - - # if BioC-PMC fails, try other fallbacks - if not success: - # check for specific publishers - if "elife" in error.lower(): # elife has an open XML repository on GitHub - success = FALLBACKS["elife"](doi, output_path) - elif ( - ("wiley" in error.lower()) - and api_keys - and ("WILEY_TDM_API_TOKEN" in api_keys) - ): - success = FALLBACKS["wiley"](paper_metadata, output_path, api_keys) - if success: - return True + if FALLBACKS["europepmc"](doi, output_path): + return {"success": True, "method": "europepmc", "filetype": "xml"} + + if FALLBACKS["bioc_pmc"](doi, output_path, mail): + return {"success": True, "method": "bioc_pmc", "filetype": "xml"} if ( - soup is not None - and (meta_pdf := soup.find("meta", {"name": "citation_pdf_url"})) - and meta_pdf.get("content") + "biorxiv" in doi.lower() + and api_keys.get("AWS_ACCESS_KEY_ID") + and api_keys.get("AWS_SECRET_ACCESS_KEY") ): - pdf_url = meta_pdf.get("content") + if FALLBACKS["s3"](doi, output_path, api_keys): + return {"success": True, "method": "biorxiv_s3", "filetype": "pdf"} + + if ( + "medrxiv" in doi.lower() + and api_keys.get("AWS_ACCESS_KEY_ID") + and api_keys.get("AWS_SECRET_ACCESS_KEY") + and "medrxiv_s3" in FALLBACKS + ): + if FALLBACKS["medrxiv_s3"](doi, output_path, api_keys): + return {"success": True, "method": "medrxiv_s3", "filetype": "pdf"} + + if "plos" in doi.lower(): + if FALLBACKS["plos"](doi, output_path): + return {"success": True, "method": "plos", "filetype": "pdf"} + + if "elife" in doi.lower(): + if FALLBACKS["elife"](doi, output_path): + return {"success": True, "method": "elife", "filetype": "xml"} + + # Non-publisher OA aggregators + if "openalex" in FALLBACKS and FALLBACKS["openalex"](doi, output_path): + return {"success": True, "method": "openalex", "filetype": "pdf"} + + if "crossref" in FALLBACKS and FALLBACKS["crossref"](doi, output_path, mail or "your_email@example.com"): + return {"success": True, "method": "crossref", "filetype": "pdf"} + + if "doaj" in FALLBACKS and FALLBACKS["doaj"](doi, output_path): + return {"success": True, "method": "doaj", "filetype": "pdf"} + + if "arxiv" in FALLBACKS and FALLBACKS["arxiv"](doi, output_path): + return {"success": True, "method": "arxiv", "filetype": "pdf"} + + # Publisher TDM APIs + if api_keys: + if api_keys.get("SPRINGER_API_KEY") and FALLBACKS.get("springer"): + if FALLBACKS["springer"](paper_metadata, output_path, api_keys): + return {"success": True, "method": "springer", "filetype": "pdf"} + if api_keys.get("WILEY_TDM_API_TOKEN"): + if FALLBACKS["wiley"]( + paper_metadata, output_path, api_keys + ): + return {"success": True, "method": "wiley", "filetype": "pdf"} + if api_keys.get("ELSEVIER_TDM_API_KEY"): + if FALLBACKS["elsevier"]( + paper_metadata, output_path, api_keys, preferred_type=preferred_type + ): + return {"success": True, "method": "elsevier", "filetype": preferred_type} + + + logger.warning(f"All download attempts failed for {doi}.") + # --- Replace the previous "save abstract as .txt when all attempts failed" block with this --- + abstract_text = None + + # 1) Try Europe PMC first (prefer AbstractText) + try: + abstract_text = _get_abstract_europepmc(doi) + except Exception: + abstract_text = None + + # 2) If no abstract yet and pmid present, try PubMed Entrez + if not abstract_text and isinstance(paper_metadata, dict) and paper_metadata.get("pubmed_id"): + pmid = str(paper_metadata.get("pubmed_id")) + abstract_text = _get_abstract_pubmed(pmid) + + # 3) If still no abstract, try Crossref for the DOI + if not abstract_text: try: - pdf_headers = { - **user_agent, - "Accept": "application/pdf,application/octet-stream;q=0.9,*/*;q=0.8", - "Referer": resolved_url, - } - pdf_response = None - # Some publisher endpoints (e.g. Nature) are sensitive to cookies and - # referer headers. Try with cookies from the landing page first. - if "response" in locals(): - try: - pdf_response = requests.get( - pdf_url, - timeout=60, - headers=pdf_headers, - cookies=response.cookies, - allow_redirects=True, - ) - pdf_response.raise_for_status() - except Exception: - pdf_response = None - - if pdf_response is None: - pdf_response = requests.get( - pdf_url, - timeout=60, - headers=pdf_headers, - allow_redirects=True, - ) - pdf_response.raise_for_status() + abstract_text = _get_abstract_crossref(doi) + except Exception: + abstract_text = None - if pdf_response.content[:4] != b"%PDF": - logger.warning( - f"The file from {url} does not appear to be a valid PDF." - ) - success = FALLBACKS["bioc_pmc"](doi, output_path) - if not success: - # Check for specific publishers - if "elife" in doi.lower(): - logger.info("Attempting fallback to eLife XML repository") - success = FALLBACKS["elife"](doi, output_path) - elif api_keys and "WILEY_TDM_API_TOKEN" in api_keys: - success = FALLBACKS["wiley"]( - paper_metadata, output_path, api_keys - ) - elif api_keys and "ELSEVIER_TDM_API_KEY" in api_keys: - success = FALLBACKS["elsevier"]( - paper_metadata, output_path, api_keys - ) - if success: - return True - else: - with open(output_path.with_suffix(".pdf"), "wb+") as f: - f.write(pdf_response.content) + if not abstract_text: + logger.warning(f"Could not retrieve abstract for {doi}.") + return {"success": False, "method": None, "filetype": None} + else: + try: + with open(output_path.with_suffix(".txt"), "w", encoding="utf-8") as f: + f.write(abstract_text) + logger.info(f"Saved abstract to {str(output_path.with_suffix('.txt'))}.") except Exception as e: - logger.warning(f"Could not download {pdf_url}: {e}") - else: # if no citation_pdf_url meta tag found, try other fallbacks - if "elife" in doi.lower(): - logger.info( - "DOI contains eLife, attempting fallback to eLife XML repository on GitHub." - ) - success = FALLBACKS["elife"](doi, output_path) - if not success: - logger.warning( - f"eLife XML fallback failed for {paper_metadata['doi']}." - ) - elif ( - api_keys and "ELSEVIER_TDM_API_KEY" in api_keys - ): # elsevier journals can be accessed via the Elsevier TDM API (requires API key) - success = FALLBACKS["elsevier"](paper_metadata, output_path, api_keys) - else: - logger.warning( - f"Retrieval failed. No citation_pdf_url meta tag found for {url} and no applicable fallback mechanism available." - ) - return success + logger.error(f"Failed to save abstract to {str(output_path)}: {e}") + # Abstract saved, but not a full text + return {"success": False, "method": "abstract", "filetype": "txt"} def save_pdf_from_dump( @@ -419,7 +415,9 @@ def save_pdf_from_dump( key_to_save: str = "doi", save_metadata: bool = False, api_keys: Optional[str] = None, -) -> None: + preferred_type: str = "pdf", + mail: Optional[str] = None +) -> Dict[str, Any]: """ Receives a path to a `.jsonl` dump with paper metadata and saves the PDF files of each paper. @@ -431,6 +429,10 @@ def save_pdf_from_dump( Has to be `doi` or `title`. Defaults to `doi`. save_metadata: A boolean indicating whether to save paper metadata as a separate json. api_keys: Path to a file with API keys. If None, API-based fallbacks will be skipped. + preferred_type: Preferred file type to download, 'pdf' or 'xml'. Defaults to 'pdf'. + mail: Optional email address to use for Unpaywall API requests. + Returns: + A dict containing per-DOI results and counts. Also writes fallback_stats.json to pdf_path. """ if not isinstance(dump_path, str): @@ -443,29 +445,305 @@ def save_pdf_from_dump( if not isinstance(key_to_save, str): raise TypeError(f"key_to_save must be a string, not {type(key_to_save)}.") - if key_to_save not in ["doi", "title", "date"]: - raise ValueError("key_to_save must be one of 'doi' or 'title'.") + if preferred_type not in ["pdf", "xml"]: + raise ValueError("preferred_type must be one of 'pdf' or 'xml'.") papers = load_jsonl(dump_path) if not isinstance(api_keys, dict): api_keys = load_api_keys(api_keys) + results_by_doi: Dict[str, Dict[str, Any]] = {} + counts_by_method: Dict[str, int] = {} + pbar = tqdm(papers, total=len(papers), desc="Processing") for i, paper in enumerate(pbar): pbar.set_description(f"Processing paper {i + 1}/{len(papers)}") if "doi" not in paper.keys() or paper["doi"] is None: - logger.warning(f"Skipping {paper['title']} since no DOI available.") + logger.warning(f"Skipping paper since no DOI available.") continue filename = paper[key_to_save].replace("/", "_") pdf_file = Path(os.path.join(pdf_path, f"{filename}.pdf")) xml_file = pdf_file.with_suffix(".xml") if pdf_file.exists(): logger.info(f"File {pdf_file} already exists. Skipping download.") + results_by_doi[paper["doi"]] = {"success": True, "method": "existing", "filetype": "pdf"} + counts_by_method["existing"] = counts_by_method.get("existing", 0) + 1 continue if xml_file.exists(): logger.info(f"File {xml_file} already exists. Skipping download.") + results_by_doi[paper["doi"]] = {"success": True, "method": "existing", "filetype": "xml"} + counts_by_method["existing"] = counts_by_method.get("existing", 0) + 1 continue output_path = str(pdf_file) - save_pdf(paper, output_path, save_metadata=save_metadata, api_keys=api_keys) + result = save_pdf( + paper, + output_path, + save_metadata=save_metadata, + api_keys=api_keys, + preferred_type=preferred_type, + mail=mail + ) + doi = paper["doi"] + results_by_doi[doi] = result + if result and result.get("method"): + if result.get("success"): + counts_by_method[result["method"]] = counts_by_method.get(result["method"], 0) + 1 + else: + # track abstract-only separately + if result.get("method") == "abstract": + counts_by_method["abstract_only"] = counts_by_method.get("abstract_only", 0) + 1 + else: + counts_by_method["failed"] = counts_by_method.get("failed", 0) + 1 + + # Save stats to file in the target directory + try: + stats = { + "total": len(papers), + "counts": counts_by_method, + "by_doi": results_by_doi, + } + stats_path = Path(pdf_path) / "fallback_stats.json" + with open(stats_path, "w", encoding="utf-8") as f: + json.dump(stats, f, ensure_ascii=False, indent=2) + logger.info(f"Saved fallback stats to {stats_path}") + except Exception as e: + logger.error(f"Failed to write fallback stats: {e}") + + return {"counts": counts_by_method, "by_doi": results_by_doi} + + +# Debug variants: try all fallbacks independently of order and record which work +# New helpers (place near other helper functions in `paperscraper/pdf/pdf.py`) +def _get_redirect_domain(doi: str, timeout: int = 10) -> Optional[str]: + """ + Resolve https://doi.org/{doi} and return the extracted domain (e.g. 'wiley') or None on failure. + """ + try: + resp = requests.get(f"https://doi.org/{doi}", timeout=timeout, allow_redirects=True) + resp.raise_for_status() + return tldextract.extract(resp.url).domain or None + except Exception: + return None + + +def _crossref_publisher_is_wiley(doi: str, timeout: int = 10) -> bool: + """ + Query Crossref works API and return True if publisher name contains 'wiley' (case-insensitive). + """ + try: + url = f"https://api.crossref.org/works/{doi}" + resp = requests.get(url, timeout=timeout) + resp.raise_for_status() + publisher = resp.json().get("message", {}).get("publisher", "") or "" + return "wiley" in publisher.lower() + except Exception: + return False + + +def _wiley_allowed(doi: str, final_url: Optional[str] = None, timeout: int = 10) -> bool: + """ + Return True if it's reasonable to attempt the Wiley TDM fallback: + - either the DOI redirect domain contains 'wiley', or + - the Crossref publisher is Wiley. + """ + # 1) check provided final_url if available + try: + if final_url: + domain = tldextract.extract(final_url).domain or "" + if "wiley" in domain.lower(): + return True + except Exception: + pass + + # 2) try resolving DOI redirect domain + try: + redirect_domain = _get_redirect_domain(doi, timeout=timeout) + if redirect_domain and "wiley" in redirect_domain.lower(): + return True + except Exception: + pass + + # 3) fallback to Crossref publisher check + try: + if _crossref_publisher_is_wiley(doi, timeout=timeout): + return True + except Exception: + pass + + return False + +def debug_save_pdf( + paper_metadata: Dict[str, Any], + filepath: Union[str, Path], + api_keys: Optional[Union[str, Dict[str, str]]] = None, + preferred_type: str = "pdf", + mail: Optional[str] = None, + save_first_only: bool = True, +) -> Dict[str, Any]: + """ + Debug version that attempts the direct method and all fallbacks independently. + Writes results per-fallback to distinct files with ".{fallback}" suffix to avoid clobbering. + + Returns a dict with keys: direct, successes (list), results (per-fallback bool), first_saved (fallback name or None). + """ + if not isinstance(api_keys, dict): + api_keys = load_api_keys(api_keys) + + doi = paper_metadata["doi"] + base_output = Path(filepath) + successes = [] + per = {} + + # Use a unique path for the initial direct check so save_pdf doesn't + # already save a fallback to the main output and interfere with later attempts. + direct_check_path = Path(str(base_output) + ".direct_check") + direct_res = save_pdf( + paper_metadata, + direct_check_path, + save_metadata=False, + api_keys=api_keys, + preferred_type=preferred_type, + mail=mail, + ) + + # Only treat "direct" as successful if the returned method is actually "direct" + is_direct = bool(direct_res.get("success") and direct_res.get("method") == "direct") + if is_direct: + successes.append("direct") + per["direct"] = is_direct + first_saved = "direct" if per["direct"] else None + + # Build a deterministic list of fallbacks + order = [ + "unpaywall", + "europepmc", + "bioc_pmc", + "plos", + "elife", + "openalex", + "crossref", + "doaj", + "arxiv", + "springer", + "wiley", + "elsevier", + ] + + def _attempt(name: str) -> bool: + # derive unique output stem for debug + out = Path(str(base_output) + f".{name}") + try: + if name == "unpaywall" and mail: + return FALLBACKS[name](doi, out, mail, None) + if name in ("europepmc", "doaj", "openalex", "arxiv"): + return FALLBACKS[name](doi, out) + if name == "crossref": + return FALLBACKS[name](doi, out, mail or "your_email@example.com") + if name in ("s3", "medrxiv_s3"): + if api_keys.get("AWS_ACCESS_KEY_ID") and api_keys.get("AWS_SECRET_ACCESS_KEY"): + return FALLBACKS[name](doi, out, api_keys) + return False + if name in ("plos", "elife"): + return FALLBACKS[name](doi, out) + if name in ("wiley", "springer"): + if name == "wiley": + if not api_keys.get("WILEY_TDM_API_TOKEN"): + return False + # Only try Wiley when applicable + try: + if not _wiley_allowed(doi): + return False + except Exception: + return False + if name == "springer" and not api_keys.get("SPRINGER_API_KEY"): + return False + return FALLBACKS[name](paper_metadata, out, api_keys) + if name == "elsevier": + if not api_keys.get("ELSEVIER_TDM_API_KEY"): + return False + return FALLBACKS[name](paper_metadata, out, api_keys, preferred_type=preferred_type) + except Exception: + return False + return False + + for fb in order: + if fb not in FALLBACKS: + per[fb] = False + continue + ok = _attempt(fb) + per[fb] = ok + if ok: + successes.append(fb) + if not first_saved: + first_saved = fb + if save_first_only: + # stop after the first saved to limit writes + break + + return {"direct": per.get("direct", False), "results": per, "successes": successes, "first_saved": first_saved} +# python +def debug_save_pdf_from_dump( + dump_path: str, + pdf_path: str, + api_keys: Optional[str] = None, + preferred_type: str = "pdf", + mail: Optional[str] = None, + save_first_only: bool = True, + save_interval: int = 10, +) -> Dict[str, Any]: + """ + Debug variant for batch processing that tests all fallbacks per paper and records which work. + Writes a debug_fallback_stats.json with detailed per-DOI outcomes. + Saves intermediate stats every `save_interval` papers so partial results are available. + """ + papers = load_jsonl(dump_path) + if not isinstance(api_keys, dict): + api_keys = load_api_keys(api_keys) + + by_doi = {} + counts: Dict[str, int] = {} + + pbar = tqdm(papers, total=len(papers), desc="Debug processing") + def _write_debug_stats(target_dir: str, by_doi_obj: Dict[str, Any], counts_obj: Dict[str, int]): + try: + stats_path = Path(target_dir) / "debug_fallback_stats.json" + tmp_path = stats_path.with_suffix(".tmp") + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump({"by_doi": by_doi_obj, "counts": counts_obj}, f, ensure_ascii=False, indent=2) + tmp_path.replace(stats_path) + logger.info(f"Saved debug fallback stats to {stats_path}") + except Exception as e: + logger.error(f"Failed to write debug fallback stats: {e}") + + for i, paper in enumerate(pbar): + if "doi" not in paper or not paper["doi"]: + continue + filename = paper["doi"].replace("/", "_") + out = str(Path(os.path.join(pdf_path, f"{filename}.pdf"))) + res = debug_save_pdf( + paper, + out, + api_keys=api_keys, + preferred_type=preferred_type, + mail=mail, + save_first_only=save_first_only, + ) + by_doi[paper["doi"]] = res + # count successes per fallback + for fb, ok in res.get("results", {}).items(): + if ok: + counts[fb] = counts.get(fb, 0) + 1 + + # periodically save partial stats so you can inspect mid-run + if save_interval > 0 and ((i + 1) % save_interval == 0): + _write_debug_stats(pdf_path, by_doi, counts) + + # write final debug stats + try: + _write_debug_stats(pdf_path, by_doi, counts) + except Exception as e: + logger.error(f"Failed to write final debug fallback stats: {e}") + return {"by_doi": by_doi, "counts": counts} +