Speed up arxiv

README.md

@@ -58,8 +58,8 @@ However, to scrape publication data from the preprint servers [biorxiv](https://
 ```py
 from paperscraper.get_dumps import biorxiv, medrxiv, chemrxiv
 chemrxiv()  #  Takes <15min -> +50K papers (~30 MB file)
-medrxiv()  #  Takes <30min -> +100K papers (~200 MB file)
-biorxiv()  # Takes <3h -> +450 papers (~800 MB file)
+medrxiv()  #  Takes <5min -> +100K papers (~200 MB file)
+biorxiv()  # Takes <1h -> +450K papers (~800 MB file)
 ```
 *NOTE*: Once the dumps are stored, please make sure to restart the python interpreter so that the changes take effect. 
 *NOTE*: If you experience API connection issues, retries and request behavior can be tuned, e.g.:
@@ -81,11 +81,21 @@ medrxiv(start_date="2023-04-01", end_date="2023-04-08")
 But watch out. The resulting `.jsonl` file will be labelled according to the current date and all your subsequent searches will be based on this file **only**. If you use this option you might want to keep an eye on the source files (`paperscraper/server_dumps/*jsonl`) to ensure they contain the paper metadata for all papers you're interested in.
 
 #### Arxiv local dump
-If you prefer local search rather than using the arxiv API:
+Local search can be faster than using the arxiv API especially if you plan many queries. Paperscraper provides two backends to bulk-download arxiv, `kaggle` and `arxiv`. The default is `kaggle` since it is much faster. Before using it, authenticate with your Kaggle account:
+
+```sh
+kaggle auth login
+```
+
+```py
+from paperscraper.get_dumps import arxiv
+arxiv(start_date='2019-01-01', end_date='2026-12-31')
+```
+NOTE: The disadvantage of `kaggle` backend is that it bulk-downloads **all** of arXiv. For small API-backed dumps, better use the `arxiv` PyPI package backend:
 
 ```py
 from paperscraper.get_dumps import arxiv
-arxiv(start_date='2024-01-01', end_date=None) # scrapes all metadata from 2024 until today.
+arxiv(start_date='2024-01-01',end_date='2024-01-04',backend='api')
 ```
 
 Afterwards you can search the local arxiv dump just like the other x-rxiv dumps.

paperscraper/arxiv/arxiv.py

@@ -25,10 +25,14 @@ def search_local_arxiv():
     global ARXIV_QUERIER
     if ARXIV_QUERIER is not None:
         return
-    dump_paths = glob.glob(os.path.join(dump_root, "arxiv*"))
+    dump_paths = [
+        path
+        for path in glob.glob(os.path.join(dump_root, "arxiv*.jsonl"))
+        if os.path.isfile(path)
+    ]
 
     if len(dump_paths) > 0:
-        path = sorted(dump_paths, reverse=True)[0]
+        path = max(dump_paths, key=os.path.getmtime)
         querier = XRXivQuery(path)
         if not querier.errored:
             ARXIV_QUERIER = querier.search_keywords

paperscraper/arxiv/kaggle.py

@@ -0,0 +1,208 @@
+"""Kaggle-backed arXiv metadata dumping utilities."""
+
+import glob
+import json
+import os
+import shutil
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from typing import Optional
+
+from tqdm import tqdm
+
+from ..utils import get_server_dumps_dir
+
+DEFAULT_KAGGLE_DATASET = "Cornell-University/arxiv"
+
+
+def arxiv_kaggle(
+    start_date: datetime,
+    end_date: datetime,
+    save_path: str,
+    kaggle_filepath: Optional[str] = None,
+) -> int:
+    """Convert a Kaggle arXiv metadata snapshot to paperscraper JSONL format.
+
+    Args:
+        start_date: Earliest paper submission date to include.
+        end_date: Latest paper submission date to include.
+        save_path: Destination JSONL path for converted papers.
+        kaggle_filepath: Existing Kaggle snapshot file. If provided, no Kaggle
+            download is attempted.
+
+    Returns:
+        Number of papers written to `save_path`.
+    """
+    cleanup_dir = default_kaggle_dir() if kaggle_filepath is None else None
+    if kaggle_filepath is None:
+        kaggle_filepath = download_kaggle_snapshot()
+
+    try:
+        written = 0
+        os.makedirs(os.path.dirname(os.path.abspath(save_path)), exist_ok=True)
+        total_size = os.path.getsize(kaggle_filepath)
+        with (
+            open(kaggle_filepath, "r", encoding="utf-8") as in_fp,
+            open(save_path, "w", encoding="utf-8") as out_fp,
+            tqdm(
+                total=total_size,
+                desc="Converting arXiv Kaggle snapshot",
+                unit="B",
+                unit_scale=True,
+            ) as progress_bar,
+        ):
+            for line in in_fp:
+                progress_bar.update(len(line.encode("utf-8")))
+                if not line.strip():
+                    continue
+
+                record = json.loads(line)
+                paper_date = get_kaggle_paper_date(record)
+                if paper_date is None or not start_date <= paper_date <= end_date:
+                    continue
+
+                if written > 0:
+                    out_fp.write(os.linesep)
+                out_fp.write(json.dumps(normalize_kaggle_record(record, paper_date)))
+                written += 1
+        return written
+    finally:
+        if cleanup_dir is not None:
+            shutil.rmtree(cleanup_dir, ignore_errors=True)
+
+
+def download_kaggle_snapshot() -> str:
+    """Download the Kaggle arXiv metadata snapshot if needed.
+
+    Returns:
+        Path to the local Kaggle snapshot JSON file.
+
+    Raises:
+        ImportError: If the `kaggle` package is not installed.
+        RuntimeError: If Kaggle authentication is missing or invalid.
+        FileNotFoundError: If the download succeeds but no snapshot JSON is found.
+    """
+    kaggle_dir = default_kaggle_dir()
+    os.makedirs(kaggle_dir, exist_ok=True)
+
+    if existing_snapshot := find_kaggle_snapshot(kaggle_dir):
+        return existing_snapshot
+
+    try:
+        from kaggle.api.kaggle_api_extended import KaggleApi
+    except ImportError as exc:
+        raise ImportError(
+            "The Kaggle backend requires the `kaggle` package. Install it with "
+            "`pip install kaggle` or `uv add kaggle`."
+        ) from exc
+    except SystemExit as exc:
+        raise RuntimeError(
+            "Kaggle authentication is required for the arXiv Kaggle backend. "
+            "Run `kaggle auth login` or configure Kaggle credentials."
+        ) from exc
+
+    api = KaggleApi()
+    try:
+        api.authenticate()
+    except Exception as exc:
+        raise RuntimeError(
+            "Kaggle authentication is required for the arXiv Kaggle backend. "
+            "Run `kaggle auth login` or configure Kaggle credentials."
+        ) from exc
+    api.dataset_download_files(
+        DEFAULT_KAGGLE_DATASET,
+        path=kaggle_dir,
+        unzip=True,
+        quiet=False,
+    )
+
+    snapshot = find_kaggle_snapshot(kaggle_dir)
+    if snapshot is None:
+        raise FileNotFoundError(f"No arXiv Kaggle snapshot found in {kaggle_dir}")
+    return snapshot
+
+
+def default_kaggle_dir() -> str:
+    """Return the default temporary directory for Kaggle arXiv downloads.
+
+    Returns:
+        Path to the default Kaggle download directory.
+    """
+    return os.path.join(get_server_dumps_dir(), "arxiv_kaggle")
+
+
+def find_kaggle_snapshot(kaggle_dir: str) -> Optional[str]:
+    """Find the arXiv metadata snapshot JSON in a Kaggle download directory.
+
+    Args:
+        kaggle_dir: Directory to search.
+
+    Returns:
+        Path to the largest candidate JSON file, or None if no candidate exists.
+    """
+    candidates = [
+        *glob.glob(os.path.join(kaggle_dir, "arxiv-metadata*.json")),
+        *glob.glob(os.path.join(kaggle_dir, "*.json")),
+    ]
+    candidates = [path for path in candidates if os.path.isfile(path)]
+    if not candidates:
+        return None
+    return sorted(candidates, key=os.path.getsize, reverse=True)[0]
+
+
+def get_kaggle_paper_date(record: dict) -> Optional[datetime]:
+    """Extract the first submission date from a Kaggle arXiv record.
+
+    Args:
+        record: Raw Kaggle arXiv metadata record.
+
+    Returns:
+        Naive UTC-normalized submission date at midnight, or None if no usable
+        date is available.
+    """
+    created = next(
+        (version.get("created") for version in record.get("versions", []) if version),
+        None,
+    )
+    if created:
+        try:
+            date = parsedate_to_datetime(created)
+            if date.tzinfo is not None:
+                date = date.astimezone(timezone.utc).replace(tzinfo=None)
+            return date.replace(hour=0, minute=0, second=0, microsecond=0)
+        except (TypeError, ValueError):
+            pass
+
+    update_date = record.get("update_date")
+    if update_date:
+        try:
+            return datetime.strptime(update_date, "%Y-%m-%d")
+        except ValueError:
+            return None
+    return None
+
+
+def normalize_kaggle_record(record: dict, paper_date: datetime) -> dict:
+    """Normalize a Kaggle arXiv record to paperscraper dump fields.
+
+    Args:
+        record: Raw Kaggle arXiv metadata record.
+        paper_date: Submission date returned by `get_kaggle_paper_date`.
+
+    Returns:
+        Dictionary with paperscraper's standard `title`, `authors`, `date`,
+        `abstract`, `journal`, and `doi` fields.
+    """
+    arxiv_id = str(record.get("id", "")).split("v")[0]
+    return {
+        "title": normalize_whitespace(record.get("title", "")),
+        "authors": normalize_whitespace(record.get("authors", "")),
+        "date": paper_date.strftime("%Y-%m-%d"),
+        "abstract": normalize_whitespace(record.get("abstract", "")),
+        "journal": normalize_whitespace(record.get("journal-ref", "")),
+        "doi": record.get("doi") or f"10.48550/arXiv.{arxiv_id}",
+    }
+
+
+def normalize_whitespace(value: Optional[str]) -> str:
+    return " ".join(str(value or "").split())

paperscraper/get_dumps/arxiv.py

@@ -3,11 +3,12 @@
 import json
 import os
 from datetime import datetime, timedelta
-from typing import Optional
+from typing import Literal, Optional
 
 from tqdm import tqdm
 
 from ..arxiv import get_arxiv_papers_api
+from ..arxiv.kaggle import arxiv_kaggle
 from ..utils import get_server_dumps_dir
 
 today = datetime.today().strftime("%Y-%m-%d")
@@ -19,6 +20,10 @@ def arxiv(
     start_date: Optional[str] = None,
     end_date: Optional[str] = None,
     save_path: str = save_path,
+    backend: Literal["api", "kaggle"] = "kaggle",
+    page_size: int = 2000,
+    delay_seconds: float = 3.0,
+    num_retries: int = 3,
 ):
     """
     Fetches papers from arXiv based on time range, i.e., start_date and end_date.
@@ -29,7 +34,17 @@ def arxiv(
         start_date (str, optional): Start date in format YYYY-MM-DD. Defaults to None.
         end_date (str, optional): End date in format YYYY-MM-DD. Defaults to None.
         save_path (str, optional): Path to save the JSONL dump. Defaults to save_path.
+        backend: Metadata source. If `api`, use the arxiv package/API. If `kaggle`,
+            use the Kaggle arXiv metadata snapshot. Defaults to `kaggle`.
+        page_size (int, optional): Number of records requested per API page.
+            arXiv allows at most 2000. Defaults to 2000.
+        delay_seconds (float, optional): Delay between API requests. arXiv asks for
+            at least 3 seconds. Defaults to 3.0.
+        num_retries (int, optional): Number of retries per API page. Defaults to 3.
     """
+    if backend not in {"api", "kaggle"}:
+        raise ValueError("backend must be one of ['api', 'kaggle']")
+
     # Set default dates
     EARLIEST_START = "1991-01-01"
     if start_date is None:
@@ -46,6 +61,13 @@ def arxiv(
             f"start_date {start_date} cannot be later than end_date {end_date}"
         )
 
+    if backend == "kaggle":
+        return arxiv_kaggle(
+            start_date=start_date,
+            end_date=end_date,
+            save_path=save_path,
+        )
+
     # Open file for writing results
     with open(save_path, "w") as fp:
         progress_bar = tqdm(total=(end_date - start_date).days + 1)
@@ -63,6 +85,11 @@ def arxiv(
                 papers = get_arxiv_papers_api(
                     query=query,
                     fields=["title", "authors", "date", "abstract", "journal", "doi"],
+                    client_options={
+                        "page_size": page_size,
+                        "delay_seconds": delay_seconds,
+                        "num_retries": num_retries,
+                    },
                     verbose=False,
                 )
                 if not papers.empty: