Update plots

.github/workflows/docs.yml

@@ -30,6 +30,11 @@ jobs:
         run: |
           uv sync --group dev
 
+      - name: Copy README assets for MkDocs
+        run: |
+          mkdir -p docs/assets
+          cp assets/ai_quantum_*.png docs/assets/
+
       - name: Build docs
         run: uv run mkdocs build
 

README.md

@@ -103,26 +103,30 @@ get_and_dump_arxiv_papers(..., backend='local')
 ### Publication keyword search
 
 Consider you want to perform a publication keyword search with the query:
-`COVID-19` **AND** `Artificial Intelligence` **AND** `Medical Imaging`. 
+`Artificial Intelligence` **AND** `Quantum Computing` **AND** `Chemistry`.
 
 * Scrape papers from PubMed:
 
 ```py
 from paperscraper.pubmed import get_and_dump_pubmed_papers
-covid19 = ['COVID-19', 'SARS-CoV-2']
-ai = ['Artificial intelligence', 'Deep learning', 'Machine learning']
-mi = ['Medical imaging']
-query = [covid19, ai, mi]
+ai = ['Artificial intelligence', 'Machine learning']
+qc = [
+    'Quantum computing', 'Quantum computer', 'Quantum information',
+    'Quantum algorithm', 'Quantum circuit', 'Quantum simulation',
+    'Quantum machine learning', 'Qubit', 'Quantum annealing'
+]
+chemistry = ['Chemistry', 'Chemical', 'Molecule', 'Molecular', 'Materials science']
+query = [ai, qc, chemistry]
 
-get_and_dump_pubmed_papers(query, output_filepath='covid19_ai_imaging.jsonl')
+get_and_dump_pubmed_papers(query, output_filepath='ai_quantum_chemistry.jsonl')
 ```
 
 * Scrape papers from arXiv:
 
 ```py
 from paperscraper.arxiv import get_and_dump_arxiv_papers
 
-get_and_dump_arxiv_papers(query, output_filepath='covid19_ai_imaging.jsonl')
+get_and_dump_arxiv_papers(query, output_filepath='ai_quantum_chemistry.jsonl')
 ```
 
 * Scrape papers from bioRiv, medRxiv or chemRxiv:
@@ -131,15 +135,20 @@ get_and_dump_arxiv_papers(query, output_filepath='covid19_ai_imaging.jsonl')
 from paperscraper.xrxiv.xrxiv_query import XRXivQuery
 
 querier = XRXivQuery('server_dumps/chemrxiv_2020-11-10.jsonl')
-querier.search_keywords(query, output_filepath='covid19_ai_imaging.jsonl')
+querier.search_keywords(query, output_filepath='ai_quantum_chemistry.jsonl')
 ```
 
 You can also use `dump_queries` to iterate over a bunch of queries for all available databases.
 
 ```py
 from paperscraper import dump_queries
 
-queries = [[covid19, ai, mi], [covid19, ai], [ai]]
+physics = [
+    'Physics', 'Physical', 'Particle', 'Condensed matter',
+    'Many-body', 'Fermion', 'Hamiltonian', 'Spin'
+]
+biology = ['Biology', 'Biological', 'Cellular', 'Genomics', 'Gene', 'Protein']
+queries = [[ai, qc, chemistry], [ai, qc, physics], [ai, qc, biology]]
 dump_queries(queries, '.')
 ```
 
@@ -148,8 +157,8 @@ Or use the harmonized interface of `QUERY_FN_DICT` to query multiple databases o
 from paperscraper.load_dumps import QUERY_FN_DICT
 print(QUERY_FN_DICT.keys())
 
-QUERY_FN_DICT['biorxiv'](query, output_filepath='biorxiv_covid_ai_imaging.jsonl')
-QUERY_FN_DICT['medrxiv'](query, output_filepath='medrxiv_covid_ai_imaging.jsonl')
+QUERY_FN_DICT['biorxiv'](query, output_filepath='biorxiv_ai_quantum_chemistry.jsonl')
+QUERY_FN_DICT['chemrxiv'](query, output_filepath='chemrxiv_ai_quantum_chemistry.jsonl')
 ```
 
 * Scrape papers from Google Scholar:
@@ -185,7 +194,7 @@ To batch download full texts from your metadata search results:
 from paperscraper.pdf import save_pdf_from_dump
 
 # Save PDFs/XMLs in current folder and name the files by their DOI
-save_pdf_from_dump('medrxiv_covid_ai_imaging.jsonl', pdf_path='.', key_to_save='doi')
+save_pdf_from_dump('ai_quantum_chemistry.jsonl', pdf_path='.', key_to_save='doi')
 ```
 
 #### Automatic Fallback Mechanisms
@@ -304,19 +313,34 @@ automatically: Venn diagrams and bar plots.
 Compare the temporal evolution of different queries across different servers.
 
 ```py
+import os
+
 from paperscraper import QUERY_FN_DICT
 from paperscraper.postprocessing import aggregate_paper
 from paperscraper.utils import get_filename_from_query, load_jsonl
 
 # Define search terms and their synonyms
-ml = ['Deep learning', 'Neural Network', 'Machine learning']
-mol = ['molecule', 'molecular', 'drug', 'ligand', 'compound']
-gnn = ['gcn', 'gnn', 'graph neural', 'graph convolutional', 'molecular graph']
-smiles = ['SMILES', 'Simplified molecular']
-fp = ['fingerprint', 'molecular fingerprint', 'fingerprints']
+ai = ['Artificial intelligence', 'Machine learning']
+qc = [
+    'Quantum computing', 'Quantum computer', 'Quantum information',
+    'Quantum algorithm', 'Quantum circuit', 'Quantum simulation',
+    'Quantum machine learning', 'Qubit', 'Quantum annealing'
+]
+chemistry = ['Chemistry', 'Chemical', 'Molecule', 'Molecular', 'Materials science']
+physics = [
+    'Physics', 'Physical', 'Particle', 'Condensed matter',
+    'Many-body', 'Fermion', 'Hamiltonian', 'Spin'
+]
+biology = ['Biology', 'Biological', 'Cellular', 'Genomics', 'Gene', 'Protein']
+medicine = ['Medicine', 'Medical', 'Clinical', 'Disease', 'Patient', 'Health']
 
 # Define queries
-queries = [[ml, mol, smiles], [ml, mol, fp], [ml, mol, gnn]]
+queries = [
+    [ai, qc, chemistry],
+    [ai, qc, physics],
+    [ai, qc, biology],
+    [ai, qc, medicine],
+]
 
 root = '../keyword_dumps'
 
@@ -328,73 +352,80 @@ for query in queries:
         # Assuming the keyword search has been performed already
         data = load_jsonl(os.path.join(root, db, filename))
 
-        # Unstructured matches are aggregated into 6 bins, 1 per year
-        # from 2015 to 2020. Sanity check is performed by having 
+        # Unstructured matches are aggregated into 8 bins, 1 per year
+        # from 2019 to 2026. Sanity check is performed by having
         # `filtering=True`, removing papers that don't contain all of
         # the keywords in query.
         data_dict[filename][db], filtered = aggregate_paper(
-            data, 2015, bins_per_year=1, filtering=True,
-            filter_keys=query, return_filtered=True
+            data, 2019, bins_per_year=1, filtering=True,
+            filter_keys=query, return_filtered=True, last_year=2026
         )
 
 # Plotting is now very simple
 from paperscraper.plotting import plot_comparison
 
 data_keys = [
-    'deeplearning_molecule_fingerprint.jsonl',
-    'deeplearning_molecule_smiles.jsonl', 
-    'deeplearning_molecule_gcn.jsonl'
+    'artificialintelligence_quantumcomputing_chemistry.jsonl',
+    'artificialintelligence_quantumcomputing_physics.jsonl',
+    'artificialintelligence_quantumcomputing_biology.jsonl',
+    'artificialintelligence_quantumcomputing_medicine.jsonl'
 ]
 plot_comparison(
     data_dict,
     data_keys,
-    title_text="'Deep Learning' AND 'Molecule' AND X",
-    keyword_text=['Fingerprint', 'SMILES', 'Graph'],
-    figpath='mol_representation'
+    x_ticks=[str(year) for year in range(2019, 2027)],
+    title_text="'Artificial intelligence' AND 'Quantum computing' AND X",
+    keyword_text=['Chemistry', 'Physics', 'Biology', 'Medicine'],
+    figpath='assets/ai_quantum_fields.png'
 )
 ```
 
-![molreps](https://github.com/jannisborn/paperscraper/blob/main/assets/molreps.png?raw=true "MolReps")
+![Artificial intelligence and quantum computing by field](assets/ai_quantum_fields.png)
 
 
 ### Venn Diagrams
 
+The Venn diagrams below use the local arXiv, bioRxiv, ChemRxiv and medRxiv dumps.
+
 ```py
 from paperscraper.plotting import (
     plot_venn_two, plot_venn_three, plot_multiple_venn
 )
 
-sizes_2020 = (30842, 14474, 2292, 35476, 1904, 1408, 376)
-sizes_2019 = (55402, 11899, 2563)
-labels_2020 = ('Medical\nImaging', 'Artificial\nIntelligence', 'COVID-19')
-labels_2019 = ['Medical Imaging', 'Artificial\nIntelligence']
+sizes_2024 = (18762, 6324, 624)
+sizes_2025 = (18774, 6966, 775, 28652, 1965, 597, 77)
+labels_2024 = ['Artificial\nIntelligence', 'Quantum\nComputing']
+labels_2025 = ('Artificial\nIntelligence', 'Quantum\nComputing', 'Chemistry')
 
-plot_venn_two(sizes_2019, labels_2019, title='2019', figpath='ai_imaging.png')
+plot_venn_two(
+    sizes_2024, labels_2024, title='2024', figpath='assets/ai_quantum_venn_2024.png'
+)
 ```
 
-![2019](https://github.com/jannisborn/paperscraper/blob/main/assets/ai_imaging.png?raw=true "2019")
+![2024 Venn diagram](assets/ai_quantum_venn_2024.png)
 
 
 ```py
 plot_venn_three(
-    sizes_2020, labels_2020, title='2020', figpath='ai_imaging_covid.png'
+    sizes_2025, labels_2025, title='2025',
+    figpath='assets/ai_quantum_chemistry_venn_2025.png'
 )
 ```
 
-![2020](https://github.com/jannisborn/paperscraper/blob/main/assets/ai_imaging_covid.png?raw=true "2020")
+![2025 Venn diagram](assets/ai_quantum_chemistry_venn_2025.png)
 
 Or plot both together:
 
 ```py
 plot_multiple_venn(
-    [sizes_2019, sizes_2020], [labels_2019, labels_2020], 
-    titles=['2019', '2020'], suptitle='Keyword search comparison', 
+    [sizes_2024, sizes_2025], [labels_2024, labels_2025],
+    titles=['2024', '2025'], suptitle='Keyword search comparison',
     gridspec_kw={'width_ratios': [1, 2]}, figsize=(10, 6),
-    figpath='both.png'
+    figpath='assets/ai_quantum_venn_both.png'
 )
 ```
 
-![both](https://github.com/jannisborn/paperscraper/blob/main/assets/both.png?raw=true "Both")
+![Venn diagram comparison](assets/ai_quantum_venn_both.png)
 
 
 

codecov.yml

@@ -3,7 +3,7 @@ coverage:
     patch: off
     project:
       default:
-        target: 80%
+        target: 70%
         threshold: 2% # Up to 2% drop/fluctuation is OK
 
   ignore:

paperscraper/plotting.py

@@ -74,11 +74,13 @@ def plot_comparison(
         preprint.append(arxiv[-1] + biorxiv[-1] + medrxiv[-1] + chemrxiv[-1])
 
     ind = np.arange(len(arxiv[0]))  # the x locations for the groups
-    width = [0.2] * len(ind)  # the width of the bars: can also be len(x) sequence
-    if len(keys) == 2:
-        pos = [-0.2, 0.2]
-    elif len(keys) == 3:
-        pos = [-0.3, 0.0, 0.3]
+    bar_width = min(0.8 / len(keys), 0.2)
+    width = [bar_width] * len(ind)
+    pos = np.linspace(
+        -bar_width * (len(keys) - 1) / 2,
+        bar_width * (len(keys) - 1) / 2,
+        len(keys),
+    )
 
     plts = []
     legend_plts = []
@@ -88,7 +90,7 @@ def plot_comparison(
         legend_platform = ["PubMed", "Preprint"]
     else:
         bars = [pubmed, arxiv, biorxiv, chemrxiv, medrxiv]
-        legend_platform = ["PubMed", "ArXiv", "BiorXiv", "ChemRxiv", "MedRxiv"]
+        legend_platform = ["PubMed", "ArXiv", "bioRxiv", "ChemRxiv", "medRxiv"]
     for idx in range(len(keys)):
         bottom = 0
 
@@ -252,7 +254,7 @@ def plot_single(
 
     else:
         bars = [pubmed, arxiv, biorxiv, chemrxiv, medrxiv]
-        legend_platform = ["PubMed", "ArXiv", "BiorXiv", "ChemRxiv", "MedRxiv"]
+        legend_platform = ["PubMed", "ArXiv", "bioRxiv", "ChemRxiv", "medRxiv"]
         if logscale:
             sums = (
                 np.array(pubmed)
@@ -370,8 +372,6 @@ def plot_venn_two(
         subsets=sizes, linestyle="solid", linewidth=0.6, color="grey", **kwargs
     )
     if kwargs.get("ax", False):
-        print(kwargs, type(kwargs))
-        print(kwargs["ax"])
         kwargs["ax"].set_title(title, fontdict={"fontweight": "bold"}, size=15)
     else:
         plt.title(title, fontdict={"fontweight": "bold"}, size=15)

paperscraper/postprocessing.py

@@ -71,20 +71,18 @@ def aggregate_paper(
         year = int(date.split("-")[0])
         if year < start_year or year > last_year:
             continue
+        title = str(paper.get("title") or "").lower()
+        abstract = str(paper.get("abstract") or "").lower()
 
         # At least one synonym per keyword needs to be in either title or
         # abstract.
         if filtering and filter_keys != list():
             # Filter out papers which undesired terms
             unwanted = False
             for unwanted_key in unwanted_keys:
-                if unwanted_key.lower() in paper["title"].lower():
+                if unwanted_key.lower() in title:
                     unwanted = True
-                if (
-                    filter_abstract
-                    and paper["abstract"] is not None
-                    and unwanted_key.lower() in paper["abstract"].lower()
-                ):
+                if filter_abstract and unwanted_key.lower() in abstract:
                     unwanted = True
             if unwanted:
                 continue
@@ -95,13 +93,9 @@ def aggregate_paper(
                 if not isinstance(key_term, list):
                     key_term = [key_term]
                 for key in key_term:
-                    if key.lower() in paper["title"].lower():
+                    if key.lower() in title:
                         got_key = True
-                    if (
-                        filter_abstract
-                        and paper["abstract"] is not None
-                        and key.lower() in paper["abstract"].lower()
-                    ):
+                    if filter_abstract and key.lower() in abstract:
                         got_key = True
                 got_keys.append(got_key)
 

paperscraper/utils.py

@@ -72,5 +72,5 @@ def load_jsonl(filepath: str) -> List[Dict[str, str]]:
     """
 
     with open(filepath, "r") as f:
-        data = [json.loads(line) for line in f.readlines()]
+        data = [json.loads(line) for line in f if line.strip()]
     return data

pyproject.toml

@@ -90,6 +90,7 @@ dev = [
   "mkdocstrings[python]>=0.24",
   "mkdocs-material-extensions",
   "pre-commit>=4.3.0",
+  "grip>=4.6.2",
 ]
 
 [tool.ruff]