Merge branch 'main' into internetarchive

Author: TimidRobot

.github/workflows/test_scripts_help.yml

@@ -0,0 +1,34 @@
+name: Test scripts' help
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  job:
+    runs-on: ubuntu-latest
+
+    steps:
+
+    # https://github.com/actions/setup-python
+    - name: Install Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Install pipenv
+      run: |
+        pip install --upgrade pip
+        pip install pipenv
+
+    # https://github.com/actions/checkout
+    - name: Checkout quantifying
+      uses: actions/checkout@v4
+
+    - name: Install Python dependencies
+      run: |
+        pipenv sync --system
+
+    - name: Test scripts' help
+      run: |
+        ./dev/test_scripts_help.sh

dev/test_scripts_help.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+# Ensure each script can display help message to ensure basic execution.
+#
+# This script must be run from within the pipenv shell or a properly configured
+# environment. For example:
+#
+#   1. Using pipenv run
+#       pipenv run ./dev/test_scripts_help.sh
+#
+#   2. Using pipenv shell
+#       pipenv shell
+#       ./dev/test_scripts_help.sh
+#
+#   3. A properly configured environment
+#      (see .github/workflows/test_scripts_help.yml)
+#
+#### SETUP ####################################################################
+
+set -o errexit
+set -o errtrace
+set -o nounset
+
+# shellcheck disable=SC2154
+trap '_es=${?};
+    printf "${0}: line ${LINENO}: \"${BASH_COMMAND}\"";
+    printf " exited with a status of ${_es}\n";
+    exit ${_es}' ERR
+
+DIR_REPO="$(cd -P -- "${0%/*}/.." && pwd -P)"
+EXIT_STATUS=0
+
+#### FUNCTIONS ################################################################
+
+test_help() {
+    local _es _script
+    for _script in $(find scripts/?-* -type f -name '*.py' | sort)
+    do
+        _es=0
+        ./"${_script}" --help &>/dev/null || _es=${?}
+        if (( _es == 0 ))
+        then
+            echo "✅ ${_script}"
+        else
+            echo "❌ ${_script}"
+            EXIT_STATUS=${_es}
+        fi
+    done
+}
+
+#### MAIN #####################################################################
+
+cd "${DIR_REPO}"
+test_help
+echo "exit status: ${EXIT_STATUS}"
+exit ${EXIT_STATUS}

scripts/1-fetch/arxiv_fetch.py

@@ -34,7 +34,7 @@
 
 # Constants
 # API Configuration
-BASE_URL = "http://export.arxiv.org/api/query?"
+BASE_URL = "https://export.arxiv.org/api/query?"
 DEFAULT_FETCH_LIMIT = 800  # Default total papers to fetch
 
 # CSV Headers

scripts/shared.py

@@ -2,13 +2,15 @@
 import logging
 import os
 import sys
+from collections import OrderedDict
 from datetime import datetime, timezone
 
 # Third-party
 from git import InvalidGitRepositoryError, NoSuchPathError, Repo
 from pandas import PeriodIndex
 from requests import Session
-from requests.adapters import HTTPAdapter, Retry
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
 # Constants
 STATUS_FORCELIST = [
@@ -33,20 +35,23 @@ def __init__(self, message, exit_code=None):
         super().__init__(self.message)
 
 
-# def get_session(accept_header=None):
-#     """Create a reusable HTTP session with retry logic."""
-#     session = Session()
-
-
 def get_session(accept_header=None, session=None):
-    """Create or configure a reusable HTTP session
-    with retry logic and headers."""
+    """
+    Create or configure a reusable HTTPS session with retry logic and
+    appropriate headers.
+    """
     if session is None:
         session = Session()
 
+    # Purge default and custom session connection adapters
+    # (With only a https:// adapter, below, unencrypted requests will fail.)
+    session.adapters = OrderedDict()
+
+    # Try again after 0s, 6s, 12s, 24s, 48s (total 90s) for the specified HTTP
+    # error codes (STATUS_FORCELIST)
     retry_strategy = Retry(
         total=5,
-        backoff_factor=10,
+        backoff_factor=3,
         status_forcelist=STATUS_FORCELIST,
         allowed_methods=["GET", "POST"],
         raise_on_status=False,