re-implement with playwright

.github/workflows/publish.yml

@@ -11,18 +11,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
+    - uses: actions/checkout@v6
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
       with:
         python-version: '3.x'
-    - name: Install Poetry
-      uses: abatilo/actions-poetry@v2.1.6
-      with:
-        poetry-version: 1.5
     - name: Build and publish
       env:
-        PYPI_USERNAME: __token__
-        PYPI_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
       run: |
-        poetry publish -u $PYPI_USERNAME -p $PYPI_PASSWORD --build
+        uv build
+        uv publish

.github/workflows/test.yml

@@ -13,38 +13,40 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.10.13 ]
+        python-version: [ "3.11" ]
 
     steps:
 
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install Poetry
-      uses: abatilo/actions-poetry@v2.1.6
-      with:
-        poetry-version: 1.5
-
     - name: Install dependencies
       run: |
-        python -m poetry install
+        uv sync
 
-    - name: Prettify with black
-      uses: psf/black@stable
-      with:
-        version: "23.12.1"
+    - name: Install patchright browser
+      run: |
+        uv run patchright install --with-deps chromium
+
+    - name: Check formatting with ruff
+      run: |
+        uv run ruff format --check bcutils sample
+
+    - name: Lint with ruff
+      run: |
+        uv run ruff check bcutils sample
 
-    - name: Lint with flake8
+    - name: Type check with mypy
       run: |
-        python -m poetry run flake8 bcutils sample
+        uv run mypy bcutils sample
 
     - name: Unit tests with pytest
       env:
         BARCHART_USERNAME: ${{ secrets.BARCHART_USERNAME }}
         BARCHART_PASSWORD: ${{ secrets.BARCHART_PASSWORD }}
       run: |
-        python -m poetry run python -m pytest
+        xvfb-run -a uv run python -m pytest

.gitignore

@@ -38,3 +38,7 @@ inc/
 /.env
 /.venv
 /data
+auth.json
+/downloads
+humanization.log
+/.claude

README.md

@@ -1,6 +1,6 @@
 # bc-utils
 
-[Barchart.com](https://www.barchart.com) allows registered users to download historic futures contract prices in CSV format. Individual contracts must be downloaded separately, which is laborious and slow. This script automates the process for daily and hourly data.
+[Barchart.com](https://www.barchart.com) allows paid subscribers to download historic futures contract prices in CSV format. Individual contracts must be downloaded separately, which is laborious and slow. This script automates the process for daily and hourly data.
 
 ## Quickstart
 
@@ -45,7 +45,15 @@ Features:
 * by default gets 120 days of data per contract, override possible per instrument
 * dry run mode to check setup
 * allows updates to previously downloaded files
-* you must be a registered user. Paid subscribers get 250 downloads a day, otherwise 5
+* you must have a Barchart subscription. Premier users get 250 downloads a day, Plus users get 5
+
+### Browser requirements
+
+Since version 0.1.8, the code in this project drives a real browser (via [patchright](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright-python)) rather than making plain HTTP requests. Install the browser once with `patchright install chromium` (or `uv run patchright install chromium` in this repo).
+
+By default, the browser runs headed (`headless=False`). On a server or CI runner with no display, run under a virtual display such as Xvfb (e.g. `xvfb-run -a python your_script.py`), or pass `headless=True`.
+
+Login/session state persists across runs in `auth_dir` (defaults to `~/.bc_utils/auth`) so you don't have to log in again every run. Treat this directory like a browser profile - don't commit it or share it.
 
 ## For pysystemtrade users
 
@@ -91,9 +99,9 @@ def download_with_pst_config():
     )
 ```
 
-4. add bc-utils to your crontab
+4. add bc-utils to your crontab. Since a headless server has no display, run under `xvfb-run` (see [Browser requirements](#browser-requirements) above):
 ```
-00 08 * * 1-7 . $HOME/.profile; cd ~/bc-utils; python3 bcutils/sample/pst.py >> $ECHO_PATH/barchart_download.txt 2>&1
+00 08 * * 1-7 . $HOME/.profile; cd ~/bc-utils; xvfb-run -a python3 bcutils/sample/pst.py >> $ECHO_PATH/barchart_download.txt 2>&1
 ```
 You could also add another entry to run the updater once a week.
 
@@ -111,7 +119,7 @@ BARCHART_CONFIG = ConfigCsvFuturesPrices(
     input_date_index_name="Time",
     input_skiprows=0,
     input_skipfooter=0,
-    input_date_format="%Y-%m-%dT%H:%M:%S%z",
+    input_date_format="%Y-%m-%dT%H:%M:%S",
     input_column_mapping=dict(
         OPEN="Open", HIGH="High", LOW="Low", FINAL="Close", VOLUME="Volume"
     ),