diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 63ea6730..00000000 --- a/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -max-line-length=88 -extend-ignore=E203 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4d147ff2..8626d527 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,18 +11,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: python-version: '3.x' - - name: Install Poetry - uses: abatilo/actions-poetry@v2.1.6 - with: - poetry-version: 1.5 - name: Build and publish env: - PYPI_USERNAME: __token__ - PYPI_PASSWORD: ${{ secrets.PYPI_TOKEN }} + UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} run: | - poetry publish -u $PYPI_USERNAME -p $PYPI_PASSWORD --build + uv build + uv publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 178f5202..78a6aaed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,38 +13,40 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ 3.10.13 ] + python-version: [ "3.11" ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.python-version }} - - name: Install Poetry - uses: abatilo/actions-poetry@v2.1.6 - with: - poetry-version: 1.5 - - name: Install dependencies run: | - python -m poetry install + uv sync - - name: Prettify with black - uses: psf/black@stable - with: - version: "23.12.1" + - name: Install patchright browser + run: | + uv run patchright install --with-deps chromium + + - name: Check formatting with ruff + run: | + uv run ruff format --check bcutils sample + + - name: Lint with ruff + run: | + uv run ruff check bcutils sample - - name: Lint with flake8 + - name: Type check with mypy run: | - python -m poetry run flake8 bcutils sample + uv run mypy bcutils sample - name: Unit tests with pytest env: BARCHART_USERNAME: ${{ secrets.BARCHART_USERNAME }} BARCHART_PASSWORD: ${{ secrets.BARCHART_PASSWORD }} run: | - python -m poetry run python -m pytest + xvfb-run -a uv run python -m pytest diff --git a/.gitignore b/.gitignore index 2e0b7c5e..e39cf45a 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ inc/ /.env /.venv /data +auth.json +/downloads +humanization.log +/.claude diff --git a/README.md b/README.md index 4691f89b..1ad9ad5e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # bc-utils -[Barchart.com](https://www.barchart.com) allows registered users to download historic futures contract prices in CSV format. Individual contracts must be downloaded separately, which is laborious and slow. This script automates the process for daily and hourly data. +[Barchart.com](https://www.barchart.com) allows paid subscribers to download historic futures contract prices in CSV format. Individual contracts must be downloaded separately, which is laborious and slow. This script automates the process for daily and hourly data. ## Quickstart @@ -45,7 +45,15 @@ Features: * by default gets 120 days of data per contract, override possible per instrument * dry run mode to check setup * allows updates to previously downloaded files -* you must be a registered user. Paid subscribers get 250 downloads a day, otherwise 5 +* you must have a Barchart subscription. Premier users get 250 downloads a day, Plus users get 5 + +### Browser requirements + +Since version 0.1.8, the code in this project drives a real browser (via [patchright](https://github.com/Kaliiiiiiiiii-Vinyzu/patchright-python)) rather than making plain HTTP requests. Install the browser once with `patchright install chromium` (or `uv run patchright install chromium` in this repo). + +By default, the browser runs headed (`headless=False`). On a server or CI runner with no display, run under a virtual display such as Xvfb (e.g. `xvfb-run -a python your_script.py`), or pass `headless=True`. + +Login/session state persists across runs in `auth_dir` (defaults to `~/.bc_utils/auth`) so you don't have to log in again every run. Treat this directory like a browser profile - don't commit it or share it. ## For pysystemtrade users @@ -91,9 +99,9 @@ def download_with_pst_config(): ) ``` -4. add bc-utils to your crontab +4. add bc-utils to your crontab. Since a headless server has no display, run under `xvfb-run` (see [Browser requirements](#browser-requirements) above): ``` -00 08 * * 1-7 . $HOME/.profile; cd ~/bc-utils; python3 bcutils/sample/pst.py >> $ECHO_PATH/barchart_download.txt 2>&1 +00 08 * * 1-7 . $HOME/.profile; cd ~/bc-utils; xvfb-run -a python3 bcutils/sample/pst.py >> $ECHO_PATH/barchart_download.txt 2>&1 ``` You could also add another entry to run the updater once a week. @@ -111,7 +119,7 @@ BARCHART_CONFIG = ConfigCsvFuturesPrices( input_date_index_name="Time", input_skiprows=0, input_skipfooter=0, - input_date_format="%Y-%m-%dT%H:%M:%S%z", + input_date_format="%Y-%m-%dT%H:%M:%S", input_column_mapping=dict( OPEN="Open", HIGH="High", LOW="Low", FINAL="Close", VOLUME="Volume" ), diff --git a/bcutils/bc_utils.py b/bcutils/bc_utils.py index f1005d49..3601dc7a 100644 --- a/bcutils/bc_utils.py +++ b/bcutils/bc_utils.py @@ -1,3 +1,4 @@ +import asyncio import calendar import enum import io @@ -6,21 +7,29 @@ import os import os.path import pytz +import random import re -import time +import sys import traceback -import urllib.parse +from dataclasses import dataclass from datetime import datetime, timedelta from itertools import cycle from pathlib import Path from random import randint import pandas as pd -import requests from bs4 import BeautifulSoup +from humanization import Humanization, HumanizationConfig +from loguru import logger as _loguru_logger +from patchright.async_api import TimeoutError as PlaywrightTimeoutError +from patchright.async_api import async_playwright from bcutils.config import CONTRACT_MAP, EXCHANGES +# redirect humanization logging to stdout instead. +_loguru_logger.remove() +_loguru_logger.add(sys.stdout) + logger = logging.getLogger(__name__) @@ -64,237 +73,537 @@ class EmptyDataException(Exception): MONTH_LIST = ["F", "G", "H", "J", "K", "M", "N", "Q", "U", "V", "X", "Z"] BARCHART_URL = "https://www.barchart.com/" +_DEFAULT_AUTH_DIR = Path.home() / ".bc_utils" / "auth" + +_HUMANIZATION_CONFIG = HumanizationConfig( + fast=True, + humanize=True, + characters_per_minute=500, + backspace_cpm=400, + timeout=10000, + stealth_mode=True, +) + -def create_bc_session(config_obj: dict, do_login=True): +@dataclass +class BarchartSession: + """ + Holds credentials for a lazy, browser-driven Barchart login. """ - Create and return a web session, optionally logging into Barchart with the supplied - credentials. - Args: - config_obj: dict containing Barchart credentials, with keys `barchart_username` - and `barchart_password` - do_login: if True, authenticate session with Barchart credentials + username: str + password: str - Returns: - A requests.Session instance +def _disable_password_manager(user_data_dir: Path) -> None: + """Pre-seed the profile so "Save password?" bubble never appears""" + profile_dir = user_data_dir / "Default" + profile_dir.mkdir(parents=True, exist_ok=True) + prefs_path = profile_dir / "Preferences" + + prefs = {} + if prefs_path.exists(): + try: + prefs = json.loads(prefs_path.read_text()) + except (json.JSONDecodeError, OSError): + prefs = {} + + prefs["credentials_enable_service"] = False + prefs.setdefault("profile", {}) + prefs["profile"]["password_manager_enabled"] = False + + prefs_path.write_text(json.dumps(prefs)) + + +async def _pause( + human: Humanization, base_min: float = 0.2, base_max: float = 1.0 +) -> None: + # wait a randomised duration, jittering base_min/base_max themselves each call + jitter = random.uniform(0.6, 1.6) + min_sec = base_min * jitter + max_sec = max(min_sec + 0.1, base_max * jitter * random.uniform(1.0, 1.4)) + await human.human_wait(min_sec=min_sec, max_sec=max_sec) + + +def _make_download_response_handler(allowance_slot: dict): + """Capture the allowance-check response (success/count, or an error when + the daily download limit is reached)""" + + async def handler(response): + if ( + response.request.method != "POST" + or response.request.url != BARCHART_URL + "my/download" + ): + return + content_type = response.headers.get("content-type", "") + if "json" not in content_type: + return + try: + body = await response.json() + except Exception: # skipcq broad by design + return + allowance_slot.update(body) + + return handler + + +async def _launch_barchart_browser( + playwright, auth_dir: Path, downloads_dir: Path, headless: bool +): + _disable_password_manager(auth_dir) + downloads_dir.mkdir(parents=True, exist_ok=True) + + context = await playwright.chromium.launch_persistent_context( + user_data_dir=str(auth_dir), + args=["--disable-blink-features=AutomationControlled"], + headless=headless, + no_viewport=True, + accept_downloads=True, + downloads_path=str(downloads_dir), + ) + page = await context.new_page() + human = Humanization(page, _HUMANIZATION_CONFIG) + return context, human + + +async def _login_async(human: Humanization, username: str, password: str) -> None: + logger.info("step: goto barchart.com") + await human.page.goto(BARCHART_URL) + await _pause(human) + + allow_all = human.page.get_by_role("button", name="Allow all") + if await allow_all.count() > 0: + logger.info("step: accept cookie banner") + await human.hover_at(allow_all) + await _pause(human) + await human.click_at(allow_all) + await _pause(human) + else: + logger.info("step: cookie banner not present, skipping") + + login_link = human.page.get_by_role("link", name="LOGIN", description="LOGIN") + if await login_link.count() > 0: + logger.info("step: click LOGIN link") + await human.hover_at(login_link) + await _pause(human) + await human.click_at(login_link) + await _pause(human) + else: + logger.info("step: LOGIN link not present, skipping") + + email_box = human.page.get_by_role("textbox", name="Login with Email") + if await email_box.count() > 0: + logger.info("step: fill email") + await human.type_at(email_box, username) + await _pause(human) + + password_box = human.page.get_by_role("textbox", name="Password") + if await password_box.count() > 0: + logger.info("step: fill password") + await human.type_at(password_box, password) + await _pause(human) + + login_button = human.page.get_by_role("button", name="Login") + if await login_button.count() > 0: + logger.info("step: submit login") + await human.hover_at(login_button) + await _pause(human) + await human.click_at(login_button) + await _pause(human) + + # if email field is still there after attempted login, it failed + if await email_box.count() > 0: + raise BCException("Invalid credentials") + + +def create_bc_session(config_obj: dict) -> BarchartSession: + """ + Validate credentials and return a BarchartSession. + Args: + config_obj: dict containing Barchart credentials + Returns: + A BarchartSession instance Raises: - Exception: if credentials are invalid + BCException: if credentials are missing """ + if "barchart_username" not in config_obj or "barchart_password" not in config_obj: + raise BCException("Missing credentials") + return BarchartSession( + username=config_obj["barchart_username"], + password=config_obj["barchart_password"], + ) + + +def _normalize_downloaded_csv(save_path: str, res: Resolution) -> int: + """fix column names, date format, remove footer; returns row count""" + dateformat = "%Y-%m-%d" if res == Resolution.Day else "%Y-%m-%d %H:%M" + + df = pd.read_csv(save_path, skipfooter=1, engine="python") + df["Time"] = pd.to_datetime(df["Time"], format=dateformat) + df.set_index("Time", inplace=True) + df.index = df.index.tz_localize(tz="US/Central").tz_convert("UTC") + df = df.rename(columns={"Latest": "Close"}) + df = df[["Open", "High", "Low", "Close", "Volume"]] + + df.to_csv(save_path, date_format="%Y-%m-%dT%H:%M:%S%z") + return len(df) + + +async def _save_prices_for_contract_async( + human: Humanization, + contract: str, + save_path: str, + start_date: datetime, + end_date: datetime, + dry_run: bool, + allowance_slot: dict, + instr_config: dict | None = None, + insufficient_check_margin_days: int = 365, +): + res = _get_resolution(save_path) - session = requests.Session() - session.headers.update({"User-Agent": "Mozilla/5.0"}) - if do_login is True and ( - "barchart_username" not in config_obj or "barchart_password" not in config_obj + # do we have this file already? + if os.path.isfile(save_path): + logger.info( + f"{res.adj} data for contract '{contract}' already downloaded " + f"({save_path}) - skipping" + ) + return HistoricalDataResult.EXISTS + + if instr_config is not None and _near_available_res_boundary( + res, start_date, instr_config, insufficient_check_margin_days ): - raise BCException("Missing credentials") + logger.info(f"step: near availability boundary, checking '{contract}'") + if await _insufficient_data_async(human, contract, res): + logger.info(f"Insufficient {res.adj} data for '{contract}' - skipping") + return HistoricalDataResult.INSUFFICIENT + + logger.info( + f"getting historic {res.adj} prices for contract '{contract}', " + f"from {start_date.strftime('%Y-%m-%d')} " + f"to {end_date.strftime('%Y-%m-%d')}" + ) + + try: + url = f"{BARCHART_URL}futures/quotes/{contract}/historical-download" + logger.info(f"step: goto historical-download page for {contract}") + response = await human.page.goto(url) + if response is None or response.status != 200: + logger.info(f"No downloadable data found for contract '{contract}'") + return HistoricalDataResult.NONE + + # give the page's own JS time to finish hydrating + await _pause(human) + + if dry_run: + logger.info(f"Not downloading {contract}, dry_run") + return HistoricalDataResult.OK + + select_value = "string:daily" if res == Resolution.Day else "string:minutes" + + allowance_slot.clear() + logger.info(f"step: select frequency ({select_value})") + frequency_select = human.page.get_by_label("Select frequency") + max_attempts = 3 + for attempt in range(max_attempts): + await frequency_select.select_option(select_value) + await _pause(human) + actual_value = await frequency_select.input_value() + if actual_value == select_value: + break + logger.warning( + f"frequency select shows '{actual_value}', expected " + f"'{select_value}' for '{contract}' - retrying " + f"(attempt {attempt + 1}/{max_attempts})" + ) + else: + logger.error( + f"frequency select never settled on '{select_value}' for " + f"'{contract}' after {max_attempts} attempts - aborting" + ) + return HistoricalDataResult.NONE + + if res == Resolution.Hour: + logger.info("step: set intraday minutes to 60") + minutes_box = human.page.get_by_role( + "spinbutton", name="Enter intraday minutes" + ) + await minutes_box.fill("") + await human.type_at(minutes_box, "60") + await _pause(human) + + logger.info("step: select ordering (asc)") + await human.page.get_by_label("Select Ordering").select_option("asc") + await _pause(human) + + # set start date + logger.info(f"step: set start date ({start_date.strftime('%m/%d/%Y')})") + start_box = human.page.get_by_role("textbox", name="Start Date") + await start_box.click() + await _pause(human) + await human.page.get_by_role("button", name="Clear").click() + await _pause(human) + await human.type_at(start_box, start_date.strftime("%m/%d/%Y")) + await human.page.keyboard.press("Escape") + await _pause(human) + + # set end date + logger.info(f"step: set end date ({end_date.strftime('%m/%d/%Y')})") + end_box = human.page.get_by_role("textbox", name="End Date") + await end_box.click() + await _pause(human) + await human.page.get_by_role("button", name="Clear").click() + await _pause(human) + await human.type_at(end_box, end_date.strftime("%m/%d/%Y")) + await human.page.keyboard.press("Escape") + await _pause(human) + + logger.info(f"step: click Download for '{contract}'") + download_link = human.page.get_by_text("Download", exact=True).first + await human.hover_at(download_link) + try: + async with human.page.expect_download(timeout=15000) as download_info: + await human.click_at(download_link) + download = await download_info.value + except PlaywrightTimeoutError: + if allowance_slot.get("error") is not None: + logger.info(f"Max daily download reached for '{contract}'") + return HistoricalDataResult.EXCEED + raise + + if allowance_slot.get("error") is not None: + logger.info(f"Max daily download reached for '{contract}'") + return HistoricalDataResult.EXCEED + + failure = await download.failure() + if failure: + logger.info(f"Barchart data problem for '{contract}', not writing") + return HistoricalDataResult.OK + + logger.info(f"step: saving download to {save_path}") + await download.save_as(save_path) + row_count = _normalize_downloaded_csv(save_path, res) + if row_count < 30: + os.remove(save_path) + logger.info(f"Insufficient {res.adj} data for '{contract}' - skipping") + return HistoricalDataResult.INSUFFICIENT - if do_login: - # GET the login page, scrape to get CSRF token - resp = session.get(BARCHART_URL + "login") - soup = BeautifulSoup(resp.text, "html.parser") - tag = soup.find(type="hidden") - csrf_token = tag.attrs["value"] logger.info( - f"GET {BARCHART_URL + 'login'}, status: {resp.status_code}, " - f"CSRF token: {csrf_token}" + f"Finished getting Barchart historic {res.adj} prices for {contract}" ) + return HistoricalDataResult.OK - # login to site with a POST - payload = { - "email": config_obj["barchart_username"], - "password": config_obj["barchart_password"], - "_token": csrf_token, - } - resp = session.post(BARCHART_URL + "login", data=payload) - logger.info(f"POST {BARCHART_URL + 'login'}, status: {resp.status_code}") - if resp.url == BARCHART_URL + "login": - raise BCException("Invalid credentials") + except Exception as e: # skipcq broad by design + logger.error(f"Error {e}") - return session + +async def _save_prices_for_contract_standalone_async( + session: BarchartSession, + contract: str, + save_path: str, + start_date: datetime, + end_date: datetime, + dry_run: bool, + headless: bool, + auth_dir: str | None = None, +): + auth_dir_path = Path(auth_dir) if auth_dir else _DEFAULT_AUTH_DIR + downloads_dir = Path(save_path).resolve().parent + + async with async_playwright() as playwright: + context, human = await _launch_barchart_browser( + playwright, auth_dir_path, downloads_dir, headless + ) + try: + await _login_async(human, session.username, session.password) + + allowance_slot: dict = {} + human.page.on("response", _make_download_response_handler(allowance_slot)) + + return await _save_prices_for_contract_async( + human, + contract, + save_path, + start_date, + end_date, + dry_run, + allowance_slot, + ) + finally: + await context.close() def save_prices_for_contract( - session: requests.Session, + session: BarchartSession, contract: str, save_path: str, start_date: datetime, end_date: datetime, dry_run: bool = False, + headless: bool = False, + auth_dir: str | None = None, ): """ Save prices for an individual futures contract. Args: - session: requests.Session instance + session: a BarchartSession instance contract: Barchart style contract identifier, eg GCH24 for March 2024 Gold save_path: full path where price file will be saved start_date: start date end_date: end date dry_run: if True, provides useful diagnostic info but does not execute + headless: if True, run the browser without a visible window. Requires a + display (e.g. Xvfb) on headless servers. Defaults to False + auth_dir: directory to persist the browser's login/session state across + runs. Defaults to ~/.bc_utils/auth Returns: A HistoricalDataResult instance, representing the result of the operation """ + return asyncio.run( + _save_prices_for_contract_standalone_async( + session, + contract, + save_path, + start_date, + end_date, + dry_run, + headless, + auth_dir, + ) + ) - res = _get_resolution(save_path) - - try: - # do we have this file already? - if os.path.isfile(save_path): - logger.info( - f"{res.adj} data for contract '{contract}' already downloaded " - f"({save_path}) - skipping\n" - ) - return HistoricalDataResult.EXISTS - if _insufficient_data(session, contract, res): - logger.info(f"Insufficient {res.adj} data for '{contract}' - skipping\n") - return HistoricalDataResult.INSUFFICIENT - - except Exception as e: # skipcq broad by design - logger.error(f"Problem: {e}, {traceback.format_exc()}") +async def _get_barchart_downloads_async( + session: BarchartSession, + contract_map, + contract_list, + instr_list, + save_dir, + start_year, + end_year, + dry_run, + do_daily, + pause_between_downloads, + default_day_count, + headless, + auth_dir, + insufficient_check_margin_days, +): + if contract_map is None: + contract_map = CONTRACT_MAP - logger.info( - f"getting historic {res.adj} prices for contract '{contract}', " - f"from {start_date.strftime('%Y-%m-%d')} " - f"to {end_date.strftime('%Y-%m-%d')}" - ) + inv_contract_map = _build_inverse_map(contract_map) - try: - # open historic data download page for required contract - url = f"{BARCHART_URL}futures/quotes/{contract}/historical-download" - hist_resp = session.get(url) - logger.info(f"GET {url}, status {hist_resp.status_code}") + max_exceeded = False - if hist_resp.status_code != 200: - logger.info(f"No downloadable data found for contract '{contract}'\n") - return HistoricalDataResult.NONE + if contract_list is None: + contract_list = _build_contract_list( + start_year, end_year, instr_list=instr_list, contract_map=contract_map + ) - xsrf = urllib.parse.unquote(hist_resp.cookies["XSRF-TOKEN"]) + auth_dir_path = Path(auth_dir) if auth_dir else _DEFAULT_AUTH_DIR + downloads_dir = Path(save_dir) if save_dir else Path(os.getcwd()) - # scrape page for csrf_token - hist_soup = BeautifulSoup(hist_resp.text, "html.parser") - hist_tag = hist_soup.find(name="meta", attrs={"name": "csrf-token"}) - hist_csrf_token = hist_tag.attrs["content"] + async with async_playwright() as playwright: + context, human = await _launch_barchart_browser( + playwright, auth_dir_path, downloads_dir, headless + ) + try: + await _login_async(human, session.username, session.password) - # check allowance - payload = {"onlyCheckPermissions": "true"} - headers = { - "content-type": "application/x-www-form-urlencoded", - "Accept-Encoding": "gzip, deflate, br", - "Referer": url, - "x-xsrf-token": xsrf, - } - resp = session.post(BARCHART_URL + "my/download", headers=headers, data=payload) + allowance_slot: dict = {} + human.page.on("response", _make_download_response_handler(allowance_slot)) - allowance = json.loads(resp.text) + for contract in contract_list: + if max_exceeded: + break - if allowance.get("error") is not None: - return HistoricalDataResult.EXCEED + for resolution in Resolution if do_daily else [Resolution.Hour]: + # work out instrument code and get config + market_code = contract[: len(contract) - 3] + instr_code = inv_contract_map[market_code.upper()] + instr_config = contract_map[instr_code] - if allowance["success"]: - logger.info( - f"POST {BARCHART_URL + 'my/download'}, " - f"status: {resp.status_code}, " - f"allowance success: {allowance['success']}, " - f"allowance count: {allowance['count']}" - ) + # get contract month and year + month, year = _get_contract_month_year(contract) - # download data - xsrf = urllib.parse.unquote(resp.cookies["XSRF-TOKEN"]) - headers = { - "content-type": "application/x-www-form-urlencoded", - "Accept-Encoding": "gzip, deflate, br", - "Referer": url, - "x-xsrf-token": xsrf, - } - - payload = { - "_token": hist_csrf_token, - "fileName": contract + "_Daily_Historical Data", - "symbol": contract, - "fields": "tradeTime.format(Y-m-d),openPrice,highPrice,lowPrice," - "lastPrice,volume", - "startDate": start_date.strftime("%Y-%m-%d"), - "endDate": end_date.strftime("%Y-%m-%d"), - "orderBy": "tradeTime", - "orderDir": "asc", - "method": "historical", - "limit": "20000", - "customView": "true", - "pageTitle": "Historical Data", - } - - dateformat = "%Y-%m-%d %H:%M" - if res == Resolution.Day: - payload["type"] = "eod" - payload["period"] = "daily" - dateformat = "%Y-%m-%d" - - elif res == Resolution.Hour: - payload["type"] = "minutes" - payload["interval"] = 60 - - if not dry_run: - resp = session.post( - BARCHART_URL + "my/download", headers=headers, data=payload - ) - logger.info( - f"POST {BARCHART_URL + 'my/download'}, " - f"status: {resp.status_code}, " - f"data length: {len(resp.content)}" - ) - if resp.status_code == 200: - if "Error retrieving data" not in resp.text: - iostr = io.StringIO(resp.text) - df = pd.read_csv(iostr, skipfooter=1, engine="python") - df["Time"] = pd.to_datetime(df["Time"], format=dateformat) - df.set_index("Time", inplace=True) - df.index = df.index.tz_localize(tz="US/Central").tz_convert( - "UTC" - ) - df = df.rename(columns={"Last": "Close"}) + # build save path + save_path = _build_save_path( + instr_code, month, year, resolution, save_dir + ) - logger.info(f"writing to: {save_path}") - df.to_csv(save_path, date_format="%Y-%m-%dT%H:%M:%S%z") + # calculate date range + start_date, end_date = _get_start_end_dates( + month, + year, + instr_config, + default_day_count=default_day_count, + ) - else: + if _before_available_res(resolution, start_date, instr_config): + date_type = "tick" if resolution == Resolution.Hour else "EOD" logger.info( - f"Barchart data problem for '{contract}', not writing" + f"{resolution.adj} prices for {contract} starting " + f"{start_date.strftime('%Y-%m-%d')} is before configured " + f"{date_type} date - skipping" ) - else: - logger.info(f"Not POSTing to {BARCHART_URL + 'my/download'}, dry_run") - - logger.info( - f"Finished getting Barchart historic {res.adj} prices for {contract}\n" - ) - - return HistoricalDataResult.OK + continue + + # download and save + result = await _save_prices_for_contract_async( + human, + contract, + save_path, + start_date, + end_date, + dry_run, + allowance_slot, + instr_config, + insufficient_check_margin_days, + ) - except Exception as e: # skipcq broad by design - logger.error(f"Error {e}") + if result in [ + HistoricalDataResult.EXISTS, + HistoricalDataResult.NONE, + HistoricalDataResult.INSUFFICIENT, + ]: + continue + elif result == HistoricalDataResult.EXCEED: + logger.info("Max daily download reached, aborting") + max_exceeded = True + break + else: + if pause_between_downloads: + # cursory attempt to not appear like a bot + await asyncio.sleep(0 if dry_run else randint(7, 15)) + finally: + await context.close() def get_barchart_downloads( - session: requests.Session, - contract_map: dict = None, - contract_list: list = None, - instr_list: list = None, - save_dir: str = None, + session: BarchartSession, + contract_map: dict | None = None, + contract_list: list | None = None, + instr_list: list | None = None, + save_dir: str | None = None, start_year: int = 1950, end_year: int = 2025, dry_run: bool = False, do_daily: bool = True, pause_between_downloads: bool = True, default_day_count: int = 400, + headless: bool = False, + auth_dir: str | None = None, + insufficient_check_margin_days: int = 365, ): """ Run a download session, performing as many contract downloads as possible, given the config, parameters, existing files, and available daily allowance. Args: - session: requests.Session instance + session: a BarchartSession instance contract_map: dict containing instrument config contract_list: optional list of Barchart contract IDs we want to download in this run. If provided, `start_year` and `start_year` are ignored. If not @@ -309,96 +618,119 @@ def get_barchart_downloads( do_daily: if True, download daily as well as hourly price files pause_between_downloads: if True, wait a random short period between downloads default_day_count: default number of days of data to download + headless: if True, run the browser without a visible window. Requires a + display (e.g. Xvfb) on headless servers. Defaults to False + auth_dir: directory to persist the browser's login/session state across + runs. Defaults to ~/.bc_utils/auth + insufficient_check_margin_days: contracts expiring within this many days of + the exchange's published tick_date/eod_date get a live insufficient-data + check before downloading, since those published limits are often + inaccurate """ - if contract_map is None: - contract_map = CONTRACT_MAP - - inv_contract_map = _build_inverse_map(contract_map) - - max_exceeded = False - try: - if contract_list is None: - contract_list = _build_contract_list( - start_year, end_year, instr_list=instr_list, contract_map=contract_map + asyncio.run( + _get_barchart_downloads_async( + session, + contract_map, + contract_list, + instr_list, + save_dir, + start_year, + end_year, + dry_run, + do_daily, + pause_between_downloads, + default_day_count, + headless, + auth_dir, + insufficient_check_margin_days, ) + ) + except Exception as e: # skipcq broad by design + logger.error(f"Error {e}") + traceback.print_exc() - for contract in contract_list: - if max_exceeded: - break - for resolution in Resolution if do_daily else [Resolution.Hour]: - # work out instrument code and get config - market_code = contract[: len(contract) - 3] - instr_code = inv_contract_map[market_code.upper()] - instr_config = contract_map[instr_code] +async def _update_barchart_downloads_async( + instr_code, + contract_map, + save_dir, + days_ago, + dry_run, + split_freq, + session: BarchartSession, + headless, + auth_dir, +): + if contract_map is None: + contract_map = CONTRACT_MAP + + from_date = datetime.now() - timedelta(days=days_ago) - # get contract month and year - month, year = _get_contract_month_year(contract) + logger.info(f"Updating contract prices for {instr_code}") - # build save path - save_path = _build_save_path( - instr_code, month, year, resolution, save_dir - ) + check_integrity_list = [] - # calculate date range - start_date, end_date = _get_start_end_dates( - month, - year, - instr_config, - default_day_count=default_day_count, - ) + file_names = _get_filenames(instr_code, save_dir, split_freq) - if _before_available_res(resolution, start_date, instr_config): - date_type = "tick" if resolution == Resolution.Hour else "EOD" - logger.info( - f"{resolution.adj} prices for {contract} starting " - f"{start_date.strftime('%Y-%m-%d')} is before configured " - f"{date_type} date - skipping\n" - ) - continue - - # download and save - result = save_prices_for_contract( - session, - contract, - save_path, - start_date, - end_date, - dry_run=dry_run, - ) + auth_dir_path = Path(auth_dir) if auth_dir else _DEFAULT_AUTH_DIR + downloads_dir = Path(save_dir) if save_dir else Path(os.getcwd()) - if result in [ - HistoricalDataResult.EXISTS, - HistoricalDataResult.NONE, - HistoricalDataResult.INSUFFICIENT, - ]: - continue - elif result == HistoricalDataResult.EXCEED: - logger.info("Max daily download reached, aborting") - max_exceeded = True - break + async with async_playwright() as playwright: + context, human = await _launch_barchart_browser( + playwright, auth_dir_path, downloads_dir, headless + ) + try: + await _login_async(human, session.username, session.password) + + for file in file_names: + instr_code = _instr_code_from_file_name(file, split_freq=split_freq) + if split_freq: + res = _res_from_file_name(file) else: - if pause_between_downloads: - # cursory attempt to not appear like a bot - time.sleep(0 if dry_run else randint(7, 15)) + res = None + contract_date = _contract_date_from_file_name(file) + contract_id = _get_barchart_id( + instr_code, contract_date.year, contract_date.month + ) - # logout - resp = session.get(BARCHART_URL + "logout", timeout=10) - logger.info(f"GET {BARCHART_URL + 'logout'}, status: {resp.status_code}") + if contract_date > from_date: + if dry_run: + print( + f"DRY RUN: would update contract {contract_id}, " + f"file {file}" + ) + else: + try: + await _update_barchart_contract_file_async( + human, contract_map, save_dir, contract_id, res + ) + except IntegrityException: + logger.error( + f"File index problem with {file}, please check" + ) + check_integrity_list.append(file) + except RecentUpdateException: + logger.warning(f"Skipping {contract_id}, recently updated") + except EmptyDataException: + logger.info(f"Empty data for {contract_id}") + finally: + await context.close() - except Exception as e: # skipcq broad by design - logger.error(f"Error {e}") - traceback.print_exc() + if len(check_integrity_list) > 0: + print(f"These files have integrity problems: {check_integrity_list}") def update_barchart_downloads( instr_code: str = "GOLD", - contract_map: dict = None, - save_dir: str = None, + contract_map: dict | None = None, + save_dir: str | None = None, days_ago: int = 360, dry_run: bool = False, split_freq: bool = True, + session: BarchartSession | None = None, + headless: bool = False, + auth_dir: str | None = None, ): """ Update recent previously downloaded files for an instrument. @@ -416,50 +748,30 @@ def update_barchart_downloads( the 1st of the month. So GCH23 would be 1st March 2023 dry_run: if True, provides useful diagnostic info but does not execute split_freq: True if we are expecting to find split frequency files + session: a BarchartSession instance from create_bc_session(). If not + provided, one is built from BARCHART_USERNAME/BARCHART_PASSWORD + environment variables + headless: if True, run the browser without a visible window. Requires a + display (e.g. Xvfb) on headless servers. Defaults to False + auth_dir: directory to persist the browser's login/session state across + runs. Defaults to ~/.bc_utils/auth """ - if contract_map is None: - contract_map = CONTRACT_MAP - - from_date = datetime.now() - timedelta(days=days_ago) - - logger.info(f"Updating contract prices for {instr_code}") - - session = requests.Session() - session.headers.update({"User-Agent": "Mozilla/5.0"}) - - check_integrity_list = [] - - file_names = _get_filenames(instr_code, save_dir, split_freq) - - for file in file_names: - instr_code = _instr_code_from_file_name(file, split_freq=split_freq) - if split_freq: - res = _res_from_file_name(file) - else: - res = None - contract_date = _contract_date_from_file_name(file) - contract_id = _get_barchart_id( - instr_code, contract_date.year, contract_date.month + if session is None: + session = create_bc_session(config_obj=_env()) + + asyncio.run( + _update_barchart_downloads_async( + instr_code, + contract_map, + save_dir, + days_ago, + dry_run, + split_freq, + session, + headless, + auth_dir, ) - - if contract_date > from_date: - if dry_run: - print(f"DRY RUN: would update contract {contract_id}, file {file}") - else: - try: - update_barchart_contract_file( - session, contract_map, save_dir, contract_id, res - ) - except IntegrityException: - logger.error(f"File index problem with {file}, please check") - check_integrity_list.append(file) - except RecentUpdateException: - logger.warning(f"Skipping {contract_id}, recently updated") - except EmptyDataException: - logger.info(f"Empty data for {contract_id}") - - if len(check_integrity_list) > 0: - print(f"These files have integrity problems: {check_integrity_list}") + ) def _get_filenames(instr_code, save_dir, split_freq: bool = True): @@ -475,28 +787,13 @@ def _get_filenames(instr_code, save_dir, split_freq: bool = True): return file_names -def update_barchart_contract_file( - session: requests.Session, +async def _update_barchart_contract_file_async( + human: Humanization, contract_map: dict, path: str, contract_id: str, res: Resolution, ): - """ - Update a previously downloaded contract price file. - - Args: - session: requests.Session instance - contract_map: dict containing instrument config - path: full path to the directory where previously downloaded files are located - contract_id: Barchart style contract identifier, eg GCH24 for March 2024 Gold - res: Resolution.Hour or Resolution.Day - Raises: - IntegrityException: raised if a problem is encountered when trying to set the - datetime column as index - RecentUpdateException: raised if the file has been recently updated - EmptyDataException: raised if the update contains no data - """ inv_contract_map = _build_inverse_map(contract_map) file = _filename_from_barchart_id(contract_id, inv_contract_map, res) @@ -523,7 +820,7 @@ def update_barchart_contract_file( f"last entry: {last_index_date}" ) - update = get_historical_prices_for_contract(session, contract_id, res) + update = await _get_historical_prices_for_contract_async(human, contract_id, res) if res == Resolution.Hour: start = last_index_date + timedelta(hours=1) else: @@ -546,67 +843,137 @@ def update_barchart_contract_file( raise EmptyDataException(f"Empty data for {contract_id}") -def get_historical_prices_for_contract( - session, instr_code: str, resolution: Resolution = Resolution.Day +async def _update_barchart_contract_file_standalone_async( + session: BarchartSession, + contract_map: dict, + path: str, + contract_id: str, + res: Resolution, + headless: bool, + auth_dir: str | None = None, +): + auth_dir_path = Path(auth_dir) if auth_dir else _DEFAULT_AUTH_DIR + downloads_dir = Path(path) + + async with async_playwright() as playwright: + context, human = await _launch_barchart_browser( + playwright, auth_dir_path, downloads_dir, headless + ) + try: + await _login_async(human, session.username, session.password) + return await _update_barchart_contract_file_async( + human, contract_map, path, contract_id, res + ) + finally: + await context.close() + + +def update_barchart_contract_file( + session: BarchartSession, + contract_map: dict, + path: str, + contract_id: str, + res: Resolution, + headless: bool = False, + auth_dir: str | None = None, +): + """ + Update a previously downloaded contract price file. + + Args: + session: a BarchartSession instance + contract_map: dict containing instrument config + path: full path to the directory where previously downloaded files are located + contract_id: Barchart style contract identifier, eg GCH24 for March 2024 Gold + res: Resolution.Hour or Resolution.Day + headless: if True, run the browser without a visible window. Requires a + display (e.g. Xvfb) on headless servers. Defaults to False + auth_dir: directory to persist the browser's login/session state across + runs. Defaults to ~/.bc_utils/auth + Raises: + IntegrityException: raised if a problem is encountered when trying to set the + datetime column as index + RecentUpdateException: raised if the file has been recently updated + EmptyDataException: raised if the update contains no data + """ + return asyncio.run( + _update_barchart_contract_file_standalone_async( + session, contract_map, path, contract_id, res, headless, auth_dir + ) + ) + + +def _historical_prices_predicate(resolution: Resolution): + # build a page.expect_response() predicate + if resolution == Resolution.Day: + url_prefix = BARCHART_URL + "proxies/timeseries/historical/queryeod.ashx" + required = ("volume=contract", "data=daily") + else: + url_prefix = BARCHART_URL + "proxies/timeseries/historical/queryminutes.ashx" + required = ("volume=contract", "interval=60") + + def predicate(response) -> bool: + return ( + response.request.method == "GET" + and response.request.url.startswith(url_prefix) + and all(token in response.request.url for token in required) + ) + + return predicate + + +async def _get_historical_prices_for_contract_async( + human: Humanization, contract_id: str, resolution: Resolution = Resolution.Day ) -> pd.DataFrame: - if not instr_code: - raise BCException("instr_code is required") + if not contract_id: + raise BCException("contract_id is required") try: - # GET the futures quote chart page, scrape to get XSRF token - # https://www.barchart.com/futures/quotes/GCM21/interactive-chart - chart_url = BARCHART_URL + f"futures/quotes/{instr_code}/interactive-chart" - chart_resp = session.get(chart_url) - xsrf = urllib.parse.unquote(chart_resp.cookies["XSRF-TOKEN"]) - - headers = { - "content-type": "text/plain; charset=UTF-8", - "Accept-Encoding": "gzip, deflate, br, zstd", - "Referer": chart_url, - "x-xsrf-token": xsrf, - } - - payload = { - "symbol": instr_code, - "maxrecords": "640", - "volume": "contract", - "order": "asc", - "dividends": "false", - "backadjust": "false", - "daystoexpiration": "1", - "contractroll": "combined", - } - - if resolution == Resolution.Day: - data_url = BARCHART_URL + "proxies/timeseries/historical/queryeod.ashx" - payload["data"] = "daily" - else: - data_url = BARCHART_URL + "proxies/timeseries/historical/queryminutes.ashx" - payload["interval"] = "60" - - # get prices for instrument from BC internal API - prices_resp = session.get(data_url, headers=headers, params=payload) - if prices_resp.status_code != 200: - raise Exception( - f"response status: {prices_resp.status_code} {prices_resp.reason}" + chart_url = f"{BARCHART_URL}futures/quotes/{contract_id}/interactive-chart" + resolution_label = "Daily" if resolution == Resolution.Day else "1 Hour" + predicate = _historical_prices_predicate(resolution) + + logger.info(f"step: goto interactive-chart page for {contract_id}") + async with human.page.expect_response( + predicate, timeout=20000 + ) as response_info: + await human.page.goto(chart_url) + await _pause(human) + + logger.info("step: click Max") + await human.page.get_by_role("button", name="Max").click() + await _pause(human) + + logger.info(f"step: switch chart resolution to {resolution_label}") + await human.page.locator("text-binding").nth(1).click() + await _pause(human) + await ( + human.page.locator("text-binding") + .filter(has_text=re.compile(rf"^{re.escape(resolution_label)}$")) + .click() ) - ratelimit = prices_resp.headers["x-ratelimit-remaining"] - if int(ratelimit) <= 15: - time.sleep(20) + response = await response_info.value + + ratelimit = response.headers.get("x-ratelimit-remaining") + if ratelimit is not None and int(ratelimit) <= 15: + await asyncio.sleep(20) logger.info( - f"GET {data_url} {instr_code}, {prices_resp.status_code}, " + f"GET {response.url} {contract_id}, {response.status}, " f"ratelimit {ratelimit}" ) # read response into dataframe - iostr = io.StringIO(prices_resp.text) + text = await response.text() + iostr = io.StringIO(text) df = pd.read_csv(iostr, header=None) # convert to expected format price_data_as_df = _raw_barchart_data_to_df(df, bar_freq=resolution) if len(df) == 0: - raise BCException(f"Zero length Barchart price data found for {instr_code}") + raise BCException( + f"Zero length Barchart price data found for {contract_id}" + ) logger.debug(f"Latest price {df.index[-1]} with {resolution}") @@ -617,6 +984,19 @@ def get_historical_prices_for_contract( raise BCException from ex +async def _insufficient_data_async( + human: Humanization, contract: str, resolution: Resolution +) -> bool: + try: + df = await _get_historical_prices_for_contract_async( + human, contract, resolution + ) + logger.info(f"step: got {len(df)} rows for '{contract}'") + return len(df) < 30 + except Exception: # skipcq broad by design + return True + + def _build_contract_list(start_year, end_year, instr_list=None, contract_map=None): contracts_per_instrument = {} contract_list = [] @@ -709,19 +1089,17 @@ def _before_available_res(resolution, start_date, instr_config): raise BCException(f"No exchange specified for {instr_config['code']}") -def _get_overview(session, contract_id): - """ - GET the futures overview page, e.g. - https://www.barchart.com/futures/quotes/B6M21/overview - :param contract_id: contract identifier - :type contract_id: str - :return: resp - :rtype: HTTP response object - """ - url = BARCHART_URL + "futures/quotes/%s/overview" % contract_id - resp = session.get(url) - logger.debug(f"GET {url}, response {resp.status_code}") - return resp +def _near_available_res_boundary(resolution, start_date, instr_config, margin_days): + # Barchart's published limits are often inaccurate. + if "exchange" not in instr_config: + raise BCException(f"No exchange specified for {instr_config['code']}") + exch = instr_config["exchange"] + if exch not in EXCHANGES: + raise BCException(f"Missing exchange config for {exch}") + exch_config = EXCHANGES[exch] + limit_key = "tick_date" if resolution == Resolution.Hour else "eod_date" + limit_date = datetime.strptime(exch_config[limit_key], "%Y-%m-%d") + return start_date <= limit_date + timedelta(days=margin_days) def _build_save_path(instr_code, month, year, res: Resolution, save_directory): @@ -746,14 +1124,6 @@ def _get_contract_month_year(contract): return month, year -def _insufficient_data(session, symbol: str, res: Resolution): - try: - df = get_historical_prices_for_contract(session, symbol, res) - return len(df) < 30 - except Exception: # skipcq broad by design - return True - - def _get_start_end_dates(month, year, instr_config=None, default_day_count: int = 400): now = datetime.now() if instr_config and "days_count" in instr_config: @@ -860,7 +1230,7 @@ def _contract_date_from_file_name(file_name): return contract_date -def _filename_from_barchart_id(contract_id, inv_map, res: Resolution): +def _filename_from_barchart_id(contract_id, inv_map, res: Resolution | None): try: month, year = _get_contract_month_year(contract_id) market_code = contract_id[: len(contract_id) - 3] @@ -886,24 +1256,22 @@ def _env(): return barchart_config -def _get_exchange_for_code(session, contract_code: str): - """ - Get the exchange for the given Barchart code - - Scrapes the info page for the given contract to grab the exchange - :param contract_code: - :return: str - """ +async def _get_exchange_for_code_async(human: Humanization, contract_code: str): + # scrape the overview page info table for the exchange name try: - resp = _get_overview(session, contract_code) - if resp.status_code == 200: - soup = BeautifulSoup(resp.text, "html.parser") + url = f"{BARCHART_URL}futures/quotes/{contract_code}/overview" + response = await human.page.goto(url) + if response is None: + return None + if response.status == 200: + html = await response.text() + soup = BeautifulSoup(html, "html.parser") table = soup.find(name="div", attrs={"class": "commodity-profile"}) - label = table.find(name="div", string="Exchange") - exchange_raw = label.next_sibling.next_sibling # whitespace counts - exchange = exchange_raw.text.strip() + label = table.find(name="div", string="Exchange") # type: ignore[union-attr] + exchange_raw = label.next_sibling.next_sibling # type: ignore[union-attr] + exchange = exchange_raw.text.strip() # type: ignore[union-attr] return exchange - if resp.status_code == 404: + if response.status == 404: print(f"Barchart page for {contract_code} not found") except Exception as e: @@ -911,6 +1279,40 @@ def _get_exchange_for_code(session, contract_code: str): return None +async def _get_exchange_for_code_standalone_async( + session: BarchartSession, + contract_code: str, + headless: bool = False, + auth_dir: str | None = None, +): + auth_dir_path = Path(auth_dir) if auth_dir else _DEFAULT_AUTH_DIR + downloads_dir = _DEFAULT_AUTH_DIR.parent / "downloads" + + async with async_playwright() as playwright: + context, human = await _launch_barchart_browser( + playwright, auth_dir_path, downloads_dir, headless + ) + try: + await _login_async(human, session.username, session.password) + return await _get_exchange_for_code_async(human, contract_code) + finally: + await context.close() + + +def _get_exchange_for_code( + session: BarchartSession, + contract_code: str, + headless: bool = False, + auth_dir: str | None = None, +): + """Get the exchange for the given Barchart code.""" + return asyncio.run( + _get_exchange_for_code_standalone_async( + session, contract_code, headless, auth_dir + ) + ) + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) get_barchart_downloads( @@ -922,13 +1324,3 @@ def _get_exchange_for_code(session, contract_code: str): do_daily=True, dry_run=False, ) - - # update_barchart_downloads( - # instr_code="FANG", - # contract_map={ - # "FANG": {"code": "FG", "cycle": "HMUZ", "exchange": "ICE/US"}, - # }, - # save_dir="/home/user/barchart_data", - # dry_run=False, - # days_ago=360, - # ) diff --git a/bcutils/tests/test_downloader.py b/bcutils/tests/test_downloader.py index 528f802f..3193fc10 100644 --- a/bcutils/tests/test_downloader.py +++ b/bcutils/tests/test_downloader.py @@ -10,6 +10,7 @@ _build_save_path, _get_contract_month_year, _get_start_end_dates, + _get_exchange_for_code, ) @@ -38,39 +39,20 @@ def test_no_credentials(self, download_dir): contract_map={ "AUD": {"code": "A6", "cycle": "HMUZ", "exchange": "CME"} }, - save_dir=download_dir, - start_year=2020, - end_year=2022, - dry_run=False, - ) - - def test_bad_credentials(self, download_dir): - with pytest.raises(Exception): - get_barchart_downloads( - create_bc_session( - config_obj=dict( - barchart_username="user@domain.com", - barchart_password="s3cr3t_321", - ) - ), - contract_map={ - "AUD": {"code": "A6", "cycle": "HMUZ", "exchange": "CME"} - }, - save_dir=download_dir, + save_dir=str(download_dir.absolute()), start_year=2020, end_year=2022, dry_run=False, ) def test_hourly(self, bc_config, download_dir): - print(bc_config.keys()) - if len(bc_config) == 0: + if not self._have_creds(bc_config): pytest.skip("Skipping test, no Barchart credentials found in env") else: get_barchart_downloads( create_bc_session(config_obj=bc_config), contract_map={"AUD": {"code": "A6", "cycle": "H", "exchange": "CME"}}, - save_dir=download_dir, + save_dir=str(download_dir), start_year=2020, end_year=2021, dry_run=False, @@ -88,7 +70,7 @@ def test_daily(self, bc_config, download_dir): get_barchart_downloads( create_bc_session(config_obj=bc_config), contract_map={"AUD": {"code": "A6", "cycle": "H", "exchange": "CME"}}, - save_dir=download_dir, + save_dir=str(download_dir), start_year=2020, end_year=2021, do_daily=True, @@ -112,15 +94,24 @@ def test_insufficient(self, bc_config, download_dir): start_date, end_date = _get_start_end_dates(month, year) result = save_prices_for_contract( - create_bc_session(config_obj=bc_config, do_login=False), - contract_key, - save_path, - start_date, - end_date, + session=create_bc_session(config_obj=bc_config), + contract=contract_key, + save_path=save_path, + start_date=start_date, + end_date=end_date, ) assert result == HistoricalDataResult.INSUFFICIENT + def test_get_exchange(self, bc_config): + if not self._have_creds(bc_config): + pytest.skip("Skipping test, no Barchart credentials found in env") + else: + exch = _get_exchange_for_code( + create_bc_session(config_obj=bc_config), "GCF24" + ) + assert exch == "COMEX" + @staticmethod def _have_creds(config: dict): return ( diff --git a/bcutils/tests/test_utils.py b/bcutils/tests/test_utils.py index d878f829..a53a777e 100644 --- a/bcutils/tests/test_utils.py +++ b/bcutils/tests/test_utils.py @@ -4,16 +4,12 @@ from bcutils.bc_utils import ( Resolution, CONTRACT_MAP, - create_bc_session, BCException, _build_save_path, - _env, _get_resolution, _filename_from_barchart_id, _build_inverse_map, - _insufficient_data, _before_available_res, - _get_exchange_for_code, _instr_code_from_file_name, _res_from_file_name, ) @@ -66,13 +62,6 @@ def test_hour_save_path(self): path = _build_save_path("AUD", 3, 2024, Resolution.Hour, "/home/user/data") assert path == "/home/user/data/Hour_AUD_20240300.csv" - def test_insufficient_data(self): - assert _insufficient_data( - create_bc_session(config_obj=_env(), do_login=False), - "TGF08", - Resolution.Day, - ) - def test_before_available_res(self): assert _before_available_res( Resolution.Day, datetime(1975, 1, 1), CONTRACT_MAP["AUD"] @@ -104,12 +93,6 @@ def test_before_available_res(self): {"code": "ABC", "cycle": "HMUZ"}, ) - def test_get_exchange(self): - exch = _get_exchange_for_code( - create_bc_session(config_obj=_env(), do_login=False), "GCF24" - ) - assert exch == "COMEX" - def test_get_instr_code_from_file_name_split(self): instr_code = _instr_code_from_file_name("Day_EURIBOR-ICE_20240900") assert instr_code == "EURIBOR-ICE" diff --git a/poetry.toml b/poetry.toml deleted file mode 100644 index fcb6a991..00000000 --- a/poetry.toml +++ /dev/null @@ -1,3 +0,0 @@ -[virtualenvs] -prefer-active-python = true -create = false diff --git a/pyproject.toml b/pyproject.toml index bd38481e..f78f7935 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,13 @@ -[tool.poetry] +[project] name = "bc-utils" version = "0.1.7" description = "Python utility automation scripts for Barchart.com" -authors = ["Andy Geach "] +authors = [ + { name = "Andy Geach", email = "andy@bugorfeature.net" } +] license = "BSD-3-Clause" readme = "README.md" -homepage = "https://github.com/bug-or-feature/bc-utils" -repository = "https://github.com/bug-or-feature/bc-utils" +requires-python = ">=3.11,<4.0" keywords = ["trading"] classifiers = [ "Development Status :: 3 - Alpha", @@ -16,39 +17,47 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Cython", "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries", "License :: OSI Approved :: BSD License", ] -packages = [ - { include = "bcutils" } +dependencies = [ + "numpy<2", + "pandas<3", + "beautifulsoup4>=4,<5", + "pyyaml>=6.0.1,<7", + "playwright>=1.62.0", + "humanization-playwright>=0.1.2", ] -[tool.poetry.dependencies] -python = ">=3.8.1,<4.0" -requests = "^2" -numpy = "<2" -pandas = "<3" -beautifulsoup4 = "^4" -pyyaml = "^6.0.1" - -[tool.poetry.group.dev.dependencies] -black = "^23.12.1" -pytest = "^7" -flake8 = "^6" +[project.urls] +Homepage = "https://github.com/bug-or-feature/bc-utils" +Repository = "https://github.com/bug-or-feature/bc-utils" +Issues = "https://github.com/bug-or-feature/bc-utils/issues" -[tool.poetry.urls] -"Issues" = "https://github.com/bug-or-feature/bc-utils/issues" +[dependency-groups] +dev = [ + "mypy>=2.3.1", + "pytest>=9,<10", + "ruff>=0.8,<0.9", +] [build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" -[tool.black] +[tool.hatch.build.targets.wheel] +packages = ["bcutils"] + +[tool.ruff] line-length = 88 -target-version = ['py310'] -required-version = '23.12.1' +target-version = "py311" + +[tool.ruff.lint] +extend-ignore = ["E203"] + +[tool.mypy] +python_version = "3.11" +ignore_missing_imports = true +no_implicit_optional = false diff --git a/sample/exchanges.py b/sample/exchanges.py index 21eff1e2..0ae76109 100644 --- a/sample/exchanges.py +++ b/sample/exchanges.py @@ -1,38 +1,71 @@ +import asyncio import json from copy import copy + +from patchright.async_api import async_playwright + from bcutils.config import CONTRACT_MAP -from bcutils.bc_utils import create_bc_session, _get_exchange_for_code +from bcutils.bc_utils import ( + create_bc_session, + _get_exchange_for_code_async, + _launch_barchart_browser, + _login_async, + _pause, + _DEFAULT_AUTH_DIR, + _env, +) -def exchanges(): - contract_map = CONTRACT_MAP - session = create_bc_session({}, do_login=False) +async def _exchanges_async(contract_map: dict | None = None): + resolved_map: dict = CONTRACT_MAP if contract_map is None else contract_map + session = create_bc_session(config_obj=_env()) exchange_map = {} + downloads_dir = _DEFAULT_AUTH_DIR.parent / "downloads" + + async with async_playwright() as playwright: + context, human = await _launch_barchart_browser( + playwright, _DEFAULT_AUTH_DIR, downloads_dir, False + ) + try: + await _login_async(human, session.username, session.password) - for instr in contract_map.keys(): - config_obj = contract_map[instr] - futures_code = config_obj["code"] - rollcycle = config_obj["cycle"] + for instr in resolved_map.keys(): + config_obj: dict = resolved_map[instr] + futures_code = config_obj["code"] + rollcycle = config_obj["cycle"] - contract_key = f"{futures_code}{rollcycle[:1]}24" - print(contract_key) - exchange = _get_exchange_for_code(session, contract_key) - print(f"Exchange for {instr} ({futures_code}): {exchange}") + contract_key = f"{futures_code}{rollcycle[:1]}24" + print(contract_key) + exchange = await _get_exchange_for_code_async(human, contract_key) + print(f"Exchange for {instr} ({futures_code}): {exchange}") - updated_config = copy(config_obj) - updated_config["exchange"] = exchange - if "tick_date" in updated_config: - del updated_config["tick_date"] - contract_map[instr] = updated_config + updated_config = copy(config_obj) + updated_config["exchange"] = exchange + if "tick_date" in updated_config: + del updated_config["tick_date"] + resolved_map[instr] = updated_config - if exchange not in exchange_map: - exchange_map[exchange] = { - "tick_date": "2000-01-01", - "eod_date": "1990-01-01", - } + if exchange not in exchange_map: + exchange_map[exchange] = { + "tick_date": "2000-01-01", + "eod_date": "1990-01-01", + } + + await _pause(human, 1, 3) + finally: + await context.close() print(json.dumps(exchange_map)) +def exchanges(contract_map: dict | None = None): + asyncio.run(_exchanges_async(contract_map)) + + if __name__ == "__main__": - exchanges() + exchanges( + { + "FTSE100": {"code": "X", "cycle": "HMUZ"}, + "AUD": {"code": "A6", "cycle": "HMUZ"}, + } + ) diff --git a/sample/pst.py b/sample/pst.py index 519a0e62..c0d7191e 100644 --- a/sample/pst.py +++ b/sample/pst.py @@ -15,7 +15,7 @@ def download_with_config(): # See /sample/private_config_sample.yaml config = load_config("./private_config.yaml") get_barchart_downloads( - create_bc_session(config), + session=create_bc_session(config), instr_list=config["barchart_download_list"], start_year=config["barchart_start_year"], end_year=config["barchart_end_year"], @@ -33,7 +33,12 @@ def update_with_config(): save_dir = config["barchart_path"] dry_run = config["barchart_dry_run"] for code in instr_list: - update_barchart_downloads(instr_code=code, save_dir=save_dir, dry_run=dry_run) + update_barchart_downloads( + session=create_bc_session(config), + instr_code=code, + save_dir=save_dir, + dry_run=dry_run, + ) def load_config(config_path): diff --git a/sample/samples.py b/sample/samples.py index a6582245..ca5374ec 100644 --- a/sample/samples.py +++ b/sample/samples.py @@ -11,11 +11,9 @@ _build_save_path, _get_contract_month_year, _get_start_end_dates, - _build_inverse_map, _env, ) from bcutils.migrate import migrate_to_multi_freq -from bcutils.config import CONTRACT_MAP logging.basicConfig(level=logging.INFO) @@ -106,8 +104,9 @@ def update_downloads(): update_barchart_downloads( instr_code="AUD", contract_map={"AUD": {"code": "A6", "cycle": "HMUZ", "exchange": "CME"}}, - save_dir=os.getcwd(), + save_dir="/Users/ageach/Dev/work/bc-utils/dl_test", dry_run=False, + days_ago=100000, ) @@ -115,23 +114,23 @@ def update_hourly_file(): # update the hourly AUD Mar 2024 price file in the current working directory contract_map = {"AUD": {"code": "A6", "cycle": "HMUZ", "exchange": "CME"}} update_barchart_contract_file( - create_bc_session(config_obj=_env()), - _build_inverse_map(contract_map), - os.getcwd(), - "A6H24", - Resolution.Hour, + session=create_bc_session(config_obj=_env()), + contract_map=contract_map, + path=os.getcwd(), + contract_id="A6H24", + res=Resolution.Hour, ) -def update_daily_file(contract_key, config=None): - if config is None: - config = CONTRACT_MAP +def update_daily_file(): + # update the daily AUD Mar 2024 price file in the current working directory + contract_map = {"AUD": {"code": "A6", "cycle": "HMUZ", "exchange": "CME"}} update_barchart_contract_file( - create_bc_session(config_obj=_env()), - config, - os.getcwd(), - contract_key, - Resolution.Day, + session=create_bc_session(config_obj=_env()), + contract_map=contract_map, + path=os.getcwd(), + contract_id="A6H24", + res=Resolution.Day, ) @@ -149,10 +148,11 @@ def rename_files_with_new_format(): if __name__ == "__main__": download_hourly() # download_hourly_and_daily() + # download_specific_contracts() # save_hourly("AUD", "A6H20") # save_daily("AUD", "A6H20") # save_hourly("CHFJPY", "UPU14") # update_downloads() # update_hourly_file() - # update_daily_file("EPM24") + # update_daily_file() # rename_files_with_new_format()