diff --git a/src/pyrecount/accessor.py b/src/pyrecount/accessor.py index e539b5f..1aa0da1 100644 --- a/src/pyrecount/accessor.py +++ b/src/pyrecount/accessor.py @@ -78,7 +78,7 @@ def __post_init__(self): root_url=self.root_url, organism=self.organism ) - def _get_project_urls(self, dtype) -> List[str]: + def get_project_urls(self, dtype) -> List[str]: project = ProjectLocator( root_organism_url=self.endpoints.root_organism_url, data_sources=self.endpoints.data_sources, @@ -99,7 +99,7 @@ async def cache(self) -> None: if dtype not in CACHEABLE_DTYPES: continue - for url in self._get_project_urls(dtype): + for url in self.get_project_urls(dtype): fpath = urlparse(url).path.lstrip("/") if path.exists(fpath): continue @@ -139,7 +139,7 @@ def _valid_metadata_url(self, url: str, project_id: str) -> bool: def _metadata_load(self) -> pl.DataFrame: cache_meta: list[pl.DataFrame] = [] join_cols = ["rail_id", "external_id", "study"] - urls = self._get_project_urls(Dtype.METADATA) + urls = self.get_project_urls(Dtype.METADATA) for project_id in self.project_ids: dfs_for_project: list[pl.DataFrame] = [] @@ -210,7 +210,7 @@ def _jxn_load(self) -> Tuple[pl.DataFrame, pl.DataFrame]: cache_mm: List[pl.DataFrame] = [] cache_meta: List[pl.DataFrame] = [] - urls = self._get_project_urls(Dtype.JXN) + urls = self.get_project_urls(Dtype.JXN) for project_id in self.project_ids: project_urls = [ @@ -261,7 +261,7 @@ def _jxn_load(self) -> Tuple[pl.DataFrame, pl.DataFrame]: ) def _bw_load(self) -> pl.DataFrame: - urls = self._get_project_urls(Dtype.BW) + urls = self.get_project_urls(Dtype.BW) for project_id in self.project_ids: project_urls = [ @@ -328,7 +328,7 @@ def _read_counts(self, rname: str): return counts_dataframe def _gene_load(self) -> pl.DataFrame: - for url in self._get_project_urls(Dtype.GENE): + for url in self.get_project_urls(Dtype.GENE): fpath = urlparse(url).path.lstrip("/") if self.annotation.value in fpath: if any(fpath.endswith(ext) for ext in Extensions.GENE.value): @@ -338,7 +338,7 @@ def _gene_load(self) -> pl.DataFrame: return annotation, counts def _exon_load(self) -> pl.DataFrame: - for url in self._get_project_urls(Dtype.EXON): + for url in self.get_project_urls(Dtype.EXON): fpath = urlparse(url).path.lstrip("/") if self.annotation.value in fpath: if any(url.endswith(ext) for ext in Extensions.EXON.value): diff --git a/src/pyrecount/api.py b/src/pyrecount/api.py index 4a89635..4a98d34 100644 --- a/src/pyrecount/api.py +++ b/src/pyrecount/api.py @@ -1,4 +1,5 @@ #! /usr/bin/env python3 +import time import logging from os import path from typing import Optional, Dict @@ -16,28 +17,40 @@ class EndpointConnector: root_url: str = "http://duffel.rail.bio/recount3" root_organism_url: str = field(init=False) - data_sources: Dict[str, str] = field(init=False) + data_sources: Dict[str, str] = field(init=False, default_factory=dict) def __post_init__(self): self.root_organism_url = path.join(self.root_url, self.organism) index = path.join(self.root_organism_url, HOMES_INDEX) resp = self._validate_endpoint(endpoint=index) + if resp: self._set_data_sources(resp) def _set_data_sources(self, resp: Response): - self.data_sources: Dict[str, str] = { + self.data_sources = { path.basename(dsource): dsource for dsource in resp.text.strip().splitlines() + if dsource.strip() } def _validate_endpoint(self, endpoint: str) -> Optional[Response]: - log.info(f"Validating endpoint {endpoint}.") - try: - resp = get(endpoint, timeout=10) - resp.raise_for_status() - return resp - except RequestException as e: - log.error(f"Error while validating endpoint {endpoint}: {e}") - return None + attempts = 3 + backoff = 2 + + for attempt in range(1, attempts + 1): + try: + # separate connect/read timeout improves reliability + resp = get(endpoint, timeout=(5, 30)) + resp.raise_for_status() + return resp + + except RequestException as e: + log.warning(f"Attempt {attempt}/{attempts} failed for {endpoint}: {e}") + + if attempt < attempts: + time.sleep(backoff**attempt) + else: + log.error(f"All retries failed for {endpoint}") + return None diff --git a/tests/test_accessor.py b/tests/test_accessor.py index 89ed18b..4076546 100644 --- a/tests/test_accessor.py +++ b/tests/test_accessor.py @@ -6,6 +6,7 @@ from pyrecount.models import Dtype, Annotation # TODO: transform raw counts +# TODO: materialize requested columns: e.g., async polars for study id column # TODO: multi-project support for exon, gene dtypes # TODO: expand sra attributes # TODO: expose Lazyframes @@ -259,7 +260,6 @@ async def test_project_gene_accessor( assert gene_counts.shape == expected_counts_shape -@pytest.mark.asyncio @pytest.mark.parametrize( "organism, dbase, project_ids, annotation, expected_shape", [