public method

src/pyrecount/accessor.py

@@ -78,7 +78,7 @@ def __post_init__(self):
             root_url=self.root_url, organism=self.organism
         )
 
-    def _get_project_urls(self, dtype) -> List[str]:
+    def get_project_urls(self, dtype) -> List[str]:
         project = ProjectLocator(
             root_organism_url=self.endpoints.root_organism_url,
             data_sources=self.endpoints.data_sources,
@@ -99,7 +99,7 @@ async def cache(self) -> None:
             if dtype not in CACHEABLE_DTYPES:
                 continue
 
-            for url in self._get_project_urls(dtype):
+            for url in self.get_project_urls(dtype):
                 fpath = urlparse(url).path.lstrip("/")
                 if path.exists(fpath):
                     continue
@@ -139,7 +139,7 @@ def _valid_metadata_url(self, url: str, project_id: str) -> bool:
     def _metadata_load(self) -> pl.DataFrame:
         cache_meta: list[pl.DataFrame] = []
         join_cols = ["rail_id", "external_id", "study"]
-        urls = self._get_project_urls(Dtype.METADATA)
+        urls = self.get_project_urls(Dtype.METADATA)
 
         for project_id in self.project_ids:
             dfs_for_project: list[pl.DataFrame] = []
@@ -210,7 +210,7 @@ def _jxn_load(self) -> Tuple[pl.DataFrame, pl.DataFrame]:
         cache_mm: List[pl.DataFrame] = []
         cache_meta: List[pl.DataFrame] = []
 
-        urls = self._get_project_urls(Dtype.JXN)
+        urls = self.get_project_urls(Dtype.JXN)
 
         for project_id in self.project_ids:
             project_urls = [
@@ -261,7 +261,7 @@ def _jxn_load(self) -> Tuple[pl.DataFrame, pl.DataFrame]:
         )
 
     def _bw_load(self) -> pl.DataFrame:
-        urls = self._get_project_urls(Dtype.BW)
+        urls = self.get_project_urls(Dtype.BW)
 
         for project_id in self.project_ids:
             project_urls = [
@@ -328,7 +328,7 @@ def _read_counts(self, rname: str):
         return counts_dataframe
 
     def _gene_load(self) -> pl.DataFrame:
-        for url in self._get_project_urls(Dtype.GENE):
+        for url in self.get_project_urls(Dtype.GENE):
             fpath = urlparse(url).path.lstrip("/")
             if self.annotation.value in fpath:
                 if any(fpath.endswith(ext) for ext in Extensions.GENE.value):
@@ -338,7 +338,7 @@ def _gene_load(self) -> pl.DataFrame:
         return annotation, counts
 
     def _exon_load(self) -> pl.DataFrame:
-        for url in self._get_project_urls(Dtype.EXON):
+        for url in self.get_project_urls(Dtype.EXON):
             fpath = urlparse(url).path.lstrip("/")
             if self.annotation.value in fpath:
                 if any(url.endswith(ext) for ext in Extensions.EXON.value):

src/pyrecount/api.py

@@ -1,4 +1,5 @@
 #! /usr/bin/env python3
+import time
 import logging
 from os import path
 from typing import Optional, Dict
@@ -16,28 +17,40 @@ class EndpointConnector:
     root_url: str = "http://duffel.rail.bio/recount3"
 
     root_organism_url: str = field(init=False)
-    data_sources: Dict[str, str] = field(init=False)
+    data_sources: Dict[str, str] = field(init=False, default_factory=dict)
 
     def __post_init__(self):
         self.root_organism_url = path.join(self.root_url, self.organism)
 
         index = path.join(self.root_organism_url, HOMES_INDEX)
         resp = self._validate_endpoint(endpoint=index)
+
         if resp:
             self._set_data_sources(resp)
 
     def _set_data_sources(self, resp: Response):
-        self.data_sources: Dict[str, str] = {
+        self.data_sources = {
             path.basename(dsource): dsource
             for dsource in resp.text.strip().splitlines()
+            if dsource.strip()
         }
 
     def _validate_endpoint(self, endpoint: str) -> Optional[Response]:
-        log.info(f"Validating endpoint {endpoint}.")
-        try:
-            resp = get(endpoint, timeout=10)
-            resp.raise_for_status()
-            return resp
-        except RequestException as e:
-            log.error(f"Error while validating endpoint {endpoint}: {e}")
-            return None
+        attempts = 3
+        backoff = 2
+
+        for attempt in range(1, attempts + 1):
+            try:
+                # separate connect/read timeout improves reliability
+                resp = get(endpoint, timeout=(5, 30))
+                resp.raise_for_status()
+                return resp
+
+            except RequestException as e:
+                log.warning(f"Attempt {attempt}/{attempts} failed for {endpoint}: {e}")
+
+                if attempt < attempts:
+                    time.sleep(backoff**attempt)
+                else:
+                    log.error(f"All retries failed for {endpoint}")
+                    return None

tests/test_accessor.py

@@ -6,6 +6,7 @@
 from pyrecount.models import Dtype, Annotation
 
 # TODO: transform raw counts
+# TODO: materialize requested columns: e.g., async polars for study id column
 # TODO: multi-project support for exon, gene dtypes
 # TODO: expand sra attributes
 # TODO: expose Lazyframes
@@ -259,7 +260,6 @@ async def test_project_gene_accessor(
     assert gene_counts.shape == expected_counts_shape
 
 
-@pytest.mark.asyncio
 @pytest.mark.parametrize(
     "organism, dbase, project_ids, annotation, expected_shape",
     [