From 0f5400b7d1fc4d4b911cbbfc1f530e9c75d7dffe Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 2 Dec 2025 15:38:25 -0500 Subject: [PATCH 001/123] This initializes a uv package in this repository. --- .python-version | 1 + main.py | 6 ++++++ pyproject.toml | 7 +++++++ 3 files changed, 14 insertions(+) create mode 100644 .python-version create mode 100644 main.py create mode 100644 pyproject.toml diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..2c07333 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/main.py b/main.py new file mode 100644 index 0000000..55226a0 --- /dev/null +++ b/main.py @@ -0,0 +1,6 @@ +def main(): + print("Hello from babel-xrefs!") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..826dbc8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "babel-xrefs" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [] From d3ffc979715ecb5b6896ac47e59dc03599d952c6 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 2 Dec 2025 15:42:12 -0500 Subject: [PATCH 002/123] Added basic CLI. --- main.py | 6 ------ pyproject.toml | 17 ++++++++++++++++- src/babel_xrefs/__init__.py | 0 src/babel_xrefs/cli.py | 9 +++++++++ 4 files changed, 25 insertions(+), 7 deletions(-) delete mode 100644 main.py create mode 100644 src/babel_xrefs/__init__.py create mode 100644 src/babel_xrefs/cli.py diff --git a/main.py b/main.py deleted file mode 100644 index 55226a0..0000000 --- a/main.py +++ /dev/null @@ -1,6 +0,0 @@ -def main(): - print("Hello from babel-xrefs!") - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 826dbc8..7c84773 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,4 +4,19 @@ version = "0.1.0" description = "Add your description here" readme = "README.md" requires-python = ">=3.11" -dependencies = [] +dependencies = [ + "click>=8.3.1", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.3.5", + "ruff>=0.11.0", +] + +[project.scripts] +babel-xrefs = "babel_xrefs.cli:main" diff --git a/src/babel_xrefs/__init__.py b/src/babel_xrefs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/babel_xrefs/cli.py b/src/babel_xrefs/cli.py new file mode 100644 index 0000000..537fe60 --- /dev/null +++ b/src/babel_xrefs/cli.py @@ -0,0 +1,9 @@ +# Command line interface for babel-xrefs +import click + +@click.command() +def main(): + pass + +if __name__ == "__main__": + main() From c2ff184ed064a39f8fbe826de720b62490fd01a7 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 2 Dec 2025 16:09:07 -0500 Subject: [PATCH 003/123] Add /data to the .gitignore. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b7faf40..67d8b31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Ignore data files. +/data + # Byte-compiled / optimized / DLL files __pycache__/ *.py[codz] From e238b8b201fb32b5e2d171e97ff81c4e1d0f6783 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 3 Dec 2025 02:27:10 -0500 Subject: [PATCH 004/123] Initial implementation of a basic xref query-er. --- pyproject.toml | 4 +- src/__init__.py | 0 src/babel_xrefs/babel_xrefs.py | 33 ++++++++++ src/babel_xrefs/cli.py | 38 +++++++++++- src/babel_xrefs/core/__init__.py | 0 src/babel_xrefs/core/downloader.py | 98 ++++++++++++++++++++++++++++++ src/babel_xrefs/core/model.py | 2 + 7 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 src/__init__.py create mode 100644 src/babel_xrefs/babel_xrefs.py create mode 100644 src/babel_xrefs/core/__init__.py create mode 100644 src/babel_xrefs/core/downloader.py create mode 100644 src/babel_xrefs/core/model.py diff --git a/pyproject.toml b/pyproject.toml index 7c84773..5696f67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,8 @@ readme = "README.md" requires-python = ">=3.11" dependencies = [ "click>=8.3.1", + "duckdb>=1.4.2", + "requests>=2.32.5", ] [build-system] @@ -19,4 +21,4 @@ dev = [ ] [project.scripts] -babel-xrefs = "babel_xrefs.cli:main" +babel-xrefs = "babel_xrefs.cli:cli" diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/babel_xrefs/babel_xrefs.py b/src/babel_xrefs/babel_xrefs.py new file mode 100644 index 0000000..3ec44ad --- /dev/null +++ b/src/babel_xrefs/babel_xrefs.py @@ -0,0 +1,33 @@ +# Babel XRefs is a tool for accessing and querying the intermediate files +# that we make available with Babel builds. This allows you to find out +# why we consider two identifiers to be identical. +import logging +import duckdb + +from babel_xrefs.core.downloader import BabelDownloader + + +class BabelXRefs: + def __init__(self, downloader: BabelDownloader): + self.downloader = downloader + + def get_curie_xrefs(self, curies: list[str]): + """ + Search for all identifiers that are cross-referenced to the given CURIE. + + :param curie: A CURIE to search for. + :return: A list of cross-references containing that CURIE. + """ + + concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') + concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/ConcordMetadata.parquet') + + # Query the Parquet files using DuckDB. + duckdb_path = self.downloader.get_output_file('output/duckdbs/xrefs.duckdb') + db = duckdb.connect(duckdb_path) + concord_table = db.read_parquet(concord_parquet) + xrefs = db.execute(f"SELECT * FROM concord_table WHERE subj IN $1 OR obj in $1", [curies]) + + # TODO: convert into case classes. + + return xrefs.fetchall() diff --git a/src/babel_xrefs/cli.py b/src/babel_xrefs/cli.py index 537fe60..2b6c96f 100644 --- a/src/babel_xrefs/cli.py +++ b/src/babel_xrefs/cli.py @@ -1,9 +1,41 @@ # Command line interface for babel-xrefs import click +import logging +import babel_xrefs +from babel_xrefs.core.downloader import BabelDownloader +from babel_xrefs.babel_xrefs import BabelXRefs -@click.command() -def main(): + +@click.group() +def cli(): pass +@cli.command("xrefs") +@click.argument("curies", type=str, required=True, nargs=-1) +@click.option("--local-dir", type=str, default="data/2025nov19", help="Local location to save Babel download files to") +@click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") +def xrefs(curies: list[str], babel_url: str, local_dir: str): + """ + Fetches and prints the cross-references (xrefs) for the given CURIEs. + + This function searches for xrefs associated with the provided CURIEs. + + \f + + :param curies: A list of CURIEs (Compact URI) for which cross-references need + to be retrieved. + :type curies: list[str] + :param babel_url: Base URL of the Babel server + :type babel_url: str + + :return: None + """ + logging.basicConfig(level=logging.INFO) + + bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir)) + xrefs = bxref.get_curie_xrefs(curies) + for xref in xrefs: + print(xref) + if __name__ == "__main__": - main() + cli() diff --git a/src/babel_xrefs/core/__init__.py b/src/babel_xrefs/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/babel_xrefs/core/downloader.py b/src/babel_xrefs/core/downloader.py new file mode 100644 index 0000000..34e4cb0 --- /dev/null +++ b/src/babel_xrefs/core/downloader.py @@ -0,0 +1,98 @@ +import os +import urllib.parse +import subprocess +import requests +import logging + +class BabelDownloader: + """ + Class for downloading Babel cross-reference files to a local directory as needed. + """ + + def __init__(self, url_base, local_path=None, retries=10): + # We assume the URL base is correct (if not, we can fix it later). + self.url_base = url_base + self.retries = retries + self.logger = logging.getLogger(BabelDownloader.__name__) + + if local_path is None: + # Default to using TMPDIR. + # TODO: replace with a real temporary directory. + tmpdir = os.environ.get("TMPDIR") + if tmpdir: + local_path = tmpdir + + # Make sure the local path is an existing directory or that we can create it. + if not os.path.exists(local_path): + os.makedirs(local_path, exist_ok=True) + self.local_path = local_path + elif os.path.exists(local_path) and os.path.isdir(local_path): + self.local_path = local_path + else: + raise ValueError(f"Invalid local_path (must be an existing directory): '{local_path}'") + + def get_output_file(self, filename): + filepath = os.path.join(self.local_path, filename) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + return filepath + + def get_downloaded_file(self, dirpath: str, chunk_size:int=1024*1024): + local_path_to_download_to = os.path.join(self.local_path, dirpath) + os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) + + url_to_download = urllib.parse.urljoin(self.url_base, dirpath) + bytes_downloaded = 0 + + wget_command_line = [ + "wget", + "--progress=bar:force:noscroll", # Display progress bar. + "--compression=auto", # Compress files if available. + "--continue", # Continue downloading in case of interruption. + f"--tries={self.retries}", + "-O" + local_path_to_download_to, + ] + + # Add URL and output file. + wget_command_line.append(url_to_download) + + # Execute wget. + self.logger.info(f"Downloading {url_to_download} using wget: {wget_command_line}") + process = subprocess.run(wget_command_line) + if process.returncode != 0: + raise RuntimeError(f"Could not execute wget {wget_command_line}: {process.stderr}") + + bytes_downloaded = os.path.getsize(local_path_to_download_to) + self.logger.info(f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes") + return local_path_to_download_to + + + def get_downloaded_dir(self, dirpath: str): + local_path_to_download_to = os.path.join(self.local_path, dirpath) + os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) + + url_to_download_recursively = urllib.parse.urljoin(self.url_base, dirpath) + + wget_command_line = [ + "wget", + "--progress=bar:force:noscroll", # Display progress bar. + "--compression=auto", # Compress files if available. + "--continue", # Continue downloading in case of interruption. + f"--tries={self.retries}", + "--recursive", + "--no-parent", + "--no-host-directories", + "--directory-prefix=" + local_path_to_download_to, + ] + + # Add URL and output file. + if url_to_download_recursively[-1] != "/": + url_to_download_recursively += "/" + wget_command_line.append(url_to_download_recursively) + + # Execute wget. + self.logger.info(f"Downloading {url_to_download_recursively} using wget: {wget_command_line}") + process = subprocess.run(wget_command_line) + if process.returncode != 0: + raise RuntimeError(f"Could not execute wget {wget_command_line}: {process.stderr}") + + return local_path_to_download_to diff --git a/src/babel_xrefs/core/model.py b/src/babel_xrefs/core/model.py new file mode 100644 index 0000000..139597f --- /dev/null +++ b/src/babel_xrefs/core/model.py @@ -0,0 +1,2 @@ + + From 18479d6530261f117bbc5fd109d9e5299c2958a4 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 3 Dec 2025 23:02:21 -0500 Subject: [PATCH 005/123] Added a method to look up a particular identifier. --- src/babel_xrefs/babel_xrefs.py | 24 +++++++++++++++++++++++- src/babel_xrefs/cli.py | 26 +++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/babel_xrefs/babel_xrefs.py b/src/babel_xrefs/babel_xrefs.py index 3ec44ad..6e171fa 100644 --- a/src/babel_xrefs/babel_xrefs.py +++ b/src/babel_xrefs/babel_xrefs.py @@ -11,6 +11,28 @@ class BabelXRefs: def __init__(self, downloader: BabelDownloader): self.downloader = downloader + def get_curie_ids(self, curies: list[str]): + """ + Search for all identifiers in the /ids/ files for a particular CURIE. + + :param curie: A CURIE to search for. + :return: A list of cross-references containing that CURIE. + """ + + identifier_parquet = self.downloader.get_downloaded_file('duckdb/Identifiers.parquet') + concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') + + # Query the Parquet files using DuckDB. + duckdb_path = self.downloader.get_output_file('output/duckdbs/xrefs.duckdb') + db = duckdb.connect(duckdb_path) + identifier_table = db.read_parquet(identifier_parquet) + xrefs = db.execute(f"SELECT * FROM identifier_table WHERE curie IN $1", [curies]) + + # TODO: convert into case classes. + + return xrefs.fetchall() + + def get_curie_xrefs(self, curies: list[str]): """ Search for all identifiers that are cross-referenced to the given CURIE. @@ -20,7 +42,7 @@ def get_curie_xrefs(self, curies: list[str]): """ concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') - concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/ConcordMetadata.parquet') + concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') # Query the Parquet files using DuckDB. duckdb_path = self.downloader.get_output_file('output/duckdbs/xrefs.duckdb') diff --git a/src/babel_xrefs/cli.py b/src/babel_xrefs/cli.py index 2b6c96f..40c7141 100644 --- a/src/babel_xrefs/cli.py +++ b/src/babel_xrefs/cli.py @@ -1,7 +1,6 @@ # Command line interface for babel-xrefs import click import logging -import babel_xrefs from babel_xrefs.core.downloader import BabelDownloader from babel_xrefs.babel_xrefs import BabelXRefs @@ -37,5 +36,30 @@ def xrefs(curies: list[str], babel_url: str, local_dir: str): for xref in xrefs: print(xref) +@cli.command("ids") +@click.argument("curies", type=str, required=True, nargs=-1) +@click.option("--local-dir", type=str, default="data/2025nov19", help="Local location to save Babel download files to") +@click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") +def ids(curies: list[str], babel_url: str, local_dir: str): + """ + Fetches and prints the ID records for the given CURIEs, along with Biolink type if provided. + + \f + + :param curies: A list of CURIEs (Compact URI) for which cross-references need + to be retrieved. + :type curies: list[str] + :param babel_url: Base URL of the Babel server + :type babel_url: str + + :return: None + """ + logging.basicConfig(level=logging.INFO) + + bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir)) + xrefs = bxref.get_curie_ids(curies) + for xref in xrefs: + print(xref) + if __name__ == "__main__": cli() From 7317bfb8383e7cc0af0b6bf48082f10dcb94af5e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 3 Dec 2025 23:29:17 -0500 Subject: [PATCH 006/123] Added CURIE expansion/recursive lookup. --- src/babel_xrefs/babel_xrefs.py | 54 ++++++++++++++++++++++++------ src/babel_xrefs/cli.py | 5 +-- src/babel_xrefs/core/downloader.py | 6 +++- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/babel_xrefs/babel_xrefs.py b/src/babel_xrefs/babel_xrefs.py index 6e171fa..f321e5e 100644 --- a/src/babel_xrefs/babel_xrefs.py +++ b/src/babel_xrefs/babel_xrefs.py @@ -1,11 +1,30 @@ # Babel XRefs is a tool for accessing and querying the intermediate files # that we make available with Babel builds. This allows you to find out # why we consider two identifiers to be identical. +import dataclasses import logging import duckdb +import functools from babel_xrefs.core.downloader import BabelDownloader +@dataclasses.dataclass(frozen=True) +class CrossReference: + filename: str + subj: str + pred: str + obj: str + + @staticmethod + def from_tuple(tuple: tuple[str, str, str, str]): + return CrossReference(filename=tuple[0], subj=tuple[1], pred=tuple[2], obj=tuple[3]) + + @property + def curies(self): + return frozenset([self.subj, self.obj]) + + def __lt__(self, other): + return (self.filename, self.subj, self.obj, self.pred) < (other.filename, other.subj, other.obj, other.pred) class BabelXRefs: def __init__(self, downloader: BabelDownloader): @@ -32,24 +51,37 @@ def get_curie_ids(self, curies: list[str]): return xrefs.fetchall() + @functools.lru_cache(maxsize=None) + def get_curie_xref(self, curie: str): + concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') + concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') + + duckdb_path = self.downloader.get_output_file('output/duckdbs/xrefs.duckdb') + db = duckdb.connect(duckdb_path) + concord_table = db.read_parquet(concord_parquet) + xref_tuples = db.execute(f"SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", [curie]).fetchall() + xrefs = list(map(lambda rec: CrossReference.from_tuple(rec), xref_tuples)) + return xrefs - def get_curie_xrefs(self, curies: list[str]): + def get_curie_xrefs(self, curies: list[str], expand: bool = False, ignore_curies_in_expansion: set = set()): """ Search for all identifiers that are cross-referenced to the given CURIE. :param curie: A CURIE to search for. + :param expand: Whether to expand the cross-references (i.e. recursively follow all identifiers). :return: A list of cross-references containing that CURIE. """ - concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') - concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') + xrefs = set() + for curie in curies: + logging.info(f"Searching for cross-references for {curie}") + xrefs.update(self.get_curie_xref(curie)) - # Query the Parquet files using DuckDB. - duckdb_path = self.downloader.get_output_file('output/duckdbs/xrefs.duckdb') - db = duckdb.connect(duckdb_path) - concord_table = db.read_parquet(concord_parquet) - xrefs = db.execute(f"SELECT * FROM concord_table WHERE subj IN $1 OR obj in $1", [curies]) - - # TODO: convert into case classes. + if expand: + # Get a unique set of referenced curies, not including the ones currently queried. + new_curies = list(set([curie for xref in xrefs for curie in xref.curies]) - set(curies) - ignore_curies_in_expansion) + if new_curies: + logging.info(f"Expanding cross-references to {new_curies}") + xrefs.update(self.get_curie_xrefs(new_curies, expand=True, ignore_curies_in_expansion=ignore_curies_in_expansion | set(new_curies))) - return xrefs.fetchall() + return sorted(xrefs) diff --git a/src/babel_xrefs/cli.py b/src/babel_xrefs/cli.py index 40c7141..5e17b04 100644 --- a/src/babel_xrefs/cli.py +++ b/src/babel_xrefs/cli.py @@ -13,7 +13,8 @@ def cli(): @click.argument("curies", type=str, required=True, nargs=-1) @click.option("--local-dir", type=str, default="data/2025nov19", help="Local location to save Babel download files to") @click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") -def xrefs(curies: list[str], babel_url: str, local_dir: str): +@click.option("--expand", is_flag=True, help="Also display xrefs for returned CURIEs") +def xrefs(curies: list[str], babel_url: str, local_dir: str, expand: bool): """ Fetches and prints the cross-references (xrefs) for the given CURIEs. @@ -32,7 +33,7 @@ def xrefs(curies: list[str], babel_url: str, local_dir: str): logging.basicConfig(level=logging.INFO) bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir)) - xrefs = bxref.get_curie_xrefs(curies) + xrefs = bxref.get_curie_xrefs(curies, expand) for xref in xrefs: print(xref) diff --git a/src/babel_xrefs/core/downloader.py b/src/babel_xrefs/core/downloader.py index 34e4cb0..9313685 100644 --- a/src/babel_xrefs/core/downloader.py +++ b/src/babel_xrefs/core/downloader.py @@ -1,8 +1,10 @@ +import functools import os import urllib.parse import subprocess import requests import logging +import functools class BabelDownloader: """ @@ -31,11 +33,13 @@ def __init__(self, url_base, local_path=None, retries=10): else: raise ValueError(f"Invalid local_path (must be an existing directory): '{local_path}'") + @functools.lru_cache(maxsize=None) def get_output_file(self, filename): filepath = os.path.join(self.local_path, filename) os.makedirs(os.path.dirname(filepath), exist_ok=True) return filepath + @functools.lru_cache(maxsize=None) def get_downloaded_file(self, dirpath: str, chunk_size:int=1024*1024): local_path_to_download_to = os.path.join(self.local_path, dirpath) os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) @@ -65,7 +69,7 @@ def get_downloaded_file(self, dirpath: str, chunk_size:int=1024*1024): self.logger.info(f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes") return local_path_to_download_to - + @functools.lru_cache(maxsize=None) def get_downloaded_dir(self, dirpath: str): local_path_to_download_to = os.path.join(self.local_path, dirpath) os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) From 642dd21b47997cfea54ea4ddb410a371b0cc9dfc Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Thu, 4 Dec 2025 00:56:06 -0500 Subject: [PATCH 007/123] Added a basic ConcordTester. --- src/babel_xrefs/cli.py | 18 ++++++++++++ src/babel_xrefs/core/nodenorm.py | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 src/babel_xrefs/core/nodenorm.py diff --git a/src/babel_xrefs/cli.py b/src/babel_xrefs/cli.py index 5e17b04..fa3ee9e 100644 --- a/src/babel_xrefs/cli.py +++ b/src/babel_xrefs/cli.py @@ -3,6 +3,7 @@ import logging from babel_xrefs.core.downloader import BabelDownloader from babel_xrefs.babel_xrefs import BabelXRefs +from babel_xrefs.core.nodenorm import NodeNorm @click.group() @@ -62,5 +63,22 @@ def ids(curies: list[str], babel_url: str, local_dir: str): for xref in xrefs: print(xref) +@cli.command("test-concord") +@click.argument("curies", type=str, required=True, nargs=-1) +@click.option("--nodenorm-url", type=str, default="https://nodenormalization-sri.renci.org/", help="NodeNorm URL to check for concord changes") +def test_concord(curies, nodenorm_url): + # We're trying to answer a simple question here: if the CURIEs we mention were combined, how would the cliques change in NodeNorm? + # By definition, this can only combine all the cliques mentioned in the CURIEs. + + nodenorm = NodeNorm(nodenorm_url) + for curie in curies: + identifiers = nodenorm.get_clique_identifiers(curie) + for identifier in identifiers: + if identifier.label: + print(f"{curie}\t{identifier.curie}\t{identifier.label}") + else: + print(f"{curie}\t{identifier.curie}\t") + + if __name__ == "__main__": cli() diff --git a/src/babel_xrefs/core/nodenorm.py b/src/babel_xrefs/core/nodenorm.py new file mode 100644 index 0000000..6c45e02 --- /dev/null +++ b/src/babel_xrefs/core/nodenorm.py @@ -0,0 +1,50 @@ +import dataclasses +import functools +import requests + +@dataclasses.dataclass +class Identifier: + curie: str + label: str = "" + taxa: list[str] = dataclasses.field(default_factory=list) + description: list[str] = dataclasses.field(default_factory=list) + + def __lt__(self, other): + return self.curie < other.curie + + @staticmethod + def from_dict(d: dict): + identifier = Identifier(curie=d['identifier']) + if 'label' in d: + identifier.label = d['label'] + if 'taxa' in d: + identifier.taxa = d['taxa'] + if 'description' in d: + identifier.description = d['description'] + return identifier + +class NodeNorm: + def __init__(self, nodenorm_url: str=""): + self.nodenorm_url = nodenorm_url + + @functools.lru_cache(maxsize=None) + def normalize_curie(self, curie: str, conflate=True, drug_chemical_conflate=False, description=False, individual_types=None, include_taxa=None): + response = requests.get(f"{self.nodenorm_url}get_normalized_nodes", params={ + "curie": curie, + "conflate": conflate, + "drug_chemical_conflate": drug_chemical_conflate, + "description": description, + "individual_types": individual_types, + "include_taxa": include_taxa, + }) + response.raise_for_status() + result = response.json() + + return result[curie] + + @functools.lru_cache(maxsize=None) + def get_clique_identifiers(self, curie, **kwargs): + result = self.normalize_curie(curie, **kwargs) + if 'equivalent_identifiers' not in result: + return None + return list(map(lambda x: Identifier.from_dict(x), result['equivalent_identifiers'])) From 75db5b28430edea67146bd736c5ec06a8e6d1670 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Thu, 4 Dec 2025 01:58:24 -0500 Subject: [PATCH 008/123] Added labels via NodeNorm. --- src/babel_xrefs/babel_xrefs.py | 42 ++++++++++++++++++++++++++++---- src/babel_xrefs/cli.py | 12 +++++---- src/babel_xrefs/core/nodenorm.py | 17 ++++++++++++- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/src/babel_xrefs/babel_xrefs.py b/src/babel_xrefs/babel_xrefs.py index f321e5e..e953c0f 100644 --- a/src/babel_xrefs/babel_xrefs.py +++ b/src/babel_xrefs/babel_xrefs.py @@ -7,6 +7,8 @@ import functools from babel_xrefs.core.downloader import BabelDownloader +from babel_xrefs.core.nodenorm import NodeNorm + @dataclasses.dataclass(frozen=True) class CrossReference: @@ -26,9 +28,26 @@ def curies(self): def __lt__(self, other): return (self.filename, self.subj, self.obj, self.pred) < (other.filename, other.subj, other.obj, other.pred) +class LabeledCrossReference(CrossReference): + subj_label: str + subj_biolink_type: str + obj_label: str + obj_biolink_type: str + + def __init__(self, subj: str, pred: str, obj: str, filename: str, subj_label: str, subj_biolink_type: str, obj_label: str, obj_biolink_type: str): + super().__init__(subj=subj, obj=obj, filename=filename, pred=pred) + self.subj_label = subj_label + self.subj_biolink_type = subj_biolink_type + self.obj_label = obj_label + self.obj_biolink_type = obj_biolink_type + + def __str__(self): + return f"""LabeledCrossReference(subj="{self.subj}", obj="{self.obj}", subj_label="{self.subj_label}", obj_label="{self.obj_label}", subj_label="{self.subj_label}", obj_label="{self.obj_label}")""" + class BabelXRefs: - def __init__(self, downloader: BabelDownloader): + def __init__(self, downloader: BabelDownloader, nodenorm: NodeNorm = None): self.downloader = downloader + self.nodenorm = nodenorm def get_curie_ids(self, curies: list[str]): """ @@ -52,7 +71,7 @@ def get_curie_ids(self, curies: list[str]): return xrefs.fetchall() @functools.lru_cache(maxsize=None) - def get_curie_xref(self, curie: str): + def get_curie_xref(self, curie: str, label_curies: bool = False): concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') @@ -61,9 +80,22 @@ def get_curie_xref(self, curie: str): concord_table = db.read_parquet(concord_parquet) xref_tuples = db.execute(f"SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", [curie]).fetchall() xrefs = list(map(lambda rec: CrossReference.from_tuple(rec), xref_tuples)) + + if label_curies: + xrefs = map(lambda xref: LabeledCrossReference( + subj=xref.subj, + obj=xref.obj, + filename=xref.filename, + pred=xref.pred, + subj_label=self.nodenorm.get_identifier(xref.subj).label, + subj_biolink_type=self.nodenorm.get_identifier(xref.subj).biolink_type, + obj_label=self.nodenorm.get_identifier(xref.obj).label, + obj_biolink_type=self.nodenorm.get_identifier(xref.obj).biolink_type, + ), xrefs) + return xrefs - def get_curie_xrefs(self, curies: list[str], expand: bool = False, ignore_curies_in_expansion: set = set()): + def get_curie_xrefs(self, curies: list[str], expand: bool = False, ignore_curies_in_expansion: set = set(), label_curies: bool = False): """ Search for all identifiers that are cross-referenced to the given CURIE. @@ -75,13 +107,13 @@ def get_curie_xrefs(self, curies: list[str], expand: bool = False, ignore_curies xrefs = set() for curie in curies: logging.info(f"Searching for cross-references for {curie}") - xrefs.update(self.get_curie_xref(curie)) + xrefs.update(self.get_curie_xref(curie, label_curies)) if expand: # Get a unique set of referenced curies, not including the ones currently queried. new_curies = list(set([curie for xref in xrefs for curie in xref.curies]) - set(curies) - ignore_curies_in_expansion) if new_curies: logging.info(f"Expanding cross-references to {new_curies}") - xrefs.update(self.get_curie_xrefs(new_curies, expand=True, ignore_curies_in_expansion=ignore_curies_in_expansion | set(new_curies))) + xrefs.update(self.get_curie_xrefs(new_curies, expand=True, ignore_curies_in_expansion=ignore_curies_in_expansion | set(new_curies), label_curies=label_curies)) return sorted(xrefs) diff --git a/src/babel_xrefs/cli.py b/src/babel_xrefs/cli.py index fa3ee9e..3493b5e 100644 --- a/src/babel_xrefs/cli.py +++ b/src/babel_xrefs/cli.py @@ -14,8 +14,10 @@ def cli(): @click.argument("curies", type=str, required=True, nargs=-1) @click.option("--local-dir", type=str, default="data/2025nov19", help="Local location to save Babel download files to") @click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") +@click.option("--nodenorm-url", type=str, default="https://nodenormalization-sri.renci.org/", help="NodeNorm URL to check for concord changes") @click.option("--expand", is_flag=True, help="Also display xrefs for returned CURIEs") -def xrefs(curies: list[str], babel_url: str, local_dir: str, expand: bool): +@click.option("--labels", is_flag=True, help="Include labels for CURIEs") +def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, expand: bool, labels: bool): """ Fetches and prints the cross-references (xrefs) for the given CURIEs. @@ -33,8 +35,8 @@ def xrefs(curies: list[str], babel_url: str, local_dir: str, expand: bool): """ logging.basicConfig(level=logging.INFO) - bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir)) - xrefs = bxref.get_curie_xrefs(curies, expand) + bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir), NodeNorm(nodenorm_url)) + xrefs = bxref.get_curie_xrefs(curies, expand, label_curies=labels) for xref in xrefs: print(xref) @@ -75,9 +77,9 @@ def test_concord(curies, nodenorm_url): identifiers = nodenorm.get_clique_identifiers(curie) for identifier in identifiers: if identifier.label: - print(f"{curie}\t{identifier.curie}\t{identifier.label}") + print(f"{curie}\t{identifier.curie}\t{identifier.label}\t{identifier.biolink_type}") else: - print(f"{curie}\t{identifier.curie}\t") + print(f"{curie}\t{identifier.curie}\t\t{identifier.biolink_type}") if __name__ == "__main__": diff --git a/src/babel_xrefs/core/nodenorm.py b/src/babel_xrefs/core/nodenorm.py index 6c45e02..5c8e0c9 100644 --- a/src/babel_xrefs/core/nodenorm.py +++ b/src/babel_xrefs/core/nodenorm.py @@ -1,11 +1,13 @@ import dataclasses import functools import requests +import logging @dataclasses.dataclass class Identifier: curie: str label: str = "" + biolink_type: str = "" taxa: list[str] = dataclasses.field(default_factory=list) description: list[str] = dataclasses.field(default_factory=list) @@ -21,6 +23,8 @@ def from_dict(d: dict): identifier.taxa = d['taxa'] if 'description' in d: identifier.description = d['description'] + if 'type' in d: + identifier.biolink_type = d['type'] return identifier class NodeNorm: @@ -28,7 +32,18 @@ def __init__(self, nodenorm_url: str=""): self.nodenorm_url = nodenorm_url @functools.lru_cache(maxsize=None) - def normalize_curie(self, curie: str, conflate=True, drug_chemical_conflate=False, description=False, individual_types=None, include_taxa=None): + def get_identifier(self, curie): + result = self.normalize_curie(curie) + logging.debug(f"Normalizing {curie} with NodeNorm to result: {result}") + for identifier in result.get('equivalent_identifiers', []): + if identifier['identifier'] == curie: + logging.debug(f"Found exact match for {curie}: {identifier}") + return Identifier.from_dict(identifier) + + return Identifier(curie=curie) + + @functools.lru_cache(maxsize=None) + def normalize_curie(self, curie: str, conflate=True, drug_chemical_conflate=True, description=True, individual_types=True, include_taxa=True): response = requests.get(f"{self.nodenorm_url}get_normalized_nodes", params={ "curie": curie, "conflate": conflate, From eddd497935517fe41c8f03741e81b018649d4eff Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Thu, 8 Jan 2026 14:58:30 -0500 Subject: [PATCH 009/123] Midnight commit: attempting to improve expansion. --- src/babel_xrefs/babel_xrefs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/babel_xrefs/babel_xrefs.py b/src/babel_xrefs/babel_xrefs.py index e953c0f..32829c1 100644 --- a/src/babel_xrefs/babel_xrefs.py +++ b/src/babel_xrefs/babel_xrefs.py @@ -104,6 +104,9 @@ def get_curie_xrefs(self, curies: list[str], expand: bool = False, ignore_curies :return: A list of cross-references containing that CURIE. """ + if ignore_curies_in_expansion: + logging.info(f"Ignoring {len(ignore_curies_in_expansion)}: {ignore_curies_in_expansion}") + xrefs = set() for curie in curies: logging.info(f"Searching for cross-references for {curie}") @@ -114,6 +117,6 @@ def get_curie_xrefs(self, curies: list[str], expand: bool = False, ignore_curies new_curies = list(set([curie for xref in xrefs for curie in xref.curies]) - set(curies) - ignore_curies_in_expansion) if new_curies: logging.info(f"Expanding cross-references to {new_curies}") - xrefs.update(self.get_curie_xrefs(new_curies, expand=True, ignore_curies_in_expansion=ignore_curies_in_expansion | set(new_curies), label_curies=label_curies)) + xrefs.update(self.get_curie_xrefs(new_curies, expand=True, ignore_curies_in_expansion=ignore_curies_in_expansion | set(curies) | set(new_curies), label_curies=label_curies)) return sorted(xrefs) From 3a55a23ad9ede26bd2f70b7d06674ffcb7020b51 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Thu, 8 Jan 2026 16:04:56 -0500 Subject: [PATCH 010/123] Added some improvements. --- src/babel_xrefs/babel_xrefs.py | 2 +- src/babel_xrefs/core/nodenorm.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/babel_xrefs/babel_xrefs.py b/src/babel_xrefs/babel_xrefs.py index 32829c1..bf32521 100644 --- a/src/babel_xrefs/babel_xrefs.py +++ b/src/babel_xrefs/babel_xrefs.py @@ -42,7 +42,7 @@ def __init__(self, subj: str, pred: str, obj: str, filename: str, subj_label: st self.obj_biolink_type = obj_biolink_type def __str__(self): - return f"""LabeledCrossReference(subj="{self.subj}", obj="{self.obj}", subj_label="{self.subj_label}", obj_label="{self.obj_label}", subj_label="{self.subj_label}", obj_label="{self.obj_label}")""" + return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", obj_label="{self.obj_label}", subj_label="{self.subj_label}", obj_label="{self.obj_label}")""" class BabelXRefs: def __init__(self, downloader: BabelDownloader, nodenorm: NodeNorm = None): diff --git a/src/babel_xrefs/core/nodenorm.py b/src/babel_xrefs/core/nodenorm.py index 5c8e0c9..018f106 100644 --- a/src/babel_xrefs/core/nodenorm.py +++ b/src/babel_xrefs/core/nodenorm.py @@ -35,6 +35,8 @@ def __init__(self, nodenorm_url: str=""): def get_identifier(self, curie): result = self.normalize_curie(curie) logging.debug(f"Normalizing {curie} with NodeNorm to result: {result}") + if not result: + return Identifier(curie=curie) for identifier in result.get('equivalent_identifiers', []): if identifier['identifier'] == curie: logging.debug(f"Found exact match for {curie}: {identifier}") From e45e2f0c99799c12f971b1a82a5787d72f9e9ca0 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sat, 14 Feb 2026 01:25:40 -0500 Subject: [PATCH 011/123] Added a CLAUDE.md by Claude.ai. --- CLAUDE.md | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6f3ed1a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,118 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +babel-xrefs is a tool for querying and exploring Babel intermediate files. It allows users to discover why two biological/chemical identifiers are considered identical by the Babel system, which handles cross-references between different ontology and database identifiers (e.g., MONDO, HP, UMLS, HGNC). + +## Development Setup + +This project uses **uv** for package management: + +```bash +# Install dependencies +uv sync + +# Install with dev dependencies +uv sync --group dev + +# Activate virtual environment (if needed) +source .venv/bin/activate + +# Run the CLI +uv run babel-xrefs --help +``` + +## Commands + +### Running the Application + +```bash +# Get cross-references for one or more CURIEs +uv run babel-xrefs xrefs MONDO:0004979 + +# Get cross-references with expansion (recursive lookup) +uv run babel-xrefs xrefs MONDO:0004979 --expand + +# Get cross-references with labels from NodeNorm +uv run babel-xrefs xrefs MONDO:0004979 --labels + +# Get ID records for CURIEs +uv run babel-xrefs ids MONDO:0004979 + +# Test concordance changes with NodeNorm +uv run babel-xrefs test-concord MONDO:0004979 HP:0000001 + +# Use custom Babel server or local directory +uv run babel-xrefs xrefs MONDO:0004979 --local-dir data/2025nov19 --babel-url https://stars.renci.org:443/var/babel/2025nov19/ +``` + +### Development Commands + +```bash +# Run tests +uv run pytest + +# Run linter +uv run ruff check + +# Format code +uv run ruff format +``` + +## Architecture + +### Core Components + +1. **BabelDownloader** (`src/babel_xrefs/core/downloader.py`): + - Downloads Babel intermediate files from a remote server using `wget` + - Caches files locally in configurable directory (default: `data/2025nov19/`) + - Uses `@functools.lru_cache` to avoid re-downloading + - **Important**: Requires `wget` to be installed on the system + +2. **BabelXRefs** (`src/babel_xrefs/babel_xrefs.py`): + - Main query engine for cross-references + - Uses DuckDB to query Parquet files (`Concord.parquet`, `Identifiers.parquet`, `Metadata.parquet`) + - Supports recursive expansion of cross-references + - Creates ephemeral DuckDB databases in `data//output/duckdbs/` + +3. **NodeNorm** (`src/babel_xrefs/core/nodenorm.py`): + - Integration with NodeNormalization API (https://nodenormalization-sri.renci.org/) + - Fetches labels, biolink types, and equivalent identifiers for CURIEs + - Uses `@functools.lru_cache` for performance + - Optional component for label enrichment + +4. **CLI** (`src/babel_xrefs/cli.py`): + - Click-based command-line interface + - Three main commands: `xrefs`, `ids`, `test-concord` + +### Data Flow + +1. User provides CURIEs via CLI +2. BabelDownloader ensures required Parquet files are downloaded +3. BabelXRefs queries files using DuckDB +4. If `--labels` or `--expand` flags are set, NodeNorm is queried for additional metadata +5. Results are printed to stdout + +### Key Design Patterns + +- **Lazy downloading**: Files are only downloaded when first accessed +- **LRU caching**: Heavy use of `@functools.lru_cache` to avoid redundant downloads and API calls +- **Recursive expansion**: The `--expand` flag recursively follows all cross-references to build complete graphs +- **DuckDB for querying**: In-memory SQL queries against Parquet files for fast lookups + +## Important Notes + +- **System dependency**: This project requires `wget` to be installed (used by BabelDownloader) +- **Data directory**: The `data/` directory is gitignored and contains downloaded Parquet files and generated DuckDB databases +- **Babel versions**: The default Babel version is `2025nov19`, but this can be customized via `--local-dir` and `--babel-url` +- **No tests yet**: The project currently has pytest configured but no test files exist +- **Empty model.py**: The `src/babel_xrefs/core/model.py` file exists but is currently empty; data classes are defined in `babel_xrefs.py` and `nodenorm.py` instead + +## File Locations + +- Source code: `src/babel_xrefs/` +- Downloaded Babel files: `data//duckdb/*.parquet` +- Generated DuckDB databases: `data//output/duckdbs/` +- Entry point: `src/babel_xrefs/cli.py` From 46e829f39893db1708c730cd3465e1c163acaad7 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sat, 14 Feb 2026 01:52:15 -0500 Subject: [PATCH 012/123] Reorganized file slightly. --- src/babel_xrefs/cli.py | 2 +- src/babel_xrefs/{ => core}/babel_xrefs.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/babel_xrefs/{ => core}/babel_xrefs.py (100%) diff --git a/src/babel_xrefs/cli.py b/src/babel_xrefs/cli.py index 3493b5e..58c4e57 100644 --- a/src/babel_xrefs/cli.py +++ b/src/babel_xrefs/cli.py @@ -2,7 +2,7 @@ import click import logging from babel_xrefs.core.downloader import BabelDownloader -from babel_xrefs.babel_xrefs import BabelXRefs +from babel_xrefs.core.babel_xrefs import BabelXRefs from babel_xrefs.core.nodenorm import NodeNorm diff --git a/src/babel_xrefs/babel_xrefs.py b/src/babel_xrefs/core/babel_xrefs.py similarity index 100% rename from src/babel_xrefs/babel_xrefs.py rename to src/babel_xrefs/core/babel_xrefs.py From 0fb11c5a7d20f755ae99eb33506b715fc6b81690 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sat, 14 Feb 2026 02:02:02 -0500 Subject: [PATCH 013/123] Claude wrote some tests. --- tests/__init__.py | 1 + tests/test_downloader.py | 194 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/test_downloader.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..61c04ac --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Tests for babel-xrefs diff --git a/tests/test_downloader.py b/tests/test_downloader.py new file mode 100644 index 0000000..718daa9 --- /dev/null +++ b/tests/test_downloader.py @@ -0,0 +1,194 @@ +""" +Tests for the BabelDownloader class. + +These tests verify that the downloader can successfully fetch large Parquet files +from the Babel server using wget and properly manage local file caching. +""" + +import os +import shutil +import pytest +from babel_xrefs.core.downloader import BabelDownloader + + +# Constants for test configuration +BABEL_URL = "https://stars.renci.org/var/babel/2025nov19/" +TEST_DATA_DIR = "data/test" +IDENTIFIERS_FILE = "duckdb/Identifiers.parquet" +MINIMUM_FILE_SIZE_GB = 2 +MINIMUM_FILE_SIZE_BYTES = MINIMUM_FILE_SIZE_GB * 1024 * 1024 * 1024 # 2GB in bytes + + +@pytest.fixture(scope="module") +def test_data_dir(): + """ + Fixture that provides a clean test data directory. + + This fixture: + - Creates the test data directory before tests run + - Yields the directory path to tests + - Cleans up (removes) the directory after all tests complete + + Scope is 'module' so the directory persists across all tests in this file, + allowing downloaded files to be reused by multiple tests. + """ + # Setup: ensure clean test directory + if os.path.exists(TEST_DATA_DIR): + shutil.rmtree(TEST_DATA_DIR) + os.makedirs(TEST_DATA_DIR, exist_ok=True) + + yield TEST_DATA_DIR + + # Teardown: remove test directory and all contents + if os.path.exists(TEST_DATA_DIR): + shutil.rmtree(TEST_DATA_DIR) + + +@pytest.fixture(scope="module") +def downloader(test_data_dir): + """ + Fixture that provides a BabelDownloader instance configured for testing. + + Args: + test_data_dir: The test data directory fixture + + Returns: + BabelDownloader: Configured downloader instance + """ + return BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + +def test_downloader_initialization(test_data_dir): + """ + Test that BabelDownloader initializes correctly with custom parameters. + + Verifies: + - Downloader accepts URL and local path + - Local path is stored correctly + - Directory is created if it doesn't exist + """ + downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + assert downloader.url_base == BABEL_URL + assert downloader.local_path == test_data_dir + assert os.path.exists(test_data_dir) + assert os.path.isdir(test_data_dir) + + +def test_download_large_parquet_file(downloader): + """ + Test downloading a large Parquet file from the Babel server. + + This test: + 1. Downloads the Identifiers.parquet file (2GB+) from the real Babel server + 2. Verifies the file was downloaded successfully + 3. Confirms the file size is at least 2GB + + Note: This test takes several minutes to complete due to the large file size. + + Args: + downloader: BabelDownloader fixture + """ + # Download the Identifiers.parquet file + downloaded_path = downloader.get_downloaded_file(IDENTIFIERS_FILE) + + # Verify the file exists + assert os.path.exists(downloaded_path), \ + f"Downloaded file does not exist at {downloaded_path}" + + # Verify it's a file, not a directory + assert os.path.isfile(downloaded_path), \ + f"Downloaded path is not a file: {downloaded_path}" + + # Get the file size in bytes + file_size_bytes = os.path.getsize(downloaded_path) + file_size_gb = file_size_bytes / (1024 * 1024 * 1024) + + # Verify the file is at least 2GB + assert file_size_bytes >= MINIMUM_FILE_SIZE_BYTES, \ + f"Downloaded file is too small: {file_size_gb:.2f}GB (expected at least {MINIMUM_FILE_SIZE_GB}GB)" + + print(f"\n✓ Successfully downloaded {IDENTIFIERS_FILE}") + print(f" Size: {file_size_gb:.2f}GB ({file_size_bytes:,} bytes)") + print(f" Path: {downloaded_path}") + + +def test_download_caching(downloader): + """ + Test that the downloader uses LRU caching to avoid re-downloading files. + + This test: + 1. Downloads the same file twice + 2. Verifies both calls return the same path + 3. Confirms the file is only downloaded once (via caching) + + Args: + downloader: BabelDownloader fixture + """ + # First download + path1 = downloader.get_downloaded_file(IDENTIFIERS_FILE) + initial_mtime = os.path.getmtime(path1) + + # Second download - should use cache + path2 = downloader.get_downloaded_file(IDENTIFIERS_FILE) + second_mtime = os.path.getmtime(path2) + + # Verify same path returned + assert path1 == path2, "Cached download returned different path" + + # Verify file wasn't modified (i.e., wasn't re-downloaded) + assert initial_mtime == second_mtime, \ + "File was modified, suggesting it was re-downloaded instead of cached" + + print(f"\n✓ Caching works correctly - file not re-downloaded") + + +def test_get_output_file(downloader): + """ + Test the get_output_file method for creating output file paths. + + This test: + 1. Creates an output file path + 2. Verifies the directory structure is created + 3. Confirms the path is in the correct location + + Args: + downloader: BabelDownloader fixture + """ + output_filename = "output/duckdbs/test.duckdb" + output_path = downloader.get_output_file(output_filename) + + # Verify the path is correct + expected_path = os.path.join(TEST_DATA_DIR, output_filename) + assert output_path == expected_path, \ + f"Output path mismatch: expected {expected_path}, got {output_path}" + + # Verify the parent directory was created + assert os.path.exists(os.path.dirname(output_path)), \ + "Parent directory for output file was not created" + + print(f"\n✓ Output file path created correctly: {output_path}") + + +def test_invalid_local_path(): + """ + Test that BabelDownloader raises an error for invalid local paths. + + This test verifies error handling when attempting to use a file path + as the local directory (should be a directory, not a file). + """ + # Create a temporary file + invalid_path = "/tmp/test_babel_invalid_file.txt" + with open(invalid_path, 'w') as f: + f.write("test") + + try: + # Attempt to create downloader with a file path instead of directory + with pytest.raises(ValueError, match="Invalid local_path"): + BabelDownloader(url_base=BABEL_URL, local_path=invalid_path) + + print("\n✓ Correctly raised ValueError for invalid local path") + finally: + # Clean up + if os.path.exists(invalid_path): + os.remove(invalid_path) From 787618937b5b6255b650d0048fdd57f195c57425 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sat, 14 Feb 2026 02:13:52 -0500 Subject: [PATCH 014/123] Improved downloader using Claude. --- pyproject.toml | 1 + src/babel_xrefs/core/downloader.py | 182 +++++++++++++++++++++-------- 2 files changed, 132 insertions(+), 51 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5696f67..ddc0a95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "click>=8.3.1", "duckdb>=1.4.2", "requests>=2.32.5", + "tqdm>=4.67.0", ] [build-system] diff --git a/src/babel_xrefs/core/downloader.py b/src/babel_xrefs/core/downloader.py index 9313685..8e9a07e 100644 --- a/src/babel_xrefs/core/downloader.py +++ b/src/babel_xrefs/core/downloader.py @@ -1,10 +1,11 @@ import functools import os import urllib.parse -import subprocess +import time import requests +from tqdm import tqdm import logging -import functools + class BabelDownloader: """ @@ -39,31 +40,126 @@ def get_output_file(self, filename): os.makedirs(os.path.dirname(filepath), exist_ok=True) return filepath + def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): + """ + Stream download from response to file with progress bar. + + Args: + response: requests.Response object with stream=True + local_path: Local file path to write to + resume_byte_pos: Starting byte position (for resume) + chunk_size: Size of chunks to read/write + """ + # Get total size from Content-Length header (may not be present) + content_length = response.headers.get('Content-Length') + if content_length: + total_size = int(content_length) + resume_byte_pos + else: + total_size = None + + # Open file in append mode if resuming, write mode otherwise + mode = 'ab' if resume_byte_pos > 0 else 'wb' + + with open(local_path, mode) as f: + with tqdm( + total=total_size, + initial=resume_byte_pos, + unit='B', + unit_scale=True, + unit_divisor=1024, + desc=os.path.basename(local_path) + ) as progress_bar: + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + progress_bar.update(len(chunk)) + + def _download_with_retry(self, url, local_path, chunk_size): + """ + Download a file with retry logic and resume capability. + + Args: + url: URL to download from + local_path: Local file path to save to + chunk_size: Size of chunks to read/write + + Raises: + RuntimeError: If all retry attempts fail + """ + for attempt in range(1, self.retries + 1): + try: + # Check if we're resuming a partial download + resume_byte_pos = 0 + if os.path.exists(local_path): + resume_byte_pos = os.path.getsize(local_path) + + # Prepare headers for resume + headers = {} + if resume_byte_pos > 0: + headers['Range'] = f'bytes={resume_byte_pos}-' + self.logger.info(f"Resuming download from byte {resume_byte_pos}") + + # Make streaming request with timeout for connection (not total time) + response = requests.get(url, headers=headers, stream=True, timeout=30) + + # Handle different response codes + if response.status_code == 416: + # Range Not Satisfiable - file already complete + self.logger.info(f"File already complete: {local_path}") + return + elif response.status_code == 206: + # Partial Content - resume successful + self.logger.info(f"Resuming download (HTTP 206)") + elif response.status_code == 200: + # OK - server doesn't support resume or no Range header was sent + if resume_byte_pos > 0: + self.logger.warning(f"Server doesn't support resume, restarting from beginning") + resume_byte_pos = 0 + # Remove partial file + if os.path.exists(local_path): + os.remove(local_path) + else: + response.raise_for_status() + + # Stream download with progress bar + self._stream_download(response, local_path, resume_byte_pos, chunk_size) + + # Success - exit retry loop + return + + except (requests.RequestException, IOError) as e: + self.logger.warning(f"Download attempt {attempt}/{self.retries} failed: {e}") + + if attempt < self.retries: + # Calculate exponential backoff with max of 60 seconds + wait_time = min(2 ** attempt, 60) + self.logger.info(f"Retrying in {wait_time} seconds...") + time.sleep(wait_time) + else: + # All retries exhausted + raise RuntimeError(f"Failed to download {url} after {self.retries} attempts: {e}") + @functools.lru_cache(maxsize=None) - def get_downloaded_file(self, dirpath: str, chunk_size:int=1024*1024): + def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): + """ + Download a file from the Babel server to local storage. + + Args: + dirpath: Relative path from url_base to the file + chunk_size: Size of chunks to download (default 1MB) + + Returns: + str: Local path to the downloaded file + """ local_path_to_download_to = os.path.join(self.local_path, dirpath) os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) url_to_download = urllib.parse.urljoin(self.url_base, dirpath) - bytes_downloaded = 0 - - wget_command_line = [ - "wget", - "--progress=bar:force:noscroll", # Display progress bar. - "--compression=auto", # Compress files if available. - "--continue", # Continue downloading in case of interruption. - f"--tries={self.retries}", - "-O" + local_path_to_download_to, - ] - - # Add URL and output file. - wget_command_line.append(url_to_download) - - # Execute wget. - self.logger.info(f"Downloading {url_to_download} using wget: {wget_command_line}") - process = subprocess.run(wget_command_line) - if process.returncode != 0: - raise RuntimeError(f"Could not execute wget {wget_command_line}: {process.stderr}") + + self.logger.info(f"Downloading {url_to_download} to {local_path_to_download_to}") + + # Download with retry logic + self._download_with_retry(url_to_download, local_path_to_download_to, chunk_size) bytes_downloaded = os.path.getsize(local_path_to_download_to) self.logger.info(f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes") @@ -71,32 +167,16 @@ def get_downloaded_file(self, dirpath: str, chunk_size:int=1024*1024): @functools.lru_cache(maxsize=None) def get_downloaded_dir(self, dirpath: str): - local_path_to_download_to = os.path.join(self.local_path, dirpath) - os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) - - url_to_download_recursively = urllib.parse.urljoin(self.url_base, dirpath) - - wget_command_line = [ - "wget", - "--progress=bar:force:noscroll", # Display progress bar. - "--compression=auto", # Compress files if available. - "--continue", # Continue downloading in case of interruption. - f"--tries={self.retries}", - "--recursive", - "--no-parent", - "--no-host-directories", - "--directory-prefix=" + local_path_to_download_to, - ] - - # Add URL and output file. - if url_to_download_recursively[-1] != "/": - url_to_download_recursively += "/" - wget_command_line.append(url_to_download_recursively) - - # Execute wget. - self.logger.info(f"Downloading {url_to_download_recursively} using wget: {wget_command_line}") - process = subprocess.run(wget_command_line) - if process.returncode != 0: - raise RuntimeError(f"Could not execute wget {wget_command_line}: {process.stderr}") - - return local_path_to_download_to + """ + Download a directory recursively. + + NOTE: This method is not implemented in the Python-based downloader. + Use get_downloaded_file() for individual files instead. + + Raises: + NotImplementedError: This method is not implemented + """ + raise NotImplementedError( + "Recursive directory downloads are not supported. " + "Use get_downloaded_file() for individual files." + ) From f3d6e52e40aba58bf08c4252b9721664f2096b92 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sat, 14 Feb 2026 02:19:09 -0500 Subject: [PATCH 015/123] Added MD5 download functionality. --- src/babel_xrefs/core/downloader.py | 96 +++++++++++++- tests/test_downloader.py | 204 ++++++++++++++++++++++++++++- 2 files changed, 298 insertions(+), 2 deletions(-) diff --git a/src/babel_xrefs/core/downloader.py b/src/babel_xrefs/core/downloader.py index 8e9a07e..93081c6 100644 --- a/src/babel_xrefs/core/downloader.py +++ b/src/babel_xrefs/core/downloader.py @@ -2,6 +2,7 @@ import os import urllib.parse import time +import hashlib import requests from tqdm import tqdm import logging @@ -40,6 +41,56 @@ def get_output_file(self, filename): os.makedirs(os.path.dirname(filepath), exist_ok=True) return filepath + def _calculate_md5(self, file_path, chunk_size=1024*1024): + """ + Calculate MD5 checksum of a file. + + Args: + file_path: Path to the file to checksum + chunk_size: Size of chunks to read (default 1MB) + + Returns: + str: Hexadecimal MD5 checksum + """ + md5_hash = hashlib.md5() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(chunk_size), b''): + md5_hash.update(chunk) + return md5_hash.hexdigest() + + def _fetch_remote_md5(self, url): + """ + Fetch MD5 checksum from remote .md5 file. + + Args: + url: URL to the .md5 file + + Returns: + str: MD5 checksum if found, None if file doesn't exist or is malformed + """ + try: + response = requests.get(url, timeout=10) + if response.status_code == 404: + self.logger.debug(f"No .md5 file found at {url}") + return None + response.raise_for_status() + + # Parse MD5 file content + # Format is typically: "md5hash filename" or just "md5hash" + content = response.text.strip() + md5_match = content.split()[0] # Take first token + + # Validate it's a valid MD5 (32 hex characters) + if len(md5_match) == 32 and all(c in '0123456789abcdef' for c in md5_match.lower()): + return md5_match.lower() + else: + self.logger.warning(f"Malformed .md5 file at {url}: {content}") + return None + + except requests.RequestException as e: + self.logger.debug(f"Could not fetch .md5 file from {url}: {e}") + return None + def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): """ Stream download from response to file with progress bar. @@ -142,7 +193,13 @@ def _download_with_retry(self, url, local_path, chunk_size): @functools.lru_cache(maxsize=None) def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): """ - Download a file from the Babel server to local storage. + Download a file from the Babel server to local storage with MD5 validation. + + If a .md5 file exists on the server, this method will: + 1. Check if the local file exists + 2. Verify its MD5 checksum matches the expected value + 3. Delete and re-download if checksums don't match + 4. Skip download if checksums match Args: dirpath: Relative path from url_base to the file @@ -155,12 +212,49 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) url_to_download = urllib.parse.urljoin(self.url_base, dirpath) + md5_url = url_to_download + '.md5' + + # Check if file already exists and validate with MD5 if available + if os.path.exists(local_path_to_download_to): + self.logger.info(f"Local file exists: {local_path_to_download_to}") + + # Try to fetch remote MD5 checksum + expected_md5 = self._fetch_remote_md5(md5_url) + + if expected_md5: + self.logger.info(f"Validating MD5 checksum (expected: {expected_md5})") + + # Calculate local file's MD5 + actual_md5 = self._calculate_md5(local_path_to_download_to, chunk_size) + self.logger.info(f"Local file MD5: {actual_md5}") + + if actual_md5 == expected_md5: + # File is valid, skip download + self.logger.info(f"MD5 checksum matches - file is valid, skipping download") + bytes_downloaded = os.path.getsize(local_path_to_download_to) + self.logger.info(f"Using existing file: {local_path_to_download_to} ({bytes_downloaded} bytes)") + return local_path_to_download_to + else: + # Checksums don't match - delete and re-download + self.logger.warning(f"MD5 checksum mismatch! Expected {expected_md5}, got {actual_md5}") + self.logger.warning(f"Deleting corrupted file and re-downloading: {local_path_to_download_to}") + os.remove(local_path_to_download_to) self.logger.info(f"Downloading {url_to_download} to {local_path_to_download_to}") # Download with retry logic self._download_with_retry(url_to_download, local_path_to_download_to, chunk_size) + # Verify MD5 after download if available + expected_md5 = self._fetch_remote_md5(md5_url) + if expected_md5: + actual_md5 = self._calculate_md5(local_path_to_download_to, chunk_size) + if actual_md5 == expected_md5: + self.logger.info(f"Post-download MD5 verification passed: {actual_md5}") + else: + self.logger.error(f"Post-download MD5 verification failed! Expected {expected_md5}, got {actual_md5}") + raise RuntimeError(f"Downloaded file has incorrect MD5 checksum") + bytes_downloaded = os.path.getsize(local_path_to_download_to) self.logger.info(f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes") return local_path_to_download_to diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 718daa9..37f9e77 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -2,12 +2,14 @@ Tests for the BabelDownloader class. These tests verify that the downloader can successfully fetch large Parquet files -from the Babel server using wget and properly manage local file caching. +from the Babel server and properly manage local file caching with MD5 validation. """ import os import shutil +import hashlib import pytest +from unittest.mock import Mock, patch, MagicMock from babel_xrefs.core.downloader import BabelDownloader @@ -192,3 +194,203 @@ def test_invalid_local_path(): # Clean up if os.path.exists(invalid_path): os.remove(invalid_path) + + +def test_md5_validation_matching_checksum(test_data_dir): + """ + Test that MD5 validation skips download when checksums match. + + This test: + 1. Creates a local file with known content + 2. Mocks the .md5 file to return the correct checksum + 3. Verifies the download is skipped (no actual HTTP download occurs) + """ + downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + # Create a test file with known content + test_file = "test_file.txt" + local_path = os.path.join(test_data_dir, test_file) + os.makedirs(os.path.dirname(local_path), exist_ok=True) + + test_content = b"This is test content for MD5 validation" + with open(local_path, 'wb') as f: + f.write(test_content) + + # Calculate the expected MD5 + expected_md5 = hashlib.md5(test_content).hexdigest() + + # Mock the _fetch_remote_md5 to return the matching checksum + with patch.object(downloader, '_fetch_remote_md5', return_value=expected_md5): + # Mock _download_with_retry to ensure it's NOT called + with patch.object(downloader, '_download_with_retry') as mock_download: + # Clear the cache before testing + downloader.get_downloaded_file.cache_clear() + + result_path = downloader.get_downloaded_file(test_file) + + # Verify the download was skipped + mock_download.assert_not_called() + assert result_path == local_path + assert os.path.exists(result_path) + + print(f"\n✓ MD5 validation correctly skipped download for matching checksum: {expected_md5}") + + +def test_md5_validation_mismatched_checksum(test_data_dir): + """ + Test that MD5 validation deletes and re-downloads file when checksums don't match. + + This test: + 1. Creates a local file with wrong content + 2. Mocks the .md5 file to return a different checksum + 3. Verifies the file is deleted and re-downloaded + """ + downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + # Create a test file with incorrect content + test_file = "test_file_mismatch.txt" + local_path = os.path.join(test_data_dir, test_file) + os.makedirs(os.path.dirname(local_path), exist_ok=True) + + wrong_content = b"This is WRONG content" + with open(local_path, 'wb') as f: + f.write(wrong_content) + + # Use a different MD5 (this is MD5 of "correct content") + correct_content = b"This is CORRECT content" + expected_md5 = hashlib.md5(correct_content).hexdigest() + + # Track whether file was deleted + original_exists = os.path.exists(local_path) + + # Mock the _fetch_remote_md5 to return the mismatched checksum + with patch.object(downloader, '_fetch_remote_md5', return_value=expected_md5): + # Mock _download_with_retry to create the "correct" file + def mock_download(url, path, chunk_size): + with open(path, 'wb') as f: + f.write(correct_content) + + with patch.object(downloader, '_download_with_retry', side_effect=mock_download): + # Clear the cache before testing + downloader.get_downloaded_file.cache_clear() + + result_path = downloader.get_downloaded_file(test_file) + + # Verify the file exists and has correct content + assert os.path.exists(result_path) + with open(result_path, 'rb') as f: + assert f.read() == correct_content + + print(f"\n✓ MD5 validation correctly deleted and re-downloaded file with mismatched checksum") + + +def test_md5_validation_no_md5_file(test_data_dir): + """ + Test that download proceeds normally when no .md5 file exists. + + This test: + 1. Mocks the .md5 file fetch to return None (404) + 2. Verifies the download proceeds normally + """ + downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + test_file = "test_file_no_md5.txt" + local_path = os.path.join(test_data_dir, test_file) + + test_content = b"Test content without MD5 file" + + # Mock the _fetch_remote_md5 to return None (no .md5 file) + with patch.object(downloader, '_fetch_remote_md5', return_value=None): + # Mock _download_with_retry to create the file + def mock_download(url, path, chunk_size): + with open(path, 'wb') as f: + f.write(test_content) + + with patch.object(downloader, '_download_with_retry', side_effect=mock_download) as mock_download_method: + # Clear the cache before testing + downloader.get_downloaded_file.cache_clear() + + result_path = downloader.get_downloaded_file(test_file) + + # Verify download was called (normal download path) + mock_download_method.assert_called_once() + assert os.path.exists(result_path) + with open(result_path, 'rb') as f: + assert f.read() == test_content + + print(f"\n✓ Download proceeded normally when no .md5 file exists") + + +def test_md5_validation_malformed_md5_file(test_data_dir): + """ + Test that download proceeds normally when .md5 file is malformed. + + This test: + 1. Mocks the .md5 file fetch to return None (malformed content) + 2. Verifies the download proceeds normally with a warning + """ + downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + test_file = "test_file_malformed_md5.txt" + local_path = os.path.join(test_data_dir, test_file) + + test_content = b"Test content with malformed MD5 file" + + # Mock the _fetch_remote_md5 to return None (malformed .md5 file) + with patch.object(downloader, '_fetch_remote_md5', return_value=None): + # Mock _download_with_retry to create the file + def mock_download(url, path, chunk_size): + with open(path, 'wb') as f: + f.write(test_content) + + with patch.object(downloader, '_download_with_retry', side_effect=mock_download) as mock_download_method: + # Clear the cache before testing + downloader.get_downloaded_file.cache_clear() + + result_path = downloader.get_downloaded_file(test_file) + + # Verify download was called (normal download path) + mock_download_method.assert_called_once() + assert os.path.exists(result_path) + + print(f"\n✓ Download proceeded normally when .md5 file is malformed") + + +def test_md5_post_download_validation(test_data_dir): + """ + Test that MD5 validation occurs after download and fails if checksum is wrong. + + This test: + 1. Downloads a new file + 2. Mocks the .md5 file to return a checksum + 3. Mocks the download to create a file with WRONG content + 4. Verifies a RuntimeError is raised for checksum mismatch + """ + downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + test_file = "test_file_post_validation.txt" + local_path = os.path.join(test_data_dir, test_file) + + # Expected content and MD5 + correct_content = b"Expected content" + expected_md5 = hashlib.md5(correct_content).hexdigest() + + # Wrong content that will be downloaded + wrong_content = b"Wrong content downloaded" + + # Mock the _fetch_remote_md5 to return the expected checksum + with patch.object(downloader, '_fetch_remote_md5', return_value=expected_md5): + # Mock _download_with_retry to create a file with WRONG content + def mock_download(url, path, chunk_size): + with open(path, 'wb') as f: + f.write(wrong_content) + + with patch.object(downloader, '_download_with_retry', side_effect=mock_download): + # Clear the cache before testing + downloader.get_downloaded_file.cache_clear() + + # Should raise RuntimeError due to post-download MD5 mismatch + with pytest.raises(RuntimeError, match="incorrect MD5 checksum"): + downloader.get_downloaded_file(test_file) + + print(f"\n✓ Post-download MD5 validation correctly detected checksum mismatch") From 76907babb71dab8f7616cf5f15abc23b83bcea70 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 15 Feb 2026 02:11:32 -0500 Subject: [PATCH 016/123] Removed empty model file. --- src/babel_xrefs/core/model.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 src/babel_xrefs/core/model.py diff --git a/src/babel_xrefs/core/model.py b/src/babel_xrefs/core/model.py deleted file mode 100644 index 139597f..0000000 --- a/src/babel_xrefs/core/model.py +++ /dev/null @@ -1,2 +0,0 @@ - - From 8e3ba7c2e6828233bb122043ec6f4167acd2fb0a Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 15 Feb 2026 02:17:13 -0500 Subject: [PATCH 017/123] Attempted to rename this package to babel-explorer. --- CLAUDE.md | 33 +++++++++---------- README.md | 2 +- pyproject.toml | 4 +-- .../__init__.py | 0 src/{babel_xrefs => babel_explorer}/cli.py | 8 ++--- .../core/__init__.py | 0 .../core/babel_xrefs.py | 4 +-- .../core/downloader.py | 0 .../core/nodenorm.py | 0 tests/__init__.py | 2 +- tests/test_downloader.py | 2 +- 11 files changed, 26 insertions(+), 29 deletions(-) rename src/{babel_xrefs => babel_explorer}/__init__.py (100%) rename src/{babel_xrefs => babel_explorer}/cli.py (94%) rename src/{babel_xrefs => babel_explorer}/core/__init__.py (100%) rename src/{babel_xrefs => babel_explorer}/core/babel_xrefs.py (97%) rename src/{babel_xrefs => babel_explorer}/core/downloader.py (100%) rename src/{babel_xrefs => babel_explorer}/core/nodenorm.py (100%) diff --git a/CLAUDE.md b/CLAUDE.md index 6f3ed1a..ceb134b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -babel-xrefs is a tool for querying and exploring Babel intermediate files. It allows users to discover why two biological/chemical identifiers are considered identical by the Babel system, which handles cross-references between different ontology and database identifiers (e.g., MONDO, HP, UMLS, HGNC). +babel-explorer is a tool for querying and exploring Babel intermediate files. It allows users to discover why two biological/chemical identifiers are considered identical by the Babel system, which handles cross-references between different ontology and database identifiers (e.g., MONDO, HP, UMLS, HGNC). ## Development Setup @@ -17,11 +17,8 @@ uv sync # Install with dev dependencies uv sync --group dev -# Activate virtual environment (if needed) -source .venv/bin/activate - # Run the CLI -uv run babel-xrefs --help +uv run babel-explorer --help ``` ## Commands @@ -30,22 +27,22 @@ uv run babel-xrefs --help ```bash # Get cross-references for one or more CURIEs -uv run babel-xrefs xrefs MONDO:0004979 +uv run babel-explorer xrefs MONDO:0004979 # Get cross-references with expansion (recursive lookup) -uv run babel-xrefs xrefs MONDO:0004979 --expand +uv run babel-explorer xrefs MONDO:0004979 --expand # Get cross-references with labels from NodeNorm -uv run babel-xrefs xrefs MONDO:0004979 --labels +uv run babel-explorer xrefs MONDO:0004979 --labels # Get ID records for CURIEs -uv run babel-xrefs ids MONDO:0004979 +uv run babel-explorer ids MONDO:0004979 # Test concordance changes with NodeNorm -uv run babel-xrefs test-concord MONDO:0004979 HP:0000001 +uv run babel-explorer test-concord MONDO:0004979 HP:0000001 # Use custom Babel server or local directory -uv run babel-xrefs xrefs MONDO:0004979 --local-dir data/2025nov19 --babel-url https://stars.renci.org:443/var/babel/2025nov19/ +uv run babel-explorer xrefs MONDO:0004979 --local-dir data/2025nov19 --babel-url https://stars.renci.org:443/var/babel/2025nov19/ ``` ### Development Commands @@ -65,25 +62,25 @@ uv run ruff format ### Core Components -1. **BabelDownloader** (`src/babel_xrefs/core/downloader.py`): +1. **BabelDownloader** (`src/babel_explorer/core/downloader.py`): - Downloads Babel intermediate files from a remote server using `wget` - Caches files locally in configurable directory (default: `data/2025nov19/`) - Uses `@functools.lru_cache` to avoid re-downloading - **Important**: Requires `wget` to be installed on the system -2. **BabelXRefs** (`src/babel_xrefs/babel_xrefs.py`): +2. **BabelXRefs** (`src/babel_explorer/babel_xrefs.py`): - Main query engine for cross-references - Uses DuckDB to query Parquet files (`Concord.parquet`, `Identifiers.parquet`, `Metadata.parquet`) - Supports recursive expansion of cross-references - Creates ephemeral DuckDB databases in `data//output/duckdbs/` -3. **NodeNorm** (`src/babel_xrefs/core/nodenorm.py`): +3. **NodeNorm** (`src/babel_explorer/core/nodenorm.py`): - Integration with NodeNormalization API (https://nodenormalization-sri.renci.org/) - Fetches labels, biolink types, and equivalent identifiers for CURIEs - Uses `@functools.lru_cache` for performance - Optional component for label enrichment -4. **CLI** (`src/babel_xrefs/cli.py`): +4. **CLI** (`src/babel_explorer/cli.py`): - Click-based command-line interface - Three main commands: `xrefs`, `ids`, `test-concord` @@ -108,11 +105,11 @@ uv run ruff format - **Data directory**: The `data/` directory is gitignored and contains downloaded Parquet files and generated DuckDB databases - **Babel versions**: The default Babel version is `2025nov19`, but this can be customized via `--local-dir` and `--babel-url` - **No tests yet**: The project currently has pytest configured but no test files exist -- **Empty model.py**: The `src/babel_xrefs/core/model.py` file exists but is currently empty; data classes are defined in `babel_xrefs.py` and `nodenorm.py` instead +- **Empty model.py**: The `src/babel_explorer/core/model.py` file exists but is currently empty; data classes are defined in `babel_explorer.py` and `nodenorm.py` instead ## File Locations -- Source code: `src/babel_xrefs/` +- Source code: `src/babel_explorer/` - Downloaded Babel files: `data//duckdb/*.parquet` - Generated DuckDB databases: `data//output/duckdbs/` -- Entry point: `src/babel_xrefs/cli.py` +- Entry point: `src/babel_explorer/cli.py` diff --git a/README.md b/README.md index 7e78ca5..d17c739 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# babel-xrefs +# babel-explorer Software for querying and exporting Babel intermediate files diff --git a/pyproject.toml b/pyproject.toml index ddc0a95..0fb8f09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "babel-xrefs" +name = "babel-explorer" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -22,4 +22,4 @@ dev = [ ] [project.scripts] -babel-xrefs = "babel_xrefs.cli:cli" +babel-explorer = "babel_explorer.cli:cli" diff --git a/src/babel_xrefs/__init__.py b/src/babel_explorer/__init__.py similarity index 100% rename from src/babel_xrefs/__init__.py rename to src/babel_explorer/__init__.py diff --git a/src/babel_xrefs/cli.py b/src/babel_explorer/cli.py similarity index 94% rename from src/babel_xrefs/cli.py rename to src/babel_explorer/cli.py index 58c4e57..b750830 100644 --- a/src/babel_xrefs/cli.py +++ b/src/babel_explorer/cli.py @@ -1,9 +1,9 @@ -# Command line interface for babel-xrefs +# Command line interface for babel-explorer import click import logging -from babel_xrefs.core.downloader import BabelDownloader -from babel_xrefs.core.babel_xrefs import BabelXRefs -from babel_xrefs.core.nodenorm import NodeNorm +from babel_explorer.core.downloader import BabelDownloader +from babel_explorer.core.babel_xrefs import BabelXRefs +from babel_explorer.core.nodenorm import NodeNorm @click.group() diff --git a/src/babel_xrefs/core/__init__.py b/src/babel_explorer/core/__init__.py similarity index 100% rename from src/babel_xrefs/core/__init__.py rename to src/babel_explorer/core/__init__.py diff --git a/src/babel_xrefs/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py similarity index 97% rename from src/babel_xrefs/core/babel_xrefs.py rename to src/babel_explorer/core/babel_xrefs.py index bf32521..6776a98 100644 --- a/src/babel_xrefs/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -6,8 +6,8 @@ import duckdb import functools -from babel_xrefs.core.downloader import BabelDownloader -from babel_xrefs.core.nodenorm import NodeNorm +from babel_explorer.core.downloader import BabelDownloader +from babel_explorer.core.nodenorm import NodeNorm @dataclasses.dataclass(frozen=True) diff --git a/src/babel_xrefs/core/downloader.py b/src/babel_explorer/core/downloader.py similarity index 100% rename from src/babel_xrefs/core/downloader.py rename to src/babel_explorer/core/downloader.py diff --git a/src/babel_xrefs/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py similarity index 100% rename from src/babel_xrefs/core/nodenorm.py rename to src/babel_explorer/core/nodenorm.py diff --git a/tests/__init__.py b/tests/__init__.py index 61c04ac..588fec0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1 @@ -# Tests for babel-xrefs +# Tests for babel-explorer diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 37f9e77..5ff3a5b 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -10,7 +10,7 @@ import hashlib import pytest from unittest.mock import Mock, patch, MagicMock -from babel_xrefs.core.downloader import BabelDownloader +from babel_explorer.core.downloader import BabelDownloader # Constants for test configuration From cf3394d04ad8d39fe16eb859b2592b4a31f5b113 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 15 Feb 2026 02:44:54 -0500 Subject: [PATCH 018/123] Add comprehensive pytest suite for all core modules - Add IdentifierRecord dataclass to babel_xrefs.py (resolves TODO) - Add 89 tests across 3 files: test_downloader (26), test_babel_xrefs (31), test_nodenorm (23) - Unit tests (71) use mocks and run without network; integration tests (18) use real downloads/APIs - Add session-scoped fixtures in conftest.py for shared Parquet file downloads - Parametrize integration tests over tests/data/valid_curies.txt for easy expansion - Add integration and slow pytest markers to pyproject.toml - Update CLAUDE.md and README.md with testing documentation Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 47 +- README.md | 56 ++- pyproject.toml | 6 + src/babel_explorer/core/babel_xrefs.py | 35 +- tests/conftest.py | 106 ++++ tests/constants.py | 26 + tests/data/valid_curies.txt | 3 + tests/test_babel_xrefs.py | 333 ++++++++++++ tests/test_downloader.py | 669 +++++++++++++------------ tests/test_nodenorm.py | 296 +++++++++++ 10 files changed, 1230 insertions(+), 347 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/constants.py create mode 100644 tests/data/valid_curies.txt create mode 100644 tests/test_babel_xrefs.py create mode 100644 tests/test_nodenorm.py diff --git a/CLAUDE.md b/CLAUDE.md index ceb134b..77f09ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,8 +48,17 @@ uv run babel-explorer xrefs MONDO:0004979 --local-dir data/2025nov19 --babel-url ### Development Commands ```bash -# Run tests -uv run pytest +# Run all tests (includes large file downloads) +uv run pytest -v + +# Run unit tests only (fast, no network) +uv run pytest -v -m "not integration" + +# Run integration tests without 2GB+ downloads +uv run pytest -v -m "integration and not slow" + +# Run a single test file +uv run pytest -v tests/test_nodenorm.py # Run linter uv run ruff check @@ -68,7 +77,7 @@ uv run ruff format - Uses `@functools.lru_cache` to avoid re-downloading - **Important**: Requires `wget` to be installed on the system -2. **BabelXRefs** (`src/babel_explorer/babel_xrefs.py`): +2. **BabelXRefs** (`src/babel_explorer/core/babel_xrefs.py`): - Main query engine for cross-references - Uses DuckDB to query Parquet files (`Concord.parquet`, `Identifiers.parquet`, `Metadata.parquet`) - Supports recursive expansion of cross-references @@ -99,17 +108,43 @@ uv run ruff format - **Recursive expansion**: The `--expand` flag recursively follows all cross-references to build complete graphs - **DuckDB for querying**: In-memory SQL queries against Parquet files for fast lookups +## Testing + +### Test Structure + +Tests live in `tests/` and are split into fast **unit tests** (mocked, no network) and slower **integration tests** (real downloads and API calls). Pytest markers control which tests run: + +- **`@pytest.mark.integration`** — requires network access (downloads Parquet files or calls NodeNorm API) +- **`@pytest.mark.slow`** — downloads very large files (2 GB+) + +| File | Unit | Integration | Slow | Total | +|------|------|-------------|------|-------| +| `tests/test_downloader.py` | 22 | 3 | 1 | 26 | +| `tests/test_babel_xrefs.py` | 22 | 8 | 1 | 31 | +| `tests/test_nodenorm.py` | 18 | 5 | 0 | 23 | + +### Test Infrastructure + +- **`tests/conftest.py`** — Session-scoped fixtures that download Parquet files once and share them across all integration tests. Teardown removes the `data/test/` directory so the next run starts fresh. +- **`tests/constants.py`** — Shared constants (URLs, file paths) and `load_curies()` helper. +- **`tests/data/valid_curies.txt`** — One CURIE per line (`#` comments allowed). Integration tests are parametrized over this list — adding a new line automatically expands test coverage. + +### Key Dataclasses + +- **`CrossReference`** — Frozen dataclass for Concord.parquet rows (filename, subj, pred, obj) +- **`LabeledCrossReference`** — Extends CrossReference with labels and biolink types from NodeNorm +- **`IdentifierRecord`** — Frozen dataclass for Identifiers.parquet rows (curie + dynamic extra fields). Returned by `BabelXRefs.get_curie_ids()`. + ## Important Notes -- **System dependency**: This project requires `wget` to be installed (used by BabelDownloader) - **Data directory**: The `data/` directory is gitignored and contains downloaded Parquet files and generated DuckDB databases - **Babel versions**: The default Babel version is `2025nov19`, but this can be customized via `--local-dir` and `--babel-url` -- **No tests yet**: The project currently has pytest configured but no test files exist -- **Empty model.py**: The `src/babel_explorer/core/model.py` file exists but is currently empty; data classes are defined in `babel_explorer.py` and `nodenorm.py` instead ## File Locations - Source code: `src/babel_explorer/` +- Tests: `tests/` +- Test CURIEs: `tests/data/valid_curies.txt` - Downloaded Babel files: `data//duckdb/*.parquet` - Generated DuckDB databases: `data//output/duckdbs/` - Entry point: `src/babel_explorer/cli.py` diff --git a/README.md b/README.md index d17c739..b545c8c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,56 @@ # babel-explorer -Software for querying and exporting Babel intermediate files +Software for querying and exploring Babel intermediate files. + +babel-explorer allows you to discover why two biological/chemical identifiers are considered identical by the [Babel](https://github.com/TranslatorSRI/Babel) system, which handles cross-references between different ontology and database identifiers (e.g., MONDO, HP, UMLS, HGNC). + +## Setup + +This project uses [uv](https://docs.astral.sh/uv/) for package management: + +```bash +uv sync --group dev +``` + +## Usage + +```bash +# Get cross-references for one or more CURIEs +uv run babel-explorer xrefs MONDO:0004979 + +# Get cross-references with expansion (recursive lookup) +uv run babel-explorer xrefs MONDO:0004979 --expand + +# Get cross-references with labels from NodeNorm +uv run babel-explorer xrefs MONDO:0004979 --labels + +# Get ID records for CURIEs +uv run babel-explorer ids MONDO:0004979 + +# Test concordance changes with NodeNorm +uv run babel-explorer test-concord MONDO:0004979 HP:0000001 +``` + +## Testing + +Tests are split into fast **unit tests** (mocked, no network) and slower **integration tests** (real file downloads and API calls), controlled by pytest markers. + +```bash +# Unit tests only — fast, no network required +uv run pytest -v -m "not integration" + +# Integration tests without 2GB+ downloads +uv run pytest -v -m "integration and not slow" + +# Full suite including large file downloads +uv run pytest -v +``` + +### Adding Test CURIEs + +Integration tests are parametrized over the CURIEs listed in `tests/data/valid_curies.txt`. Add a new CURIE on its own line to automatically expand test coverage: + +``` +# tests/data/valid_curies.txt +MONDO:0004979 +HP:0000001 +``` diff --git a/pyproject.toml b/pyproject.toml index 0fb8f09..922fa1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,9 @@ dev = [ [project.scripts] babel-explorer = "babel_explorer.cli:cli" + +[tool.pytest.ini_options] +markers = [ + "integration: tests requiring network access (deselect with '-m \"not integration\"')", + "slow: tests downloading very large files 2GB+ (deselect with '-m \"not slow\"')", +] diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 6776a98..1e82125 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -44,17 +44,39 @@ def __init__(self, subj: str, pred: str, obj: str, filename: str, subj_label: st def __str__(self): return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", obj_label="{self.obj_label}", subj_label="{self.subj_label}", obj_label="{self.obj_label}")""" +@dataclasses.dataclass(frozen=True) +class IdentifierRecord: + """A record from the Identifiers.parquet file.""" + curie: str + extra_fields: tuple = () + + @staticmethod + def from_row(row: tuple, column_names: list[str]): + """Create an IdentifierRecord from a DuckDB result row and its column names.""" + curie_idx = column_names.index('curie') + extra = tuple( + (col, row[i]) for i, col in enumerate(column_names) if i != curie_idx + ) + return IdentifierRecord(curie=row[curie_idx], extra_fields=extra) + + def __str__(self): + parts = [f"curie={self.curie!r}"] + for name, value in self.extra_fields: + parts.append(f"{name}={value!r}") + return f"IdentifierRecord({', '.join(parts)})" + + class BabelXRefs: def __init__(self, downloader: BabelDownloader, nodenorm: NodeNorm = None): self.downloader = downloader self.nodenorm = nodenorm - def get_curie_ids(self, curies: list[str]): + def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: """ Search for all identifiers in the /ids/ files for a particular CURIE. - :param curie: A CURIE to search for. - :return: A list of cross-references containing that CURIE. + :param curies: A list of CURIEs to search for. + :return: A list of IdentifierRecords containing those CURIEs. """ identifier_parquet = self.downloader.get_downloaded_file('duckdb/Identifiers.parquet') @@ -64,11 +86,10 @@ def get_curie_ids(self, curies: list[str]): duckdb_path = self.downloader.get_output_file('output/duckdbs/xrefs.duckdb') db = duckdb.connect(duckdb_path) identifier_table = db.read_parquet(identifier_parquet) - xrefs = db.execute(f"SELECT * FROM identifier_table WHERE curie IN $1", [curies]) - - # TODO: convert into case classes. + result = db.execute(f"SELECT * FROM identifier_table WHERE curie IN $1", [curies]) - return xrefs.fetchall() + column_names = [desc[0] for desc in result.description] + return [IdentifierRecord.from_row(row, column_names) for row in result.fetchall()] @functools.lru_cache(maxsize=None) def get_curie_xref(self, curie: str, label_curies: bool = False): diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f3df2fe --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,106 @@ +""" +Shared fixtures for babel-explorer tests. + +Session-scoped fixtures download Babel files once and share them across all test modules. +Teardown removes the test data directory so the next run starts fresh. +""" + +import os +import shutil + +import pytest + +from babel_explorer.core.downloader import BabelDownloader +from babel_explorer.core.babel_xrefs import BabelXRefs +from babel_explorer.core.nodenorm import NodeNorm + +from tests.constants import ( + BABEL_URL, + NODENORM_URL, + TEST_DATA_DIR, + CONCORD_FILE, + METADATA_FILE, + IDENTIFIERS_FILE, + load_curies, +) + + +# --------------------------------------------------------------------------- +# Session-scoped fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def valid_curies() -> list[str]: + """Load test CURIEs from tests/data/valid_curies.txt.""" + curies = load_curies() + assert len(curies) > 0, "No CURIEs found in valid_curies.txt" + return curies + + +@pytest.fixture(scope="session") +def test_data_dir(): + """ + Provide a clean test data directory for the entire session. + + Creates the directory before tests, removes it after all tests complete. + """ + if os.path.exists(TEST_DATA_DIR): + shutil.rmtree(TEST_DATA_DIR) + os.makedirs(TEST_DATA_DIR, exist_ok=True) + + yield TEST_DATA_DIR + + if os.path.exists(TEST_DATA_DIR): + shutil.rmtree(TEST_DATA_DIR) + + +@pytest.fixture(scope="session") +def shared_downloader(test_data_dir) -> BabelDownloader: + """A BabelDownloader pointed at the test data directory.""" + return BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + +@pytest.fixture(scope="session") +def downloaded_concord(shared_downloader) -> str: + """Download duckdb/Concord.parquet (~626 MB). Returns the local path.""" + return shared_downloader.get_downloaded_file(CONCORD_FILE) + + +@pytest.fixture(scope="session") +def downloaded_metadata(shared_downloader) -> str: + """Download duckdb/Metadata.parquet (small). Returns the local path.""" + return shared_downloader.get_downloaded_file(METADATA_FILE) + + +@pytest.fixture(scope="session") +def downloaded_parquet_files(downloaded_concord, downloaded_metadata) -> dict[str, str]: + """Dict of {relative_name: local_path} for Concord and Metadata files.""" + return { + CONCORD_FILE: downloaded_concord, + METADATA_FILE: downloaded_metadata, + } + + +@pytest.fixture(scope="session") +def downloaded_identifiers(shared_downloader) -> str: + """Download duckdb/Identifiers.parquet (2 GB+). Returns the local path.""" + return shared_downloader.get_downloaded_file(IDENTIFIERS_FILE) + + +@pytest.fixture(scope="session") +def nodenorm() -> NodeNorm: + """A NodeNorm client pointed at the public API.""" + return NodeNorm(nodenorm_url=NODENORM_URL) + + +@pytest.fixture(scope="session") +def babel_xrefs(shared_downloader, downloaded_parquet_files) -> BabelXRefs: + """A BabelXRefs instance (no NodeNorm) with Concord + Metadata already downloaded.""" + return BabelXRefs(shared_downloader) + + +@pytest.fixture(scope="session") +def babel_xrefs_with_nodenorm(shared_downloader, nodenorm, downloaded_parquet_files) -> BabelXRefs: + """A BabelXRefs instance with NodeNorm, Concord + Metadata already downloaded.""" + return BabelXRefs(shared_downloader, nodenorm) diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 0000000..8366840 --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,26 @@ +"""Shared constants for babel-explorer tests.""" + +import pathlib + +BABEL_URL = "https://stars.renci.org/var/babel/2025nov19/" +NODENORM_URL = "https://nodenormalization-sri.renci.org/" +TEST_DATA_DIR = "data/test" + +# Parquet file paths (relative to the Babel server / local data dir) +CONCORD_FILE = "duckdb/Concord.parquet" +METADATA_FILE = "duckdb/Metadata.parquet" +IDENTIFIERS_FILE = "duckdb/Identifiers.parquet" + +# Path to the valid CURIEs file +VALID_CURIES_PATH = pathlib.Path(__file__).parent / "data" / "valid_curies.txt" + + +def load_curies(path: pathlib.Path = VALID_CURIES_PATH) -> list[str]: + """Read CURIEs from a text file, skipping comments and blank lines.""" + curies = [] + with open(path) as f: + for line in f: + stripped = line.strip() + if stripped and not stripped.startswith("#"): + curies.append(stripped) + return curies diff --git a/tests/data/valid_curies.txt b/tests/data/valid_curies.txt new file mode 100644 index 0000000..9f2f87c --- /dev/null +++ b/tests/data/valid_curies.txt @@ -0,0 +1,3 @@ +# Valid CURIEs for integration tests. +# Add new CURIEs here to expand test coverage — tests are parametrized over this list. +MONDO:0004979 diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py new file mode 100644 index 0000000..052d09c --- /dev/null +++ b/tests/test_babel_xrefs.py @@ -0,0 +1,333 @@ +""" +Tests for BabelXRefs, CrossReference, LabeledCrossReference, and IdentifierRecord. + +Unit tests use mocks; integration tests query real Parquet files via DuckDB. +""" + +import pytest +from unittest.mock import patch, MagicMock + +from babel_explorer.core.babel_xrefs import ( + BabelXRefs, + CrossReference, + LabeledCrossReference, + IdentifierRecord, +) +from babel_explorer.core.downloader import BabelDownloader +from babel_explorer.core.nodenorm import NodeNorm + +from tests.constants import load_curies + +VALID_CURIES = load_curies() + + +# ========================================================================== +# Unit Tests — CrossReference +# ========================================================================== + + +class TestCrossReference: + def test_creation(self): + xr = CrossReference(filename="f.txt", subj="A:1", pred="skos:exactMatch", obj="B:2") + assert xr.filename == "f.txt" + assert xr.subj == "A:1" + assert xr.pred == "skos:exactMatch" + assert xr.obj == "B:2" + + def test_from_tuple(self): + t = ("file.tsv", "MONDO:1", "owl:sameAs", "HP:2") + xr = CrossReference.from_tuple(t) + assert xr.filename == "file.tsv" + assert xr.subj == "MONDO:1" + assert xr.pred == "owl:sameAs" + assert xr.obj == "HP:2" + + def test_curies_property(self): + xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + assert xr.curies == frozenset({"A:1", "B:2"}) + + def test_frozen_immutability(self): + xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + with pytest.raises(AttributeError): + xr.subj = "changed" + + def test_equality(self): + a = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + b = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + assert a == b + + def test_hashability(self): + a = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + b = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + assert hash(a) == hash(b) + assert len({a, b}) == 1 + + def test_lt_ordering(self): + a = CrossReference(filename="a.tsv", subj="A:1", pred="p", obj="B:2") + b = CrossReference(filename="b.tsv", subj="A:1", pred="p", obj="B:2") + assert a < b + + def test_sorting(self): + items = [ + CrossReference(filename="c", subj="C:1", pred="p", obj="D:1"), + CrossReference(filename="a", subj="A:1", pred="p", obj="B:1"), + CrossReference(filename="b", subj="B:1", pred="p", obj="C:1"), + ] + result = sorted(items) + assert [x.filename for x in result] == ["a", "b", "c"] + + +# ========================================================================== +# Unit Tests — LabeledCrossReference +# ========================================================================== + + +class TestLabeledCrossReference: + def test_creation(self): + lxr = LabeledCrossReference( + subj="A:1", pred="p", obj="B:2", filename="f", + subj_label="Alpha", subj_biolink_type="biolink:Disease", + obj_label="Beta", obj_biolink_type="biolink:Gene", + ) + assert lxr.subj == "A:1" + assert lxr.subj_label == "Alpha" + assert lxr.obj_biolink_type == "biolink:Gene" + + def test_inherits_from_cross_reference(self): + lxr = LabeledCrossReference( + subj="A:1", pred="p", obj="B:2", filename="f", + subj_label="", subj_biolink_type="", obj_label="", obj_biolink_type="", + ) + assert isinstance(lxr, CrossReference) + + def test_curies_property(self): + lxr = LabeledCrossReference( + subj="A:1", pred="p", obj="B:2", filename="f", + subj_label="", subj_biolink_type="", obj_label="", obj_biolink_type="", + ) + assert lxr.curies == frozenset({"A:1", "B:2"}) + + def test_str(self): + lxr = LabeledCrossReference( + subj="A:1", pred="p", obj="B:2", filename="f", + subj_label="Alpha", subj_biolink_type="biolink:Disease", + obj_label="Beta", obj_biolink_type="biolink:Gene", + ) + s = str(lxr) + assert "A:1" in s + assert "B:2" in s + assert "Alpha" in s + + +# ========================================================================== +# Unit Tests — IdentifierRecord +# ========================================================================== + + +class TestIdentifierRecord: + def test_creation(self): + rec = IdentifierRecord(curie="MONDO:0004979") + assert rec.curie == "MONDO:0004979" + assert rec.extra_fields == () + + def test_from_row(self): + row = ("MONDO:0004979", "Disease", "asthma") + cols = ["curie", "category", "label"] + rec = IdentifierRecord.from_row(row, cols) + assert rec.curie == "MONDO:0004979" + assert ("category", "Disease") in rec.extra_fields + assert ("label", "asthma") in rec.extra_fields + + def test_frozen(self): + rec = IdentifierRecord(curie="X:1") + with pytest.raises(AttributeError): + rec.curie = "changed" + + def test_str(self): + rec = IdentifierRecord(curie="X:1", extra_fields=(("type", "Gene"),)) + s = str(rec) + assert "X:1" in s + assert "type" in s + assert "Gene" in s + + +# ========================================================================== +# Unit Tests — BabelXRefs (mocked) +# ========================================================================== + + +class TestBabelXRefsInit: + def test_init_without_nodenorm(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + bx = BabelXRefs(dl) + assert bx.downloader is dl + assert bx.nodenorm is None + + def test_init_with_nodenorm(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + nn = NodeNorm("https://example.com/") + bx = BabelXRefs(dl, nn) + assert bx.nodenorm is nn + + +class TestBabelXRefsMocked: + """Mocked query tests — no DuckDB or Parquet files needed.""" + + def _make_bx(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + return BabelXRefs(dl) + + def test_get_curie_xref_calls_downloader(self, tmp_path): + bx = self._make_bx(tmp_path) + mock_result = MagicMock() + mock_result.fetchall.return_value = [ + ("concord.tsv", "A:1", "skos:exactMatch", "B:2"), + ] + mock_db = MagicMock() + mock_db.read_parquet.return_value = "table" + mock_db.execute.return_value = mock_result + + with patch.object(bx.downloader, 'get_downloaded_file', return_value="/fake/path") as mock_dl: + with patch.object(bx.downloader, 'get_output_file', return_value="/fake/db"): + with patch("babel_explorer.core.babel_xrefs.duckdb.connect", return_value=mock_db): + bx.get_curie_xref.cache_clear() + result = bx.get_curie_xref("A:1") + # Downloader should be called for Concord and Metadata + assert mock_dl.call_count == 2 + result_list = list(result) + assert len(result_list) == 1 + assert isinstance(result_list[0], CrossReference) + + def test_get_curie_xrefs_no_expand(self, tmp_path): + bx = self._make_bx(tmp_path) + xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + with patch.object(bx, 'get_curie_xref', return_value=[xr]): + bx.get_curie_xref.cache_clear() + result = bx.get_curie_xrefs(["A:1"], expand=False) + assert len(result) == 1 + assert result[0] == xr + + def test_get_curie_xrefs_with_expand(self, tmp_path): + bx = self._make_bx(tmp_path) + xr1 = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + xr2 = CrossReference(filename="f", subj="B:2", pred="p", obj="C:3") + + def mock_get_curie_xref(curie, label_curies=False): + if curie == "A:1": + return [xr1] + elif curie == "B:2": + return [xr2] + return [] + + with patch.object(bx, 'get_curie_xref', side_effect=mock_get_curie_xref): + result = bx.get_curie_xrefs(["A:1"], expand=True) + assert xr1 in result + assert xr2 in result + + def test_results_are_sorted(self, tmp_path): + bx = self._make_bx(tmp_path) + xr_b = CrossReference(filename="b", subj="B:1", pred="p", obj="C:1") + xr_a = CrossReference(filename="a", subj="A:1", pred="p", obj="B:1") + + with patch.object(bx, 'get_curie_xref', return_value=[xr_b, xr_a]): + result = bx.get_curie_xrefs(["X:1"], expand=False) + assert result == [xr_a, xr_b] + + +# ========================================================================== +# Integration Tests — require downloaded Parquet files +# ========================================================================== + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xref(babel_xrefs, curie): + """get_curie_xref returns non-empty CrossReferences with the queried CURIE.""" + babel_xrefs.get_curie_xref.cache_clear() + results = list(babel_xrefs.get_curie_xref(curie)) + assert len(results) > 0, f"No cross-references found for {curie}" + for xr in results: + assert isinstance(xr, CrossReference) + assert curie in (xr.subj, xr.obj) + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xref_returns_known_xrefs(babel_xrefs, curie): + """At least one cross-reference is found.""" + babel_xrefs.get_curie_xref.cache_clear() + results = list(babel_xrefs.get_curie_xref(curie)) + assert len(results) >= 1 + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xrefs_single_no_expand(babel_xrefs, curie): + """get_curie_xrefs without expansion returns sorted, non-empty results.""" + babel_xrefs.get_curie_xref.cache_clear() + results = babel_xrefs.get_curie_xrefs([curie], expand=False) + assert len(results) > 0 + assert results == sorted(results) + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xrefs_expansion_finds_more(babel_xrefs, curie): + """Expanded results are at least as many as non-expanded.""" + babel_xrefs.get_curie_xref.cache_clear() + non_expanded = babel_xrefs.get_curie_xrefs([curie], expand=False) + babel_xrefs.get_curie_xref.cache_clear() + expanded = babel_xrefs.get_curie_xrefs([curie], expand=True) + assert len(expanded) >= len(non_expanded) + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xrefs_expanded_includes_original(babel_xrefs, curie): + """Non-expanded results are a subset of expanded results.""" + babel_xrefs.get_curie_xref.cache_clear() + non_expanded = set(babel_xrefs.get_curie_xrefs([curie], expand=False)) + babel_xrefs.get_curie_xref.cache_clear() + expanded = set(babel_xrefs.get_curie_xrefs([curie], expand=True)) + assert non_expanded.issubset(expanded) + + +@pytest.mark.integration +def test_get_curie_xref_caching(babel_xrefs): + """Cached calls return the same object.""" + curie = VALID_CURIES[0] + babel_xrefs.get_curie_xref.cache_clear() + r1 = babel_xrefs.get_curie_xref(curie) + r2 = babel_xrefs.get_curie_xref(curie) + assert r1 is r2 + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xref_with_labels(babel_xrefs_with_nodenorm, curie): + """With labels, returns LabeledCrossReference objects.""" + babel_xrefs_with_nodenorm.get_curie_xref.cache_clear() + results = list(babel_xrefs_with_nodenorm.get_curie_xref(curie, label_curies=True)) + assert len(results) > 0 + for xr in results: + assert isinstance(xr, LabeledCrossReference) + + +@pytest.mark.integration +def test_get_curie_xref_nonexistent_curie(babel_xrefs): + """A made-up CURIE returns an empty list.""" + babel_xrefs.get_curie_xref.cache_clear() + results = list(babel_xrefs.get_curie_xref("FAKE:9999999999")) + assert results == [] + + +@pytest.mark.integration +@pytest.mark.slow +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_ids(babel_xrefs, downloaded_identifiers, curie): + """get_curie_ids returns non-empty IdentifierRecord objects.""" + results = babel_xrefs.get_curie_ids([curie]) + assert len(results) > 0 + for rec in results: + assert isinstance(rec, IdentifierRecord) + assert rec.curie == curie diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 5ff3a5b..912cd0a 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -1,396 +1,399 @@ """ Tests for the BabelDownloader class. -These tests verify that the downloader can successfully fetch large Parquet files -from the Babel server and properly manage local file caching with MD5 validation. +Unit tests use mocks and run without network access. +Integration tests download real files from the Babel server. """ -import os -import shutil import hashlib -import pytest -from unittest.mock import Mock, patch, MagicMock -from babel_explorer.core.downloader import BabelDownloader - - -# Constants for test configuration -BABEL_URL = "https://stars.renci.org/var/babel/2025nov19/" -TEST_DATA_DIR = "data/test" -IDENTIFIERS_FILE = "duckdb/Identifiers.parquet" -MINIMUM_FILE_SIZE_GB = 2 -MINIMUM_FILE_SIZE_BYTES = MINIMUM_FILE_SIZE_GB * 1024 * 1024 * 1024 # 2GB in bytes - - -@pytest.fixture(scope="module") -def test_data_dir(): - """ - Fixture that provides a clean test data directory. - - This fixture: - - Creates the test data directory before tests run - - Yields the directory path to tests - - Cleans up (removes) the directory after all tests complete - - Scope is 'module' so the directory persists across all tests in this file, - allowing downloaded files to be reused by multiple tests. - """ - # Setup: ensure clean test directory - if os.path.exists(TEST_DATA_DIR): - shutil.rmtree(TEST_DATA_DIR) - os.makedirs(TEST_DATA_DIR, exist_ok=True) - - yield TEST_DATA_DIR - - # Teardown: remove test directory and all contents - if os.path.exists(TEST_DATA_DIR): - shutil.rmtree(TEST_DATA_DIR) - - -@pytest.fixture(scope="module") -def downloader(test_data_dir): - """ - Fixture that provides a BabelDownloader instance configured for testing. - - Args: - test_data_dir: The test data directory fixture - - Returns: - BabelDownloader: Configured downloader instance - """ - return BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) - - -def test_downloader_initialization(test_data_dir): - """ - Test that BabelDownloader initializes correctly with custom parameters. - - Verifies: - - Downloader accepts URL and local path - - Local path is stored correctly - - Directory is created if it doesn't exist - """ - downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) - - assert downloader.url_base == BABEL_URL - assert downloader.local_path == test_data_dir - assert os.path.exists(test_data_dir) - assert os.path.isdir(test_data_dir) - - -def test_download_large_parquet_file(downloader): - """ - Test downloading a large Parquet file from the Babel server. - - This test: - 1. Downloads the Identifiers.parquet file (2GB+) from the real Babel server - 2. Verifies the file was downloaded successfully - 3. Confirms the file size is at least 2GB +import os +import tempfile - Note: This test takes several minutes to complete due to the large file size. +import pytest +import requests +from unittest.mock import Mock, patch - Args: - downloader: BabelDownloader fixture - """ - # Download the Identifiers.parquet file - downloaded_path = downloader.get_downloaded_file(IDENTIFIERS_FILE) +from babel_explorer.core.downloader import BabelDownloader - # Verify the file exists - assert os.path.exists(downloaded_path), \ - f"Downloaded file does not exist at {downloaded_path}" +from tests.constants import CONCORD_FILE - # Verify it's a file, not a directory - assert os.path.isfile(downloaded_path), \ - f"Downloaded path is not a file: {downloaded_path}" - # Get the file size in bytes - file_size_bytes = os.path.getsize(downloaded_path) - file_size_gb = file_size_bytes / (1024 * 1024 * 1024) +# ========================================================================== +# Unit Tests — no network required +# ========================================================================== + + +class TestBabelDownloaderInit: + """Tests for BabelDownloader constructor.""" + + def test_constructor_stores_url_and_path(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + assert dl.url_base == "https://example.com/" + assert dl.local_path == str(tmp_path) + + def test_creates_directory_if_missing(self, tmp_path): + new_dir = str(tmp_path / "nested" / "dir") + dl = BabelDownloader(url_base="https://example.com/", local_path=new_dir) + assert os.path.isdir(new_dir) + assert dl.local_path == new_dir + + def test_custom_retries(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), retries=3) + assert dl.retries == 3 + + def test_default_retries(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + assert dl.retries == 10 + + def test_invalid_path_raises_value_error(self): + """Using a file path (not a directory) should raise ValueError.""" + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(b"not a directory") + f.flush() + try: + with pytest.raises(ValueError, match="Invalid local_path"): + BabelDownloader(url_base="https://example.com/", local_path=f.name) + finally: + os.unlink(f.name) + + +class TestGetOutputFile: + """Tests for get_output_file.""" + + def test_returns_correct_path(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + result = dl.get_output_file("output/duckdbs/test.duckdb") + assert result == os.path.join(str(tmp_path), "output/duckdbs/test.duckdb") + + def test_creates_parent_directories(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + result = dl.get_output_file("deep/nested/dir/file.txt") + assert os.path.isdir(os.path.dirname(result)) + + def test_lru_caching(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + result1 = dl.get_output_file("some/file.txt") + result2 = dl.get_output_file("some/file.txt") + assert result1 is result2 # identity check — same cached object + + +class TestCalculateMd5: + """Tests for _calculate_md5.""" + + def test_correct_hash(self, tmp_path): + content = b"Hello, world!" + expected = hashlib.md5(content).hexdigest() + file_path = tmp_path / "test.bin" + file_path.write_bytes(content) + + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + assert dl._calculate_md5(str(file_path)) == expected + + def test_different_chunk_sizes_same_result(self, tmp_path): + content = b"A" * 5000 + expected = hashlib.md5(content).hexdigest() + file_path = tmp_path / "chunks.bin" + file_path.write_bytes(content) + + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + assert dl._calculate_md5(str(file_path), chunk_size=100) == expected + assert dl._calculate_md5(str(file_path), chunk_size=4096) == expected + + +class TestFetchRemoteMd5: + """Tests for _fetch_remote_md5.""" + + def _make_dl(self, tmp_path): + return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + + def test_valid_md5_response(self, tmp_path): + dl = self._make_dl(tmp_path) + mock_resp = Mock() + mock_resp.status_code = 200 + mock_resp.text = "d41d8cd98f00b204e9800998ecf8427e filename.parquet\n" + mock_resp.raise_for_status = Mock() + with patch("babel_explorer.core.downloader.requests.get", return_value=mock_resp): + result = dl._fetch_remote_md5("https://example.com/file.md5") + assert result == "d41d8cd98f00b204e9800998ecf8427e" + + def test_hash_only_format(self, tmp_path): + dl = self._make_dl(tmp_path) + mock_resp = Mock() + mock_resp.status_code = 200 + mock_resp.text = "d41d8cd98f00b204e9800998ecf8427e\n" + mock_resp.raise_for_status = Mock() + with patch("babel_explorer.core.downloader.requests.get", return_value=mock_resp): + result = dl._fetch_remote_md5("https://example.com/file.md5") + assert result == "d41d8cd98f00b204e9800998ecf8427e" + + def test_404_returns_none(self, tmp_path): + dl = self._make_dl(tmp_path) + mock_resp = Mock() + mock_resp.status_code = 404 + with patch("babel_explorer.core.downloader.requests.get", return_value=mock_resp): + assert dl._fetch_remote_md5("https://example.com/missing.md5") is None + + def test_malformed_returns_none(self, tmp_path): + dl = self._make_dl(tmp_path) + mock_resp = Mock() + mock_resp.status_code = 200 + mock_resp.text = "not-a-valid-md5-hash\n" + mock_resp.raise_for_status = Mock() + with patch("babel_explorer.core.downloader.requests.get", return_value=mock_resp): + assert dl._fetch_remote_md5("https://example.com/bad.md5") is None + + def test_network_error_returns_none(self, tmp_path): + dl = self._make_dl(tmp_path) + with patch("babel_explorer.core.downloader.requests.get", side_effect=requests.ConnectionError("fail")): + assert dl._fetch_remote_md5("https://example.com/err.md5") is None + + +class TestMd5ValidationFlow: + """Tests for the MD5 validation logic inside get_downloaded_file.""" + + def test_matching_checksum_skips_download(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + test_file = "test.txt" + content = b"test content" + local_path = tmp_path / test_file + local_path.write_bytes(content) + expected_md5 = hashlib.md5(content).hexdigest() + + with patch.object(dl, '_fetch_remote_md5', return_value=expected_md5): + with patch.object(dl, '_download_with_retry') as mock_dl: + dl.get_downloaded_file.cache_clear() + result = dl.get_downloaded_file(test_file) + mock_dl.assert_not_called() + assert result == str(local_path) + + def test_mismatched_checksum_triggers_redownload(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + test_file = "mismatch.txt" + local_path = tmp_path / test_file + local_path.write_bytes(b"wrong content") + correct_content = b"correct content" + expected_md5 = hashlib.md5(correct_content).hexdigest() + + def fake_download(url, path, chunk_size): + with open(path, 'wb') as f: + f.write(correct_content) - # Verify the file is at least 2GB - assert file_size_bytes >= MINIMUM_FILE_SIZE_BYTES, \ - f"Downloaded file is too small: {file_size_gb:.2f}GB (expected at least {MINIMUM_FILE_SIZE_GB}GB)" + with patch.object(dl, '_fetch_remote_md5', return_value=expected_md5): + with patch.object(dl, '_download_with_retry', side_effect=fake_download): + dl.get_downloaded_file.cache_clear() + result = dl.get_downloaded_file(test_file) + assert os.path.exists(result) + with open(result, 'rb') as f: + assert f.read() == correct_content - print(f"\n✓ Successfully downloaded {IDENTIFIERS_FILE}") - print(f" Size: {file_size_gb:.2f}GB ({file_size_bytes:,} bytes)") - print(f" Path: {downloaded_path}") - - -def test_download_caching(downloader): - """ - Test that the downloader uses LRU caching to avoid re-downloading files. + def test_no_md5_proceeds_normally(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + test_file = "no_md5.txt" + content = b"downloaded content" - This test: - 1. Downloads the same file twice - 2. Verifies both calls return the same path - 3. Confirms the file is only downloaded once (via caching) - - Args: - downloader: BabelDownloader fixture - """ - # First download - path1 = downloader.get_downloaded_file(IDENTIFIERS_FILE) - initial_mtime = os.path.getmtime(path1) + def fake_download(url, path, chunk_size): + with open(path, 'wb') as f: + f.write(content) - # Second download - should use cache - path2 = downloader.get_downloaded_file(IDENTIFIERS_FILE) - second_mtime = os.path.getmtime(path2) + with patch.object(dl, '_fetch_remote_md5', return_value=None): + with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: + dl.get_downloaded_file.cache_clear() + result = dl.get_downloaded_file(test_file) + mock_dl.assert_called_once() + assert os.path.exists(result) - # Verify same path returned - assert path1 == path2, "Cached download returned different path" + def test_post_download_validation_fail_raises(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + test_file = "post_fail.txt" + correct_md5 = hashlib.md5(b"expected").hexdigest() - # Verify file wasn't modified (i.e., wasn't re-downloaded) - assert initial_mtime == second_mtime, \ - "File was modified, suggesting it was re-downloaded instead of cached" + def fake_download(url, path, chunk_size): + with open(path, 'wb') as f: + f.write(b"wrong data after download") - print(f"\n✓ Caching works correctly - file not re-downloaded") + with patch.object(dl, '_fetch_remote_md5', return_value=correct_md5): + with patch.object(dl, '_download_with_retry', side_effect=fake_download): + dl.get_downloaded_file.cache_clear() + with pytest.raises(RuntimeError, match="incorrect MD5 checksum"): + dl.get_downloaded_file(test_file) + def test_post_download_validation_pass(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + test_file = "post_pass.txt" + content = b"correct content" + expected_md5 = hashlib.md5(content).hexdigest() -def test_get_output_file(downloader): - """ - Test the get_output_file method for creating output file paths. + def fake_download(url, path, chunk_size): + with open(path, 'wb') as f: + f.write(content) + + with patch.object(dl, '_fetch_remote_md5', return_value=expected_md5): + with patch.object(dl, '_download_with_retry', side_effect=fake_download): + dl.get_downloaded_file.cache_clear() + result = dl.get_downloaded_file(test_file) + assert os.path.exists(result) - This test: - 1. Creates an output file path - 2. Verifies the directory structure is created - 3. Confirms the path is in the correct location - Args: - downloader: BabelDownloader fixture - """ - output_filename = "output/duckdbs/test.duckdb" - output_path = downloader.get_output_file(output_filename) +class TestDownloadWithRetry: + """Tests for _download_with_retry.""" - # Verify the path is correct - expected_path = os.path.join(TEST_DATA_DIR, output_filename) - assert output_path == expected_path, \ - f"Output path mismatch: expected {expected_path}, got {output_path}" - - # Verify the parent directory was created - assert os.path.exists(os.path.dirname(output_path)), \ - "Parent directory for output file was not created" + def test_retries_exhausted_raises_runtime_error(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), retries=2) + with patch("babel_explorer.core.downloader.requests.get", side_effect=requests.ConnectionError("fail")): + with patch("babel_explorer.core.downloader.time.sleep"): # skip waiting + with pytest.raises(RuntimeError, match="Failed to download"): + dl._download_with_retry("https://example.com/file", str(tmp_path / "f"), 1024) - print(f"\n✓ Output file path created correctly: {output_path}") + def test_succeeds_on_second_attempt(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), retries=3) + out_path = str(tmp_path / "retry_success.bin") + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {'Content-Length': '5'} + mock_response.iter_content = Mock(return_value=[b"hello"]) -def test_invalid_local_path(): - """ - Test that BabelDownloader raises an error for invalid local paths. - - This test verifies error handling when attempting to use a file path - as the local directory (should be a directory, not a file). - """ - # Create a temporary file - invalid_path = "/tmp/test_babel_invalid_file.txt" - with open(invalid_path, 'w') as f: - f.write("test") - - try: - # Attempt to create downloader with a file path instead of directory - with pytest.raises(ValueError, match="Invalid local_path"): - BabelDownloader(url_base=BABEL_URL, local_path=invalid_path) - - print("\n✓ Correctly raised ValueError for invalid local path") - finally: - # Clean up - if os.path.exists(invalid_path): - os.remove(invalid_path) - - -def test_md5_validation_matching_checksum(test_data_dir): - """ - Test that MD5 validation skips download when checksums match. - - This test: - 1. Creates a local file with known content - 2. Mocks the .md5 file to return the correct checksum - 3. Verifies the download is skipped (no actual HTTP download occurs) - """ - downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) - - # Create a test file with known content - test_file = "test_file.txt" - local_path = os.path.join(test_data_dir, test_file) - os.makedirs(os.path.dirname(local_path), exist_ok=True) - - test_content = b"This is test content for MD5 validation" - with open(local_path, 'wb') as f: - f.write(test_content) - - # Calculate the expected MD5 - expected_md5 = hashlib.md5(test_content).hexdigest() - - # Mock the _fetch_remote_md5 to return the matching checksum - with patch.object(downloader, '_fetch_remote_md5', return_value=expected_md5): - # Mock _download_with_retry to ensure it's NOT called - with patch.object(downloader, '_download_with_retry') as mock_download: - # Clear the cache before testing - downloader.get_downloaded_file.cache_clear() - - result_path = downloader.get_downloaded_file(test_file) - - # Verify the download was skipped - mock_download.assert_not_called() - assert result_path == local_path - assert os.path.exists(result_path) + side_effects = [requests.ConnectionError("first fail"), mock_response] - print(f"\n✓ MD5 validation correctly skipped download for matching checksum: {expected_md5}") + with patch("babel_explorer.core.downloader.requests.get", side_effect=side_effects): + with patch("babel_explorer.core.downloader.time.sleep"): + dl._download_with_retry("https://example.com/file", out_path, 1024) + assert os.path.exists(out_path) + def test_resume_sends_range_header(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = tmp_path / "partial.bin" + out_path.write_bytes(b"partial") # 7 bytes -def test_md5_validation_mismatched_checksum(test_data_dir): - """ - Test that MD5 validation deletes and re-downloads file when checksums don't match. - - This test: - 1. Creates a local file with wrong content - 2. Mocks the .md5 file to return a different checksum - 3. Verifies the file is deleted and re-downloaded - """ - downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) - - # Create a test file with incorrect content - test_file = "test_file_mismatch.txt" - local_path = os.path.join(test_data_dir, test_file) - os.makedirs(os.path.dirname(local_path), exist_ok=True) - - wrong_content = b"This is WRONG content" - with open(local_path, 'wb') as f: - f.write(wrong_content) - - # Use a different MD5 (this is MD5 of "correct content") - correct_content = b"This is CORRECT content" - expected_md5 = hashlib.md5(correct_content).hexdigest() - - # Track whether file was deleted - original_exists = os.path.exists(local_path) - - # Mock the _fetch_remote_md5 to return the mismatched checksum - with patch.object(downloader, '_fetch_remote_md5', return_value=expected_md5): - # Mock _download_with_retry to create the "correct" file - def mock_download(url, path, chunk_size): - with open(path, 'wb') as f: - f.write(correct_content) - - with patch.object(downloader, '_download_with_retry', side_effect=mock_download): - # Clear the cache before testing - downloader.get_downloaded_file.cache_clear() + mock_response = Mock() + mock_response.status_code = 206 + mock_response.headers = {'Content-Length': '3'} + mock_response.iter_content = Mock(return_value=[b"end"]) - result_path = downloader.get_downloaded_file(test_file) + with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response) as mock_get: + dl._download_with_retry("https://example.com/file", str(out_path), 1024) + _, kwargs = mock_get.call_args + assert kwargs['headers'] == {'Range': 'bytes=7-'} - # Verify the file exists and has correct content - assert os.path.exists(result_path) - with open(result_path, 'rb') as f: - assert f.read() == correct_content + def test_http_416_file_already_complete(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = tmp_path / "complete.bin" + out_path.write_bytes(b"full file") - print(f"\n✓ MD5 validation correctly deleted and re-downloaded file with mismatched checksum") + mock_response = Mock() + mock_response.status_code = 416 + with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): + dl._download_with_retry("https://example.com/file", str(out_path), 1024) + # Should return without error + assert out_path.read_bytes() == b"full file" -def test_md5_validation_no_md5_file(test_data_dir): - """ - Test that download proceeds normally when no .md5 file exists. + def test_server_no_resume_restarts_download(self, tmp_path): + """When server responds 200 (instead of 206), partial file is removed and download restarts.""" + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = tmp_path / "no_resume.bin" + out_path.write_bytes(b"partial") - This test: - 1. Mocks the .md5 file fetch to return None (404) - 2. Verifies the download proceeds normally - """ - downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {'Content-Length': '12'} + mock_response.iter_content = Mock(return_value=[b"full content"]) - test_file = "test_file_no_md5.txt" - local_path = os.path.join(test_data_dir, test_file) + with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): + dl._download_with_retry("https://example.com/file", str(out_path), 1024) + assert out_path.read_bytes() == b"full content" - test_content = b"Test content without MD5 file" - # Mock the _fetch_remote_md5 to return None (no .md5 file) - with patch.object(downloader, '_fetch_remote_md5', return_value=None): - # Mock _download_with_retry to create the file - def mock_download(url, path, chunk_size): - with open(path, 'wb') as f: - f.write(test_content) +class TestStreamDownload: + """Tests for _stream_download.""" - with patch.object(downloader, '_download_with_retry', side_effect=mock_download) as mock_download_method: - # Clear the cache before testing - downloader.get_downloaded_file.cache_clear() + def test_writes_chunks(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = str(tmp_path / "stream.bin") - result_path = downloader.get_downloaded_file(test_file) + mock_response = Mock() + mock_response.headers = {'Content-Length': '10'} + mock_response.iter_content = Mock(return_value=[b"hello", b"world"]) - # Verify download was called (normal download path) - mock_download_method.assert_called_once() - assert os.path.exists(result_path) - with open(result_path, 'rb') as f: - assert f.read() == test_content + dl._stream_download(mock_response, out_path, resume_byte_pos=0, chunk_size=1024) + with open(out_path, 'rb') as f: + assert f.read() == b"helloworld" - print(f"\n✓ Download proceeded normally when no .md5 file exists") + def test_append_mode_on_resume(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = tmp_path / "append.bin" + out_path.write_bytes(b"start") + mock_response = Mock() + mock_response.headers = {'Content-Length': '3'} + mock_response.iter_content = Mock(return_value=[b"end"]) -def test_md5_validation_malformed_md5_file(test_data_dir): - """ - Test that download proceeds normally when .md5 file is malformed. + dl._stream_download(mock_response, str(out_path), resume_byte_pos=5, chunk_size=1024) + assert out_path.read_bytes() == b"startend" - This test: - 1. Mocks the .md5 file fetch to return None (malformed content) - 2. Verifies the download proceeds normally with a warning - """ - downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) - test_file = "test_file_malformed_md5.txt" - local_path = os.path.join(test_data_dir, test_file) +class TestGetDownloadedFileCaching: + """Tests for get_downloaded_file LRU caching.""" - test_content = b"Test content with malformed MD5 file" + def test_cache_returns_same_result(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + content = b"cached content" - # Mock the _fetch_remote_md5 to return None (malformed .md5 file) - with patch.object(downloader, '_fetch_remote_md5', return_value=None): - # Mock _download_with_retry to create the file - def mock_download(url, path, chunk_size): + def fake_download(url, path, chunk_size): with open(path, 'wb') as f: - f.write(test_content) + f.write(content) - with patch.object(downloader, '_download_with_retry', side_effect=mock_download) as mock_download_method: - # Clear the cache before testing - downloader.get_downloaded_file.cache_clear() + with patch.object(dl, '_fetch_remote_md5', return_value=None): + with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: + dl.get_downloaded_file.cache_clear() + r1 = dl.get_downloaded_file("cached.txt") + r2 = dl.get_downloaded_file("cached.txt") + assert r1 == r2 + mock_dl.assert_called_once() # only one actual download - result_path = downloader.get_downloaded_file(test_file) - # Verify download was called (normal download path) - mock_download_method.assert_called_once() - assert os.path.exists(result_path) +class TestGetDownloadedDir: + """Tests for get_downloaded_dir.""" - print(f"\n✓ Download proceeded normally when .md5 file is malformed") + def test_raises_not_implemented(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + dl.get_downloaded_dir.cache_clear() + with pytest.raises(NotImplementedError): + dl.get_downloaded_dir("some/dir") -def test_md5_post_download_validation(test_data_dir): - """ - Test that MD5 validation occurs after download and fails if checksum is wrong. +# ========================================================================== +# Integration Tests — require network access +# ========================================================================== - This test: - 1. Downloads a new file - 2. Mocks the .md5 file to return a checksum - 3. Mocks the download to create a file with WRONG content - 4. Verifies a RuntimeError is raised for checksum mismatch - """ - downloader = BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) - test_file = "test_file_post_validation.txt" - local_path = os.path.join(test_data_dir, test_file) +@pytest.mark.integration +def test_download_concord_parquet(downloaded_concord): + """Verify Concord.parquet downloads and is > 100 MB.""" + assert os.path.isfile(downloaded_concord) + size = os.path.getsize(downloaded_concord) + assert size > 100 * 1024 * 1024, f"Concord.parquet too small: {size} bytes" - # Expected content and MD5 - correct_content = b"Expected content" - expected_md5 = hashlib.md5(correct_content).hexdigest() - # Wrong content that will be downloaded - wrong_content = b"Wrong content downloaded" +@pytest.mark.integration +def test_download_metadata_parquet(downloaded_metadata): + """Verify Metadata.parquet downloads and is non-empty.""" + assert os.path.isfile(downloaded_metadata) + assert os.path.getsize(downloaded_metadata) > 0 - # Mock the _fetch_remote_md5 to return the expected checksum - with patch.object(downloader, '_fetch_remote_md5', return_value=expected_md5): - # Mock _download_with_retry to create a file with WRONG content - def mock_download(url, path, chunk_size): - with open(path, 'wb') as f: - f.write(wrong_content) - with patch.object(downloader, '_download_with_retry', side_effect=mock_download): - # Clear the cache before testing - downloader.get_downloaded_file.cache_clear() +@pytest.mark.integration +def test_download_caching_real_files(shared_downloader, downloaded_concord): + """Second call returns same path and file is not re-downloaded.""" + path2 = shared_downloader.get_downloaded_file(CONCORD_FILE) + assert path2 == downloaded_concord + assert os.path.getmtime(downloaded_concord) == os.path.getmtime(path2) - # Should raise RuntimeError due to post-download MD5 mismatch - with pytest.raises(RuntimeError, match="incorrect MD5 checksum"): - downloader.get_downloaded_file(test_file) - print(f"\n✓ Post-download MD5 validation correctly detected checksum mismatch") +@pytest.mark.integration +@pytest.mark.slow +def test_download_identifiers_parquet(downloaded_identifiers): + """Verify Identifiers.parquet downloads and is > 2 GB.""" + assert os.path.isfile(downloaded_identifiers) + size = os.path.getsize(downloaded_identifiers) + assert size > 2 * 1024 * 1024 * 1024, f"Identifiers.parquet too small: {size} bytes" diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py new file mode 100644 index 0000000..2322eef --- /dev/null +++ b/tests/test_nodenorm.py @@ -0,0 +1,296 @@ +""" +Tests for NodeNorm and Identifier classes. + +Unit tests use mocks; integration tests call the real NodeNorm API. +""" + +import pytest +from unittest.mock import Mock, patch + +import requests + +from babel_explorer.core.nodenorm import NodeNorm, Identifier + +from tests.constants import load_curies + +VALID_CURIES = load_curies() + + +# ========================================================================== +# Unit Tests — Identifier +# ========================================================================== + + +class TestIdentifier: + def test_creation_with_defaults(self): + ident = Identifier(curie="MONDO:0004979") + assert ident.curie == "MONDO:0004979" + assert ident.label == "" + assert ident.biolink_type == "" + assert ident.taxa == [] + assert ident.description == [] + + def test_full_creation(self): + ident = Identifier( + curie="MONDO:0004979", + label="asthma", + biolink_type="biolink:Disease", + taxa=["NCBITaxon:9606"], + description=["A chronic respiratory disease"], + ) + assert ident.label == "asthma" + assert ident.biolink_type == "biolink:Disease" + assert ident.taxa == ["NCBITaxon:9606"] + + def test_from_dict_minimal(self): + d = {"identifier": "X:1"} + ident = Identifier.from_dict(d) + assert ident.curie == "X:1" + assert ident.label == "" + + def test_from_dict_full(self): + d = { + "identifier": "X:1", + "label": "Alpha", + "type": ["biolink:NamedThing"], + "taxa": ["NCBITaxon:9606"], + "description": ["Some thing"], + } + ident = Identifier.from_dict(d) + assert ident.curie == "X:1" + assert ident.label == "Alpha" + assert ident.biolink_type == ["biolink:NamedThing"] + assert ident.taxa == ["NCBITaxon:9606"] + + def test_from_dict_partial(self): + d = {"identifier": "X:1", "label": "Beta"} + ident = Identifier.from_dict(d) + assert ident.curie == "X:1" + assert ident.label == "Beta" + assert ident.biolink_type == "" + + def test_lt_ordering(self): + a = Identifier(curie="A:1") + b = Identifier(curie="B:2") + assert a < b + + def test_sorting(self): + items = [Identifier(curie="C:3"), Identifier(curie="A:1"), Identifier(curie="B:2")] + result = sorted(items) + assert [x.curie for x in result] == ["A:1", "B:2", "C:3"] + + +# ========================================================================== +# Unit Tests — NodeNorm (mocked) +# ========================================================================== + + +class TestNodeNormInit: + def test_default_url(self): + nn = NodeNorm() + assert nn.nodenorm_url == "" + + def test_custom_url(self): + nn = NodeNorm(nodenorm_url="https://custom.api/") + assert nn.nodenorm_url == "https://custom.api/" + + +class TestNormalizeCurieMocked: + def _make_nn(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + nn.normalize_curie.cache_clear() + return nn + + def test_correct_api_endpoint_and_params(self): + nn = self._make_nn() + mock_resp = Mock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"X:1": {"id": {"identifier": "X:1"}}} + mock_resp.raise_for_status = Mock() + + with patch("babel_explorer.core.nodenorm.requests.get", return_value=mock_resp) as mock_get: + nn.normalize_curie("X:1") + mock_get.assert_called_once() + args, kwargs = mock_get.call_args + assert args[0] == "https://example.com/get_normalized_nodes" + assert kwargs["params"]["curie"] == "X:1" + + def test_returns_result_for_curie(self): + nn = self._make_nn() + expected = {"id": {"identifier": "X:1"}, "equivalent_identifiers": []} + mock_resp = Mock() + mock_resp.json.return_value = {"X:1": expected} + mock_resp.raise_for_status = Mock() + + with patch("babel_explorer.core.nodenorm.requests.get", return_value=mock_resp): + result = nn.normalize_curie("X:1") + assert result == expected + + def test_lru_caching(self): + nn = self._make_nn() + mock_resp = Mock() + mock_resp.json.return_value = {"X:1": {"id": "X:1"}} + mock_resp.raise_for_status = Mock() + + with patch("babel_explorer.core.nodenorm.requests.get", return_value=mock_resp) as mock_get: + nn.normalize_curie("X:1") + nn.normalize_curie("X:1") + mock_get.assert_called_once() + + def test_http_error_raises(self): + nn = self._make_nn() + mock_resp = Mock() + mock_resp.raise_for_status.side_effect = requests.HTTPError("500 Server Error") + + with patch("babel_explorer.core.nodenorm.requests.get", return_value=mock_resp): + with pytest.raises(requests.HTTPError): + nn.normalize_curie("BAD:1") + + +class TestGetIdentifierMocked: + def _make_nn(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + nn.normalize_curie.cache_clear() + nn.get_identifier.cache_clear() + return nn + + def test_exact_match_found(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [ + {"identifier": "X:1", "label": "Alpha", "type": ["biolink:Disease"]}, + {"identifier": "X:2", "label": "Beta"}, + ], + } + with patch.object(nn, 'normalize_curie', return_value=api_result): + ident = nn.get_identifier("X:1") + assert ident.curie == "X:1" + assert ident.label == "Alpha" + + def test_no_match_returns_bare_identifier(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [ + {"identifier": "X:2", "label": "Beta"}, + ], + } + with patch.object(nn, 'normalize_curie', return_value=api_result): + ident = nn.get_identifier("X:1") + assert ident.curie == "X:1" + assert ident.label == "" + + def test_falsy_result_returns_bare_identifier(self): + nn = self._make_nn() + with patch.object(nn, 'normalize_curie', return_value=None): + ident = nn.get_identifier("X:1") + assert ident.curie == "X:1" + assert ident.label == "" + + def test_caching(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [ + {"identifier": "X:1", "label": "Alpha"}, + ], + } + with patch.object(nn, 'normalize_curie', return_value=api_result) as mock_norm: + nn.get_identifier("X:1") + nn.get_identifier("X:1") + mock_norm.assert_called_once() + + +class TestGetCliqueIdentifiersMocked: + def _make_nn(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + nn.normalize_curie.cache_clear() + nn.get_clique_identifiers.cache_clear() + return nn + + def test_success_returns_list(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [ + {"identifier": "X:1", "label": "Alpha"}, + {"identifier": "X:2", "label": "Beta"}, + ], + } + with patch.object(nn, 'normalize_curie', return_value=api_result): + result = nn.get_clique_identifiers("X:1") + assert len(result) == 2 + assert all(isinstance(x, Identifier) for x in result) + + def test_missing_key_returns_none(self): + nn = self._make_nn() + api_result = {"id": {"identifier": "X:1"}} # no equivalent_identifiers + with patch.object(nn, 'normalize_curie', return_value=api_result): + result = nn.get_clique_identifiers("X:1") + assert result is None + + def test_caching(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [{"identifier": "X:1"}], + } + with patch.object(nn, 'normalize_curie', return_value=api_result) as mock_norm: + nn.get_clique_identifiers("X:1") + nn.get_clique_identifiers("X:1") + mock_norm.assert_called_once() + + +# ========================================================================== +# Integration Tests — require real NodeNorm API +# ========================================================================== + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_normalize_curie_real_api(nodenorm, curie): + """normalize_curie returns a dict with expected keys.""" + nodenorm.normalize_curie.cache_clear() + result = nodenorm.normalize_curie(curie) + assert isinstance(result, dict) + assert "id" in result + assert "equivalent_identifiers" in result + assert "type" in result + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_identifier_real_api(nodenorm, curie): + """get_identifier returns an Identifier with non-empty label and biolink_type.""" + nodenorm.normalize_curie.cache_clear() + nodenorm.get_identifier.cache_clear() + ident = nodenorm.get_identifier(curie) + assert isinstance(ident, Identifier) + assert ident.curie == curie + assert ident.label != "" + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_clique_identifiers_real_api(nodenorm, curie): + """get_clique_identifiers returns a non-empty list of Identifiers.""" + nodenorm.normalize_curie.cache_clear() + nodenorm.get_clique_identifiers.cache_clear() + result = nodenorm.get_clique_identifiers(curie) + assert result is not None + assert len(result) > 0 + assert all(isinstance(x, Identifier) for x in result) + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_clique_identifiers_has_known_ids(nodenorm, curie): + """At least one equivalent identifier is returned.""" + nodenorm.normalize_curie.cache_clear() + nodenorm.get_clique_identifiers.cache_clear() + result = nodenorm.get_clique_identifiers(curie) + assert len(result) >= 1 + + +@pytest.mark.integration +def test_normalize_curie_nonexistent(nodenorm): + """A made-up CURIE returns None.""" + nodenorm.normalize_curie.cache_clear() + result = nodenorm.normalize_curie("FAKENS:9999999999") + assert result is None From 79500345d03ea85a7cd859c278ad48eb0571e718 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 2 Mar 2026 17:35:29 -0500 Subject: [PATCH 019/123] Added uv.lock (not sure why it wasn't added previously). --- uv.lock | 295 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 uv.lock diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..56af50a --- /dev/null +++ b/uv.lock @@ -0,0 +1,295 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "babel-explorer" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "click" }, + { name = "duckdb" }, + { name = "requests" }, + { name = "tqdm" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.3.1" }, + { name = "duckdb", specifier = ">=1.4.2" }, + { name = "requests", specifier = ">=2.32.5" }, + { name = "tqdm", specifier = ">=4.67.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.3.5" }, + { name = "ruff", specifier = ">=0.11.0" }, +] + +[[package]] +name = "certifi" +version = "2026.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/27/c6491ff4954e58a10f69ad90aca8a1b6fe9c5d3c6f380907af3c37435b59/charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8", size = 206988, upload-time = "2025-10-14T04:40:33.79Z" }, + { url = "https://files.pythonhosted.org/packages/94/59/2e87300fe67ab820b5428580a53cad894272dbb97f38a7a814a2a1ac1011/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0", size = 147324, upload-time = "2025-10-14T04:40:34.961Z" }, + { url = "https://files.pythonhosted.org/packages/07/fb/0cf61dc84b2b088391830f6274cb57c82e4da8bbc2efeac8c025edb88772/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3", size = 142742, upload-time = "2025-10-14T04:40:36.105Z" }, + { url = "https://files.pythonhosted.org/packages/62/8b/171935adf2312cd745d290ed93cf16cf0dfe320863ab7cbeeae1dcd6535f/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc", size = 160863, upload-time = "2025-10-14T04:40:37.188Z" }, + { url = "https://files.pythonhosted.org/packages/09/73/ad875b192bda14f2173bfc1bc9a55e009808484a4b256748d931b6948442/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897", size = 157837, upload-time = "2025-10-14T04:40:38.435Z" }, + { url = "https://files.pythonhosted.org/packages/6d/fc/de9cce525b2c5b94b47c70a4b4fb19f871b24995c728e957ee68ab1671ea/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381", size = 151550, upload-time = "2025-10-14T04:40:40.053Z" }, + { url = "https://files.pythonhosted.org/packages/55/c2/43edd615fdfba8c6f2dfbd459b25a6b3b551f24ea21981e23fb768503ce1/charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815", size = 149162, upload-time = "2025-10-14T04:40:41.163Z" }, + { url = "https://files.pythonhosted.org/packages/03/86/bde4ad8b4d0e9429a4e82c1e8f5c659993a9a863ad62c7df05cf7b678d75/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0", size = 150019, upload-time = "2025-10-14T04:40:42.276Z" }, + { url = "https://files.pythonhosted.org/packages/1f/86/a151eb2af293a7e7bac3a739b81072585ce36ccfb4493039f49f1d3cae8c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161", size = 143310, upload-time = "2025-10-14T04:40:43.439Z" }, + { url = "https://files.pythonhosted.org/packages/b5/fe/43dae6144a7e07b87478fdfc4dbe9efd5defb0e7ec29f5f58a55aeef7bf7/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4", size = 162022, upload-time = "2025-10-14T04:40:44.547Z" }, + { url = "https://files.pythonhosted.org/packages/80/e6/7aab83774f5d2bca81f42ac58d04caf44f0cc2b65fc6db2b3b2e8a05f3b3/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89", size = 149383, upload-time = "2025-10-14T04:40:46.018Z" }, + { url = "https://files.pythonhosted.org/packages/4f/e8/b289173b4edae05c0dde07f69f8db476a0b511eac556dfe0d6bda3c43384/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569", size = 159098, upload-time = "2025-10-14T04:40:47.081Z" }, + { url = "https://files.pythonhosted.org/packages/d8/df/fe699727754cae3f8478493c7f45f777b17c3ef0600e28abfec8619eb49c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224", size = 152991, upload-time = "2025-10-14T04:40:48.246Z" }, + { url = "https://files.pythonhosted.org/packages/1a/86/584869fe4ddb6ffa3bd9f491b87a01568797fb9bd8933f557dba9771beaf/charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a", size = 99456, upload-time = "2025-10-14T04:40:49.376Z" }, + { url = "https://files.pythonhosted.org/packages/65/f6/62fdd5feb60530f50f7e38b4f6a1d5203f4d16ff4f9f0952962c044e919a/charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016", size = 106978, upload-time = "2025-10-14T04:40:50.844Z" }, + { url = "https://files.pythonhosted.org/packages/7a/9d/0710916e6c82948b3be62d9d398cb4fcf4e97b56d6a6aeccd66c4b2f2bd5/charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1", size = 99969, upload-time = "2025-10-14T04:40:52.272Z" }, + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "duckdb" +version = "1.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/9d/ab66a06e416d71b7bdcb9904cdf8d4db3379ef632bb8e9495646702d9718/duckdb-1.4.4.tar.gz", hash = "sha256:8bba52fd2acb67668a4615ee17ee51814124223de836d9e2fdcbc4c9021b3d3c", size = 18419763, upload-time = "2026-01-26T11:50:37.68Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/68/19233412033a2bc5a144a3f531f64e3548d4487251e3f16b56c31411a06f/duckdb-1.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5ba684f498d4e924c7e8f30dd157da8da34c8479746c5011b6c0e037e9c60ad2", size = 28883816, upload-time = "2026-01-26T11:49:01.009Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3e/cec70e546c298ab76d80b990109e111068d82cca67942c42328eaa7d6fdb/duckdb-1.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5536eb952a8aa6ae56469362e344d4e6403cc945a80bc8c5c2ebdd85d85eb64b", size = 15339662, upload-time = "2026-01-26T11:49:04.058Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f0/cf4241a040ec4f571859a738007ec773b642fbc27df4cbcf34b0c32ea559/duckdb-1.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:47dd4162da6a2be59a0aef640eb08d6360df1cf83c317dcc127836daaf3b7f7c", size = 13670044, upload-time = "2026-01-26T11:49:06.627Z" }, + { url = "https://files.pythonhosted.org/packages/11/64/de2bb4ec1e35ec9ebf6090a95b930fc56934a0ad6f34a24c5972a14a77ef/duckdb-1.4.4-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6cb357cfa3403910e79e2eb46c8e445bb1ee2fd62e9e9588c6b999df4256abc1", size = 18409951, upload-time = "2026-01-26T11:49:09.808Z" }, + { url = "https://files.pythonhosted.org/packages/79/a2/ac0f5ee16df890d141304bcd48733516b7202c0de34cd3555634d6eb4551/duckdb-1.4.4-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c25d5b0febda02b7944e94fdae95aecf952797afc8cb920f677b46a7c251955", size = 20411739, upload-time = "2026-01-26T11:49:12.652Z" }, + { url = "https://files.pythonhosted.org/packages/37/a2/9a3402edeedaecf72de05fe9ff7f0303d701b8dfc136aea4a4be1a5f7eee/duckdb-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:6703dd1bb650025b3771552333d305d62ddd7ff182de121483d4e042ea6e2e00", size = 12256972, upload-time = "2026-01-26T11:49:15.468Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/052ea6dcdf35b259fd182eff3efd8d75a071de4010c9807556098df137b9/duckdb-1.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:bf138201f56e5d6fc276a25138341b3523e2f84733613fc43f02c54465619a95", size = 13006696, upload-time = "2026-01-26T11:49:18.054Z" }, + { url = "https://files.pythonhosted.org/packages/58/33/beadaa69f8458afe466126f2c5ee48c4759cc9d5d784f8703d44e0b52c3c/duckdb-1.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ddcfd9c6ff234da603a1edd5fd8ae6107f4d042f74951b65f91bc5e2643856b3", size = 28896535, upload-time = "2026-01-26T11:49:21.232Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/82413f386df10467affc87f65bac095b7c88dbd9c767584164d5f4dc4cb8/duckdb-1.4.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6792ca647216bd5c4ff16396e4591cfa9b4a72e5ad7cdd312cec6d67e8431a7c", size = 15349716, upload-time = "2026-01-26T11:49:23.989Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/c13d396fd4e9bf970916dc5b4fea410c1b10fe531069aea65f1dcf849a71/duckdb-1.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1f8d55843cc940e36261689054f7dfb6ce35b1f5b0953b0d355b6adb654b0d52", size = 13672403, upload-time = "2026-01-26T11:49:26.741Z" }, + { url = "https://files.pythonhosted.org/packages/db/77/2446a0b44226bb95217748d911c7ca66a66ca10f6481d5178d9370819631/duckdb-1.4.4-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c65d15c440c31e06baaebfd2c06d71ce877e132779d309f1edf0a85d23c07e92", size = 18419001, upload-time = "2026-01-26T11:49:29.353Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a3/97715bba30040572fb15d02c26f36be988d48bc00501e7ac02b1d65ef9d0/duckdb-1.4.4-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b297eff642503fd435a9de5a9cb7db4eccb6f61d61a55b30d2636023f149855f", size = 20437385, upload-time = "2026-01-26T11:49:32.302Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0a/18b9167adf528cbe3867ef8a84a5f19f37bedccb606a8a9e59cfea1880c8/duckdb-1.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:d525de5f282b03aa8be6db86b1abffdceae5f1055113a03d5b50cd2fb8cf2ef8", size = 12267343, upload-time = "2026-01-26T11:49:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/f8/15/37af97f5717818f3d82d57414299c293b321ac83e048c0a90bb8b6a09072/duckdb-1.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:50f2eb173c573811b44aba51176da7a4e5c487113982be6a6a1c37337ec5fa57", size = 13007490, upload-time = "2026-01-26T11:49:37.413Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fe/64810fee20030f2bf96ce28b527060564864ce5b934b50888eda2cbf99dd/duckdb-1.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:337f8b24e89bc2e12dadcfe87b4eb1c00fd920f68ab07bc9b70960d6523b8bc3", size = 28899349, upload-time = "2026-01-26T11:49:40.294Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9b/3c7c5e48456b69365d952ac201666053de2700f5b0144a699a4dc6854507/duckdb-1.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0509b39ea7af8cff0198a99d206dca753c62844adab54e545984c2e2c1381616", size = 15350691, upload-time = "2026-01-26T11:49:43.242Z" }, + { url = "https://files.pythonhosted.org/packages/a6/7b/64e68a7b857ed0340045501535a0da99ea5d9d5ea3708fec0afb8663eb27/duckdb-1.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fb94de6d023de9d79b7edc1ae07ee1d0b4f5fa8a9dcec799650b5befdf7aafec", size = 13672311, upload-time = "2026-01-26T11:49:46.069Z" }, + { url = "https://files.pythonhosted.org/packages/09/5b/3e7aa490841784d223de61beb2ae64e82331501bf5a415dc87a0e27b4663/duckdb-1.4.4-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d636ceda422e7babd5e2f7275f6a0d1a3405e6a01873f00d38b72118d30c10b", size = 18422740, upload-time = "2026-01-26T11:49:49.034Z" }, + { url = "https://files.pythonhosted.org/packages/53/32/256df3dbaa198c58539ad94f9a41e98c2c8ff23f126b8f5f52c7dcd0a738/duckdb-1.4.4-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7df7351328ffb812a4a289732f500d621e7de9942a3a2c9b6d4afcf4c0e72526", size = 20435578, upload-time = "2026-01-26T11:49:51.946Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f0/620323fd87062ea43e527a2d5ed9e55b525e0847c17d3b307094ddab98a2/duckdb-1.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:6fb1225a9ea5877421481d59a6c556a9532c32c16c7ae6ca8d127e2b878c9389", size = 12268083, upload-time = "2026-01-26T11:49:54.615Z" }, + { url = "https://files.pythonhosted.org/packages/e5/07/a397fdb7c95388ba9c055b9a3d38dfee92093f4427bc6946cf9543b1d216/duckdb-1.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:f28a18cc790217e5b347bb91b2cab27aafc557c58d3d8382e04b4fe55d0c3f66", size = 13006123, upload-time = "2026-01-26T11:49:57.092Z" }, + { url = "https://files.pythonhosted.org/packages/97/a6/f19e2864e651b0bd8e4db2b0c455e7e0d71e0d4cd2cd9cc052f518e43eb3/duckdb-1.4.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:25874f8b1355e96178079e37312c3ba6d61a2354f51319dae860cf21335c3a20", size = 28909554, upload-time = "2026-01-26T11:50:00.107Z" }, + { url = "https://files.pythonhosted.org/packages/0e/93/8a24e932c67414fd2c45bed83218e62b73348996bf859eda020c224774b2/duckdb-1.4.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:452c5b5d6c349dc5d1154eb2062ee547296fcbd0c20e9df1ed00b5e1809089da", size = 15353804, upload-time = "2026-01-26T11:50:03.382Z" }, + { url = "https://files.pythonhosted.org/packages/62/13/e5378ff5bb1d4397655d840b34b642b1b23cdd82ae19599e62dc4b9461c9/duckdb-1.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8e5c2d8a0452df55e092959c0bfc8ab8897ac3ea0f754cb3b0ab3e165cd79aff", size = 13676157, upload-time = "2026-01-26T11:50:06.232Z" }, + { url = "https://files.pythonhosted.org/packages/2d/94/24364da564b27aeebe44481f15bd0197a0b535ec93f188a6b1b98c22f082/duckdb-1.4.4-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1af6e76fe8bd24875dc56dd8e38300d64dc708cd2e772f67b9fbc635cc3066a3", size = 18426882, upload-time = "2026-01-26T11:50:08.97Z" }, + { url = "https://files.pythonhosted.org/packages/26/0a/6ae31b2914b4dc34243279b2301554bcbc5f1a09ccc82600486c49ab71d1/duckdb-1.4.4-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0440f59e0cd9936a9ebfcf7a13312eda480c79214ffed3878d75947fc3b7d6d", size = 20435641, upload-time = "2026-01-26T11:50:12.188Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b1/fd5c37c53d45efe979f67e9bd49aaceef640147bb18f0699a19edd1874d6/duckdb-1.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:59c8d76016dde854beab844935b1ec31de358d4053e792988108e995b18c08e7", size = 12762360, upload-time = "2026-01-26T11:50:14.76Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2d/13e6024e613679d8a489dd922f199ef4b1d08a456a58eadd96dc2f05171f/duckdb-1.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:53cd6423136ab44383ec9955aefe7599b3fb3dd1fe006161e6396d8167e0e0d4", size = 13458633, upload-time = "2026-01-26T11:50:17.657Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/04/eab13a954e763b0606f460443fcbf6bb5a0faf06890ea3754ff16523dce5/ruff-0.15.2.tar.gz", hash = "sha256:14b965afee0969e68bb871eba625343b8673375f457af4abe98553e8bbb98342", size = 4558148, upload-time = "2026-02-19T22:32:20.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/70/3a4dc6d09b13cb3e695f28307e5d889b2e1a66b7af9c5e257e796695b0e6/ruff-0.15.2-py3-none-linux_armv6l.whl", hash = "sha256:120691a6fdae2f16d65435648160f5b81a9625288f75544dc40637436b5d3c0d", size = 10430565, upload-time = "2026-02-19T22:32:41.824Z" }, + { url = "https://files.pythonhosted.org/packages/71/0b/bb8457b56185ece1305c666dc895832946d24055be90692381c31d57466d/ruff-0.15.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a89056d831256099658b6bba4037ac6dd06f49d194199215befe2bb10457ea5e", size = 10820354, upload-time = "2026-02-19T22:32:07.366Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c1/e0532d7f9c9e0b14c46f61b14afd563298b8b83f337b6789ddd987e46121/ruff-0.15.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e36dee3a64be0ebd23c86ffa3aa3fd3ac9a712ff295e192243f814a830b6bd87", size = 10170767, upload-time = "2026-02-19T22:32:13.188Z" }, + { url = "https://files.pythonhosted.org/packages/47/e8/da1aa341d3af017a21c7a62fb5ec31d4e7ad0a93ab80e3a508316efbcb23/ruff-0.15.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9fb47b6d9764677f8c0a193c0943ce9a05d6763523f132325af8a858eadc2b9", size = 10529591, upload-time = "2026-02-19T22:32:02.547Z" }, + { url = "https://files.pythonhosted.org/packages/93/74/184fbf38e9f3510231fbc5e437e808f0b48c42d1df9434b208821efcd8d6/ruff-0.15.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f376990f9d0d6442ea9014b19621d8f2aaf2b8e39fdbfc79220b7f0c596c9b80", size = 10260771, upload-time = "2026-02-19T22:32:36.938Z" }, + { url = "https://files.pythonhosted.org/packages/05/ac/605c20b8e059a0bc4b42360414baa4892ff278cec1c91fff4be0dceedefd/ruff-0.15.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dcc987551952d73cbf5c88d9fdee815618d497e4df86cd4c4824cc59d5dd75f", size = 11045791, upload-time = "2026-02-19T22:32:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/fd/52/db6e419908f45a894924d410ac77d64bdd98ff86901d833364251bd08e22/ruff-0.15.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:42a47fd785cbe8c01b9ff45031af875d101b040ad8f4de7bbb716487c74c9a77", size = 11879271, upload-time = "2026-02-19T22:32:29.305Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d8/7992b18f2008bdc9231d0f10b16df7dda964dbf639e2b8b4c1b4e91b83af/ruff-0.15.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbe9f49354866e575b4c6943856989f966421870e85cd2ac94dccb0a9dcb2fea", size = 11303707, upload-time = "2026-02-19T22:32:22.492Z" }, + { url = "https://files.pythonhosted.org/packages/d7/02/849b46184bcfdd4b64cde61752cc9a146c54759ed036edd11857e9b8443b/ruff-0.15.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7a672c82b5f9887576087d97be5ce439f04bbaf548ee987b92d3a7dede41d3a", size = 11149151, upload-time = "2026-02-19T22:32:44.234Z" }, + { url = "https://files.pythonhosted.org/packages/70/04/f5284e388bab60d1d3b99614a5a9aeb03e0f333847e2429bebd2aaa1feec/ruff-0.15.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ecc64f46f7019e2bcc3cdc05d4a7da958b629a5ab7033195e11a438403d956", size = 11091132, upload-time = "2026-02-19T22:32:24.691Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ae/88d844a21110e14d92cf73d57363fab59b727ebeabe78009b9ccb23500af/ruff-0.15.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8dcf243b15b561c655c1ef2f2b0050e5d50db37fe90115507f6ff37d865dc8b4", size = 10504717, upload-time = "2026-02-19T22:32:26.75Z" }, + { url = "https://files.pythonhosted.org/packages/64/27/867076a6ada7f2b9c8292884ab44d08fd2ba71bd2b5364d4136f3cd537e1/ruff-0.15.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dab6941c862c05739774677c6273166d2510d254dac0695c0e3f5efa1b5585de", size = 10263122, upload-time = "2026-02-19T22:32:10.036Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ef/faf9321d550f8ebf0c6373696e70d1758e20ccdc3951ad7af00c0956be7c/ruff-0.15.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1b9164f57fc36058e9a6806eb92af185b0697c9fe4c7c52caa431c6554521e5c", size = 10735295, upload-time = "2026-02-19T22:32:39.227Z" }, + { url = "https://files.pythonhosted.org/packages/2f/55/e8089fec62e050ba84d71b70e7834b97709ca9b7aba10c1a0b196e493f97/ruff-0.15.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:80d24fcae24d42659db7e335b9e1531697a7102c19185b8dc4a028b952865fd8", size = 11241641, upload-time = "2026-02-19T22:32:34.617Z" }, + { url = "https://files.pythonhosted.org/packages/23/01/1c30526460f4d23222d0fabd5888868262fd0e2b71a00570ca26483cd993/ruff-0.15.2-py3-none-win32.whl", hash = "sha256:fd5ff9e5f519a7e1bd99cbe8daa324010a74f5e2ebc97c6242c08f26f3714f6f", size = 10507885, upload-time = "2026-02-19T22:32:15.635Z" }, + { url = "https://files.pythonhosted.org/packages/5c/10/3d18e3bbdf8fc50bbb4ac3cc45970aa5a9753c5cb51bf9ed9a3cd8b79fa3/ruff-0.15.2-py3-none-win_amd64.whl", hash = "sha256:d20014e3dfa400f3ff84830dfb5755ece2de45ab62ecea4af6b7262d0fb4f7c5", size = 11623725, upload-time = "2026-02-19T22:32:04.947Z" }, + { url = "https://files.pythonhosted.org/packages/6d/78/097c0798b1dab9f8affe73da9642bb4500e098cb27fd8dc9724816ac747b/ruff-0.15.2-py3-none-win_arm64.whl", hash = "sha256:cabddc5822acdc8f7b5527b36ceac55cc51eec7b1946e60181de8fe83ca8876e", size = 10941649, upload-time = "2026-02-19T22:32:18.108Z" }, +] + +[[package]] +name = "tqdm" +version = "4.67.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] From 47535d3d723bd70b7260aa971b131560d150585e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 2 Mar 2026 17:36:52 -0500 Subject: [PATCH 020/123] Update CLAUDE.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 77f09ed..9fb436f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,10 +72,10 @@ uv run ruff format ### Core Components 1. **BabelDownloader** (`src/babel_explorer/core/downloader.py`): - - Downloads Babel intermediate files from a remote server using `wget` + - Downloads Babel intermediate files from a remote HTTP(S) server using Python's `requests` library (streaming downloads) - Caches files locally in configurable directory (default: `data/2025nov19/`) - Uses `@functools.lru_cache` to avoid re-downloading - - **Important**: Requires `wget` to be installed on the system + - **Important**: Requires network access but no external tools like `wget` 2. **BabelXRefs** (`src/babel_explorer/core/babel_xrefs.py`): - Main query engine for cross-references From a0719fa8ce658a6a3a1fdb4a134d161657bffe39 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 2 Mar 2026 17:38:01 -0500 Subject: [PATCH 021/123] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 922fa1b..eafcfc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "babel-explorer" version = "0.1.0" -description = "Add your description here" +description = "Tool for querying and exploring Babel APIs and intermediate files" readme = "README.md" requires-python = ">=3.11" dependencies = [ From ab67878b0b093286e766d46dab82455c9e5dfd97 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 2 Mar 2026 17:46:17 -0500 Subject: [PATCH 022/123] Update src/babel_explorer/core/babel_xrefs.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/core/babel_xrefs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 1e82125..95fda85 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -42,7 +42,7 @@ def __init__(self, subj: str, pred: str, obj: str, filename: str, subj_label: st self.obj_biolink_type = obj_biolink_type def __str__(self): - return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", obj_label="{self.obj_label}", subj_label="{self.subj_label}", obj_label="{self.obj_label}")""" + return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", subj_biolink_type="{self.subj_biolink_type}", obj_label="{self.obj_label}", obj_biolink_type="{self.obj_biolink_type}")""" @dataclasses.dataclass(frozen=True) class IdentifierRecord: From 96523caa30c812b6c26f5aed1ee9d0bfa6947fbc Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 2 Mar 2026 17:47:47 -0500 Subject: [PATCH 023/123] Update src/babel_explorer/core/nodenorm.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/core/nodenorm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index 018f106..a9c6752 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -32,7 +32,7 @@ def __init__(self, nodenorm_url: str=""): self.nodenorm_url = nodenorm_url @functools.lru_cache(maxsize=None) - def get_identifier(self, curie): + def get_identifier(self, curie: str): result = self.normalize_curie(curie) logging.debug(f"Normalizing {curie} with NodeNorm to result: {result}") if not result: From bf1173218d99d3a729a205e91d521302cb872bea Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 2 Mar 2026 18:26:05 -0500 Subject: [PATCH 024/123] Replace MD5 checksumming with HTTP header caching and freshness window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove _calculate_md5/_fetch_remote_md5 (too slow on 2.5-3.9 GB files) - Add sidecar .meta JSON files (ETag, Last-Modified, Content-Length, last_checked) - Three-tier logic: freshness window → HEAD/ETag check → full re-download - Add freshness_seconds param to BabelDownloader (default 3h) - Add --check-download CLI option to xrefs and ids commands (e.g. 3h, never) - Update tests: replace MD5 test classes with meta/ETag/tier coverage Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/cli.py | 28 +- src/babel_explorer/core/downloader.py | 203 ++++++----- tests/test_downloader.py | 463 +++++++++++++++++++------- 3 files changed, 490 insertions(+), 204 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index b750830..5acdcac 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -6,6 +6,17 @@ from babel_explorer.core.nodenorm import NodeNorm +def parse_duration(value: str) -> float: + """Parse a duration string like '3h', '30m', '1d', '7200', or 'never' → seconds.""" + units = {"s": 1, "m": 60, "h": 3600, "d": 86400} + lower = value.lower() + if lower == "never": + return float("inf") + if lower[-1] in units: + return int(lower[:-1]) * units[lower[-1]] + return int(lower) # bare seconds + + @click.group() def cli(): pass @@ -17,7 +28,11 @@ def cli(): @click.option("--nodenorm-url", type=str, default="https://nodenormalization-sri.renci.org/", help="NodeNorm URL to check for concord changes") @click.option("--expand", is_flag=True, help="Also display xrefs for returned CURIEs") @click.option("--labels", is_flag=True, help="Include labels for CURIEs") -def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, expand: bool, labels: bool): +@click.option("--check-download", type=str, default="3h", show_default=True, + help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " + "'never' always checks via HTTP HEAD; '0' same as 'never'.") +def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, expand: bool, labels: bool, + check_download: str): """ Fetches and prints the cross-references (xrefs) for the given CURIEs. @@ -35,7 +50,8 @@ def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, expan """ logging.basicConfig(level=logging.INFO) - bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir), NodeNorm(nodenorm_url)) + freshness = parse_duration(check_download) + bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness), NodeNorm(nodenorm_url)) xrefs = bxref.get_curie_xrefs(curies, expand, label_curies=labels) for xref in xrefs: print(xref) @@ -44,7 +60,10 @@ def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, expan @click.argument("curies", type=str, required=True, nargs=-1) @click.option("--local-dir", type=str, default="data/2025nov19", help="Local location to save Babel download files to") @click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") -def ids(curies: list[str], babel_url: str, local_dir: str): +@click.option("--check-download", type=str, default="3h", show_default=True, + help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " + "'never' always checks via HTTP HEAD; '0' same as 'never'.") +def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): """ Fetches and prints the ID records for the given CURIEs, along with Biolink type if provided. @@ -60,7 +79,8 @@ def ids(curies: list[str], babel_url: str, local_dir: str): """ logging.basicConfig(level=logging.INFO) - bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir)) + freshness = parse_duration(check_download) + bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness)) xrefs = bxref.get_curie_ids(curies) for xref in xrefs: print(xref) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 93081c6..43c3daf 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -1,9 +1,10 @@ import functools +import json import os import urllib.parse import time -import hashlib import requests +from datetime import datetime, timezone from tqdm import tqdm import logging @@ -13,10 +14,11 @@ class BabelDownloader: Class for downloading Babel cross-reference files to a local directory as needed. """ - def __init__(self, url_base, local_path=None, retries=10): + def __init__(self, url_base, local_path=None, retries=10, freshness_seconds=3 * 3600): # We assume the URL base is correct (if not, we can fix it later). self.url_base = url_base self.retries = retries + self.freshness_seconds = freshness_seconds self.logger = logging.getLogger(BabelDownloader.__name__) if local_path is None: @@ -41,55 +43,115 @@ def get_output_file(self, filename): os.makedirs(os.path.dirname(filepath), exist_ok=True) return filepath - def _calculate_md5(self, file_path, chunk_size=1024*1024): + def _get_meta_path(self, local_path): + """Return the sidecar metadata file path for a given local file.""" + return local_path + ".meta" + + def _load_meta(self, local_path): + """Load sidecar metadata JSON, or return None if not found/invalid.""" + meta_path = self._get_meta_path(local_path) + if not os.path.exists(meta_path): + return None + try: + with open(meta_path, "r") as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return None + + def _save_meta(self, local_path, headers, update_last_checked=True): + """ + Write a sidecar .meta JSON file next to local_path. + + Args: + local_path: Path to the downloaded file + headers: Response headers dict (or requests.structures.CaseInsensitiveDict) + update_last_checked: If True, set last_checked to now + """ + meta = {} + if "ETag" in headers: + meta["etag"] = headers["ETag"] + if "Last-Modified" in headers: + meta["last_modified"] = headers["Last-Modified"] + if "Content-Length" in headers: + meta["content_length"] = int(headers["Content-Length"]) + if update_last_checked: + meta["last_checked"] = datetime.now(timezone.utc).isoformat() + + meta_path = self._get_meta_path(local_path) + with open(meta_path, "w") as f: + json.dump(meta, f, indent=2) + + def _is_within_freshness(self, meta, freshness_seconds): """ - Calculate MD5 checksum of a file. + Return True if last_checked is within freshness_seconds of now. Args: - file_path: Path to the file to checksum - chunk_size: Size of chunks to read (default 1MB) + meta: dict loaded from .meta file + freshness_seconds: Number of seconds; float('inf') means always fresh Returns: - str: Hexadecimal MD5 checksum + bool """ - md5_hash = hashlib.md5() - with open(file_path, 'rb') as f: - for chunk in iter(lambda: f.read(chunk_size), b''): - md5_hash.update(chunk) - return md5_hash.hexdigest() + if freshness_seconds == float("inf"): + return True + last_checked_str = meta.get("last_checked") + if not last_checked_str: + return False + try: + last_checked = datetime.fromisoformat(last_checked_str) + age = (datetime.now(timezone.utc) - last_checked).total_seconds() + return age < freshness_seconds + except (ValueError, TypeError): + return False - def _fetch_remote_md5(self, url): + def _etag_matches(self, url, meta): """ - Fetch MD5 checksum from remote .md5 file. + Do a HEAD request and check if the ETag (or Last-Modified + Content-Length) + matches the stored metadata. If they match, update last_checked in the .meta file. Args: - url: URL to the .md5 file + url: URL to HEAD + meta: dict loaded from .meta file (may have etag, last_modified, content_length) Returns: - str: MD5 checksum if found, None if file doesn't exist or is malformed + bool: True if remote matches local meta (file is still current) """ try: - response = requests.get(url, timeout=10) - if response.status_code == 404: - self.logger.debug(f"No .md5 file found at {url}") - return None + response = requests.head(url, timeout=30) response.raise_for_status() + except requests.RequestException as e: + self.logger.warning(f"HEAD request failed for {url}: {e}") + return False + + remote_headers = response.headers + + # Primary check: ETag + local_etag = meta.get("etag") + remote_etag = remote_headers.get("ETag") + if local_etag and remote_etag: + if local_etag == remote_etag: + self.logger.info(f"ETag matches ({remote_etag}), file is current") + # Update last_checked in the .meta file + # We need the local_path to update — derive it from URL + # Caller will handle updating; return True + return True + else: + self.logger.info(f"ETag changed: {local_etag!r} → {remote_etag!r}, re-downloading") + return False - # Parse MD5 file content - # Format is typically: "md5hash filename" or just "md5hash" - content = response.text.strip() - md5_match = content.split()[0] # Take first token + # Fallback: Last-Modified + Content-Length + local_lm = meta.get("last_modified") + remote_lm = remote_headers.get("Last-Modified") + local_cl = meta.get("content_length") + remote_cl = remote_headers.get("Content-Length") - # Validate it's a valid MD5 (32 hex characters) - if len(md5_match) == 32 and all(c in '0123456789abcdef' for c in md5_match.lower()): - return md5_match.lower() - else: - self.logger.warning(f"Malformed .md5 file at {url}: {content}") - return None + if local_lm and remote_lm and local_lm == remote_lm: + if local_cl is None or remote_cl is None or int(remote_cl) == local_cl: + self.logger.info(f"Last-Modified matches ({remote_lm}), file is current") + return True - except requests.RequestException as e: - self.logger.debug(f"Could not fetch .md5 file from {url}: {e}") - return None + self.logger.info("Cannot confirm file is current (no matching ETag or Last-Modified), will re-download") + return False def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): """ @@ -134,6 +196,9 @@ def _download_with_retry(self, url, local_path, chunk_size): local_path: Local file path to save to chunk_size: Size of chunks to read/write + Returns: + requests.structures.CaseInsensitiveDict: Response headers from the final request + Raises: RuntimeError: If all retry attempts fail """ @@ -157,7 +222,7 @@ def _download_with_retry(self, url, local_path, chunk_size): if response.status_code == 416: # Range Not Satisfiable - file already complete self.logger.info(f"File already complete: {local_path}") - return + return response.headers elif response.status_code == 206: # Partial Content - resume successful self.logger.info(f"Resuming download (HTTP 206)") @@ -176,7 +241,7 @@ def _download_with_retry(self, url, local_path, chunk_size): self._stream_download(response, local_path, resume_byte_pos, chunk_size) # Success - exit retry loop - return + return response.headers except (requests.RequestException, IOError) as e: self.logger.warning(f"Download attempt {attempt}/{self.retries} failed: {e}") @@ -193,13 +258,12 @@ def _download_with_retry(self, url, local_path, chunk_size): @functools.lru_cache(maxsize=None) def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): """ - Download a file from the Babel server to local storage with MD5 validation. + Download a file from the Babel server to local storage with ETag-based caching. - If a .md5 file exists on the server, this method will: - 1. Check if the local file exists - 2. Verify its MD5 checksum matches the expected value - 3. Delete and re-download if checksums don't match - 4. Skip download if checksums match + Three-tier freshness logic: + 1. If .meta exists and last_checked is within freshness window → return immediately + 2. If .meta exists but stale → HEAD request to compare ETag; return if unchanged + 3. If ETag changed or no .meta → full re-download Args: dirpath: Relative path from url_base to the file @@ -212,48 +276,37 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) url_to_download = urllib.parse.urljoin(self.url_base, dirpath) - md5_url = url_to_download + '.md5' - # Check if file already exists and validate with MD5 if available if os.path.exists(local_path_to_download_to): - self.logger.info(f"Local file exists: {local_path_to_download_to}") - - # Try to fetch remote MD5 checksum - expected_md5 = self._fetch_remote_md5(md5_url) - - if expected_md5: - self.logger.info(f"Validating MD5 checksum (expected: {expected_md5})") - - # Calculate local file's MD5 - actual_md5 = self._calculate_md5(local_path_to_download_to, chunk_size) - self.logger.info(f"Local file MD5: {actual_md5}") + meta = self._load_meta(local_path_to_download_to) + if meta is not None: + # Tier 1: within freshness window — skip all network calls + if self._is_within_freshness(meta, self.freshness_seconds): + self.logger.info(f"File within freshness window, skipping check: {local_path_to_download_to}") + return local_path_to_download_to - if actual_md5 == expected_md5: - # File is valid, skip download - self.logger.info(f"MD5 checksum matches - file is valid, skipping download") - bytes_downloaded = os.path.getsize(local_path_to_download_to) - self.logger.info(f"Using existing file: {local_path_to_download_to} ({bytes_downloaded} bytes)") + # Tier 2: stale but maybe unchanged — HEAD request + if self._etag_matches(url_to_download, meta): + # Update last_checked timestamp + meta["last_checked"] = datetime.now(timezone.utc).isoformat() + meta_path = self._get_meta_path(local_path_to_download_to) + with open(meta_path, "w") as f: + json.dump(meta, f, indent=2) + self.logger.info(f"ETag matches, using existing file: {local_path_to_download_to}") return local_path_to_download_to - else: - # Checksums don't match - delete and re-download - self.logger.warning(f"MD5 checksum mismatch! Expected {expected_md5}, got {actual_md5}") - self.logger.warning(f"Deleting corrupted file and re-downloading: {local_path_to_download_to}") - os.remove(local_path_to_download_to) + + # Tier 3: ETag changed — delete and re-download + self.logger.warning(f"Remote file changed, re-downloading: {local_path_to_download_to}") + os.remove(local_path_to_download_to) self.logger.info(f"Downloading {url_to_download} to {local_path_to_download_to}") - # Download with retry logic - self._download_with_retry(url_to_download, local_path_to_download_to, chunk_size) + # Download with retry logic; get response headers back + response_headers = self._download_with_retry(url_to_download, local_path_to_download_to, chunk_size) - # Verify MD5 after download if available - expected_md5 = self._fetch_remote_md5(md5_url) - if expected_md5: - actual_md5 = self._calculate_md5(local_path_to_download_to, chunk_size) - if actual_md5 == expected_md5: - self.logger.info(f"Post-download MD5 verification passed: {actual_md5}") - else: - self.logger.error(f"Post-download MD5 verification failed! Expected {expected_md5}, got {actual_md5}") - raise RuntimeError(f"Downloaded file has incorrect MD5 checksum") + # Save sidecar metadata + if response_headers is not None: + self._save_meta(local_path_to_download_to, response_headers) bytes_downloaded = os.path.getsize(local_path_to_download_to) self.logger.info(f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes") diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 912cd0a..045e402 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -5,13 +5,14 @@ Integration tests download real files from the Babel server. """ -import hashlib +import json import os import tempfile +from datetime import datetime, timezone, timedelta import pytest import requests -from unittest.mock import Mock, patch +from unittest.mock import Mock, patch, MagicMock from babel_explorer.core.downloader import BabelDownloader @@ -45,6 +46,14 @@ def test_default_retries(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) assert dl.retries == 10 + def test_default_freshness_seconds(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + assert dl.freshness_seconds == 3 * 3600 + + def test_custom_freshness_seconds(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), freshness_seconds=0) + assert dl.freshness_seconds == 0 + def test_invalid_path_raises_value_error(self): """Using a file path (not a directory) should raise ValueError.""" with tempfile.NamedTemporaryFile(delete=False) as f: @@ -77,161 +86,360 @@ def test_lru_caching(self, tmp_path): assert result1 is result2 # identity check — same cached object -class TestCalculateMd5: - """Tests for _calculate_md5.""" +class TestSaveMeta: + """Tests for _save_meta.""" - def test_correct_hash(self, tmp_path): - content = b"Hello, world!" - expected = hashlib.md5(content).hexdigest() - file_path = tmp_path / "test.bin" - file_path.write_bytes(content) + def _make_dl(self, tmp_path): + return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - assert dl._calculate_md5(str(file_path)) == expected + def test_writes_all_fields(self, tmp_path): + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "test.parquet") + # Create the file so the path is valid + open(file_path, 'wb').close() + + headers = { + "ETag": '"abc123"', + "Last-Modified": "Wed, 03 Dec 2025 15:54:19 GMT", + "Content-Length": "12345", + } + dl._save_meta(file_path, headers) + + meta_path = file_path + ".meta" + assert os.path.exists(meta_path) + with open(meta_path) as f: + meta = json.load(f) + + assert meta["etag"] == '"abc123"' + assert meta["last_modified"] == "Wed, 03 Dec 2025 15:54:19 GMT" + assert meta["content_length"] == 12345 + assert "last_checked" in meta + + def test_last_checked_is_recent_utc(self, tmp_path): + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "f.parquet") + open(file_path, 'wb').close() - def test_different_chunk_sizes_same_result(self, tmp_path): - content = b"A" * 5000 - expected = hashlib.md5(content).hexdigest() - file_path = tmp_path / "chunks.bin" - file_path.write_bytes(content) + dl._save_meta(file_path, {"ETag": '"x"'}) - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - assert dl._calculate_md5(str(file_path), chunk_size=100) == expected - assert dl._calculate_md5(str(file_path), chunk_size=4096) == expected + with open(file_path + ".meta") as f: + meta = json.load(f) + + last_checked = datetime.fromisoformat(meta["last_checked"]) + age = (datetime.now(timezone.utc) - last_checked).total_seconds() + assert age < 5 # written less than 5 seconds ago + + def test_missing_headers_not_written(self, tmp_path): + """Headers not present in the response should not appear in .meta.""" + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "sparse.parquet") + open(file_path, 'wb').close() + + dl._save_meta(file_path, {}) + with open(file_path + ".meta") as f: + meta = json.load(f) -class TestFetchRemoteMd5: - """Tests for _fetch_remote_md5.""" + assert "etag" not in meta + assert "last_modified" not in meta + assert "content_length" not in meta + assert "last_checked" in meta + + +class TestLoadMeta: + """Tests for _load_meta.""" def _make_dl(self, tmp_path): return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - def test_valid_md5_response(self, tmp_path): + def test_returns_none_if_no_meta_file(self, tmp_path): dl = self._make_dl(tmp_path) - mock_resp = Mock() - mock_resp.status_code = 200 - mock_resp.text = "d41d8cd98f00b204e9800998ecf8427e filename.parquet\n" - mock_resp.raise_for_status = Mock() - with patch("babel_explorer.core.downloader.requests.get", return_value=mock_resp): - result = dl._fetch_remote_md5("https://example.com/file.md5") - assert result == "d41d8cd98f00b204e9800998ecf8427e" + assert dl._load_meta(str(tmp_path / "nonexistent.parquet")) is None + + def test_returns_dict_for_valid_meta(self, tmp_path): + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "f.parquet") + open(file_path, 'wb').close() + meta_data = {"etag": '"abc"', "last_checked": "2026-01-01T00:00:00+00:00"} + with open(file_path + ".meta", "w") as f: + json.dump(meta_data, f) + + result = dl._load_meta(file_path) + assert result == meta_data + + def test_returns_none_for_corrupt_meta(self, tmp_path): + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "corrupt.parquet") + open(file_path, 'wb').close() + with open(file_path + ".meta", "w") as f: + f.write("not valid json {{{") + + assert dl._load_meta(file_path) is None + + +class TestIsWithinFreshness: + """Tests for _is_within_freshness.""" - def test_hash_only_format(self, tmp_path): + def _make_dl(self, tmp_path): + return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + + def test_returns_true_when_recent(self, tmp_path): + dl = self._make_dl(tmp_path) + recent = datetime.now(timezone.utc).isoformat() + meta = {"last_checked": recent} + assert dl._is_within_freshness(meta, 3600) is True + + def test_returns_false_when_stale(self, tmp_path): + dl = self._make_dl(tmp_path) + old = (datetime.now(timezone.utc) - timedelta(hours=5)).isoformat() + meta = {"last_checked": old} + assert dl._is_within_freshness(meta, 3600) is False + + def test_returns_false_when_missing_last_checked(self, tmp_path): dl = self._make_dl(tmp_path) + assert dl._is_within_freshness({}, 3600) is False + + def test_returns_true_when_freshness_is_inf(self, tmp_path): + dl = self._make_dl(tmp_path) + old = (datetime.now(timezone.utc) - timedelta(days=365)).isoformat() + meta = {"last_checked": old} + assert dl._is_within_freshness(meta, float("inf")) is True + + def test_returns_false_when_freshness_is_zero(self, tmp_path): + dl = self._make_dl(tmp_path) + just_now = datetime.now(timezone.utc).isoformat() + meta = {"last_checked": just_now} + # Even with freshness=0, age >= 0 so it's not < 0 + assert dl._is_within_freshness(meta, 0) is False + + +class TestEtagMatches: + """Tests for _etag_matches.""" + + def _make_dl(self, tmp_path): + return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + + def test_returns_true_on_matching_etag(self, tmp_path): + dl = self._make_dl(tmp_path) + meta = {"etag": '"abc123"'} mock_resp = Mock() - mock_resp.status_code = 200 - mock_resp.text = "d41d8cd98f00b204e9800998ecf8427e\n" + mock_resp.headers = {"ETag": '"abc123"'} mock_resp.raise_for_status = Mock() - with patch("babel_explorer.core.downloader.requests.get", return_value=mock_resp): - result = dl._fetch_remote_md5("https://example.com/file.md5") - assert result == "d41d8cd98f00b204e9800998ecf8427e" + with patch("babel_explorer.core.downloader.requests.head", return_value=mock_resp): + assert dl._etag_matches("https://example.com/f.parquet", meta) is True - def test_404_returns_none(self, tmp_path): + def test_returns_false_on_different_etag(self, tmp_path): dl = self._make_dl(tmp_path) + meta = {"etag": '"old"'} mock_resp = Mock() - mock_resp.status_code = 404 - with patch("babel_explorer.core.downloader.requests.get", return_value=mock_resp): - assert dl._fetch_remote_md5("https://example.com/missing.md5") is None + mock_resp.headers = {"ETag": '"new"'} + mock_resp.raise_for_status = Mock() + with patch("babel_explorer.core.downloader.requests.head", return_value=mock_resp): + assert dl._etag_matches("https://example.com/f.parquet", meta) is False - def test_malformed_returns_none(self, tmp_path): + def test_fallback_last_modified_match(self, tmp_path): dl = self._make_dl(tmp_path) + lm = "Wed, 03 Dec 2025 15:54:19 GMT" + meta = {"last_modified": lm, "content_length": 100} mock_resp = Mock() - mock_resp.status_code = 200 - mock_resp.text = "not-a-valid-md5-hash\n" + mock_resp.headers = {"Last-Modified": lm, "Content-Length": "100"} mock_resp.raise_for_status = Mock() - with patch("babel_explorer.core.downloader.requests.get", return_value=mock_resp): - assert dl._fetch_remote_md5("https://example.com/bad.md5") is None + with patch("babel_explorer.core.downloader.requests.head", return_value=mock_resp): + assert dl._etag_matches("https://example.com/f.parquet", meta) is True - def test_network_error_returns_none(self, tmp_path): + def test_returns_false_on_request_error(self, tmp_path): dl = self._make_dl(tmp_path) - with patch("babel_explorer.core.downloader.requests.get", side_effect=requests.ConnectionError("fail")): - assert dl._fetch_remote_md5("https://example.com/err.md5") is None + meta = {"etag": '"abc"'} + with patch("babel_explorer.core.downloader.requests.head", + side_effect=requests.ConnectionError("fail")): + assert dl._etag_matches("https://example.com/f.parquet", meta) is False -class TestMd5ValidationFlow: - """Tests for the MD5 validation logic inside get_downloaded_file.""" +class TestGetDownloadedFileTiers: + """Tests for the three-tier logic in get_downloaded_file.""" - def test_matching_checksum_skips_download(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - test_file = "test.txt" - content = b"test content" - local_path = tmp_path / test_file - local_path.write_bytes(content) - expected_md5 = hashlib.md5(content).hexdigest() - - with patch.object(dl, '_fetch_remote_md5', return_value=expected_md5): - with patch.object(dl, '_download_with_retry') as mock_dl: + def _make_dl(self, tmp_path, freshness=3600): + return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), + freshness_seconds=freshness) + + # --- Tier 1: within freshness window --- + + def test_tier1_returns_immediately_no_http(self, tmp_path): + """File + fresh .meta → no network calls at all.""" + dl = self._make_dl(tmp_path, freshness=3600) + test_file = "duckdb/test.parquet" + local = tmp_path / "duckdb" / "test.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data") + + meta = {"etag": '"abc"', "last_checked": datetime.now(timezone.utc).isoformat()} + with open(str(local) + ".meta", "w") as f: + json.dump(meta, f) + + with patch("babel_explorer.core.downloader.requests.head") as mock_head: + with patch("babel_explorer.core.downloader.requests.get") as mock_get: dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) - mock_dl.assert_not_called() - assert result == str(local_path) - - def test_mismatched_checksum_triggers_redownload(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - test_file = "mismatch.txt" - local_path = tmp_path / test_file - local_path.write_bytes(b"wrong content") - correct_content = b"correct content" - expected_md5 = hashlib.md5(correct_content).hexdigest() + mock_head.assert_not_called() + mock_get.assert_not_called() + assert result == str(local) + + # --- Tier 2: stale .meta, ETag matches --- + + def test_tier2_head_check_no_redownload(self, tmp_path): + """Stale .meta + matching ETag → HEAD only, no GET.""" + dl = self._make_dl(tmp_path, freshness=0) + test_file = "duckdb/test.parquet" + local = tmp_path / "duckdb" / "test.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data") + + old_ts = (datetime.now(timezone.utc) - timedelta(hours=5)).isoformat() + meta = {"etag": '"abc"', "last_checked": old_ts} + with open(str(local) + ".meta", "w") as f: + json.dump(meta, f) + + mock_head_resp = Mock() + mock_head_resp.headers = {"ETag": '"abc"'} + mock_head_resp.raise_for_status = Mock() + + with patch("babel_explorer.core.downloader.requests.head", return_value=mock_head_resp): + with patch("babel_explorer.core.downloader.requests.get") as mock_get: + dl.get_downloaded_file.cache_clear() + result = dl.get_downloaded_file(test_file) + mock_get.assert_not_called() + assert result == str(local) + + def test_tier2_updates_last_checked_after_head(self, tmp_path): + """After successful HEAD match, last_checked in .meta is updated.""" + dl = self._make_dl(tmp_path, freshness=0) + test_file = "duckdb/upd.parquet" + local = tmp_path / "duckdb" / "upd.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data") + + old_ts = (datetime.now(timezone.utc) - timedelta(hours=5)).isoformat() + meta = {"etag": '"abc"', "last_checked": old_ts} + with open(str(local) + ".meta", "w") as f: + json.dump(meta, f) + + mock_head_resp = Mock() + mock_head_resp.headers = {"ETag": '"abc"'} + mock_head_resp.raise_for_status = Mock() + + with patch("babel_explorer.core.downloader.requests.head", return_value=mock_head_resp): + dl.get_downloaded_file.cache_clear() + dl.get_downloaded_file(test_file) + + with open(str(local) + ".meta") as f: + updated_meta = json.load(f) + updated_ts = datetime.fromisoformat(updated_meta["last_checked"]) + assert (datetime.now(timezone.utc) - updated_ts).total_seconds() < 5 + + # --- Tier 3: ETag changed, re-download --- + + def test_tier3_redownloads_when_etag_changed(self, tmp_path): + """Changed ETag → file deleted and re-downloaded.""" + dl = self._make_dl(tmp_path, freshness=0) + test_file = "duckdb/changed.parquet" + local = tmp_path / "duckdb" / "changed.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"old data") + + old_ts = (datetime.now(timezone.utc) - timedelta(hours=5)).isoformat() + meta = {"etag": '"old"', "last_checked": old_ts} + with open(str(local) + ".meta", "w") as f: + json.dump(meta, f) + + mock_head_resp = Mock() + mock_head_resp.headers = {"ETag": '"new"'} + mock_head_resp.raise_for_status = Mock() + + new_content = b"new data" def fake_download(url, path, chunk_size): with open(path, 'wb') as f: - f.write(correct_content) + f.write(new_content) + return {"ETag": '"new"', "Content-Length": str(len(new_content))} - with patch.object(dl, '_fetch_remote_md5', return_value=expected_md5): + with patch("babel_explorer.core.downloader.requests.head", return_value=mock_head_resp): with patch.object(dl, '_download_with_retry', side_effect=fake_download): dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) - assert os.path.exists(result) - with open(result, 'rb') as f: - assert f.read() == correct_content - def test_no_md5_proceeds_normally(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - test_file = "no_md5.txt" - content = b"downloaded content" + assert open(result, 'rb').read() == new_content + + # --- No .meta: fresh download --- + + def test_downloads_when_no_meta(self, tmp_path): + """No file and no .meta → download happens, .meta is saved.""" + dl = self._make_dl(tmp_path) + test_file = "duckdb/new.parquet" + local_path = str(tmp_path / "duckdb" / "new.parquet") + content = b"fresh download" def fake_download(url, path, chunk_size): + os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, 'wb') as f: f.write(content) - - with patch.object(dl, '_fetch_remote_md5', return_value=None): - with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: - dl.get_downloaded_file.cache_clear() - result = dl.get_downloaded_file(test_file) - mock_dl.assert_called_once() - assert os.path.exists(result) - - def test_post_download_validation_fail_raises(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - test_file = "post_fail.txt" - correct_md5 = hashlib.md5(b"expected").hexdigest() + return {"ETag": '"fresh"', "Content-Length": str(len(content))} + + with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: + dl.get_downloaded_file.cache_clear() + result = dl.get_downloaded_file(test_file) + mock_dl.assert_called_once() + + assert os.path.exists(result) + assert open(result, 'rb').read() == content + # .meta should be saved + meta_path = result + ".meta" + assert os.path.exists(meta_path) + with open(meta_path) as f: + saved_meta = json.load(f) + assert saved_meta["etag"] == '"fresh"' + + def test_downloads_when_file_exists_but_no_meta(self, tmp_path): + """File exists but no .meta → treats as unknown, triggers full download flow.""" + dl = self._make_dl(tmp_path, freshness=3600) + test_file = "duckdb/nometa.parquet" + local = tmp_path / "duckdb" / "nometa.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"old content") + # No .meta file + + new_content = b"refreshed" def fake_download(url, path, chunk_size): with open(path, 'wb') as f: - f.write(b"wrong data after download") + f.write(new_content) + return {"ETag": '"new"'} - with patch.object(dl, '_fetch_remote_md5', return_value=correct_md5): - with patch.object(dl, '_download_with_retry', side_effect=fake_download): - dl.get_downloaded_file.cache_clear() - with pytest.raises(RuntimeError, match="incorrect MD5 checksum"): - dl.get_downloaded_file(test_file) + with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: + dl.get_downloaded_file.cache_clear() + result = dl.get_downloaded_file(test_file) + mock_dl.assert_called_once() + + assert open(result, 'rb').read() == new_content + + +class TestGetDownloadedFileCaching: + """Tests for get_downloaded_file LRU caching.""" - def test_post_download_validation_pass(self, tmp_path): + def test_cache_returns_same_result(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - test_file = "post_pass.txt" - content = b"correct content" - expected_md5 = hashlib.md5(content).hexdigest() + content = b"cached content" def fake_download(url, path, chunk_size): with open(path, 'wb') as f: f.write(content) + return {} - with patch.object(dl, '_fetch_remote_md5', return_value=expected_md5): - with patch.object(dl, '_download_with_retry', side_effect=fake_download): - dl.get_downloaded_file.cache_clear() - result = dl.get_downloaded_file(test_file) - assert os.path.exists(result) + with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: + dl.get_downloaded_file.cache_clear() + r1 = dl.get_downloaded_file("cached.txt") + r2 = dl.get_downloaded_file("cached.txt") + assert r1 == r2 + mock_dl.assert_called_once() # only one actual download class TestDownloadWithRetry: @@ -282,6 +490,7 @@ def test_http_416_file_already_complete(self, tmp_path): mock_response = Mock() mock_response.status_code = 416 + mock_response.headers = {} with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): dl._download_with_retry("https://example.com/file", str(out_path), 1024) @@ -303,6 +512,20 @@ def test_server_no_resume_restarts_download(self, tmp_path): dl._download_with_retry("https://example.com/file", str(out_path), 1024) assert out_path.read_bytes() == b"full content" + def test_returns_response_headers(self, tmp_path): + """_download_with_retry should return response headers.""" + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = str(tmp_path / "headers.bin") + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {'Content-Length': '5', 'ETag': '"abc"'} + mock_response.iter_content = Mock(return_value=[b"hello"]) + + with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): + headers = dl._download_with_retry("https://example.com/file", out_path, 1024) + assert headers['ETag'] == '"abc"' + class TestStreamDownload: """Tests for _stream_download.""" @@ -332,26 +555,6 @@ def test_append_mode_on_resume(self, tmp_path): assert out_path.read_bytes() == b"startend" -class TestGetDownloadedFileCaching: - """Tests for get_downloaded_file LRU caching.""" - - def test_cache_returns_same_result(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - content = b"cached content" - - def fake_download(url, path, chunk_size): - with open(path, 'wb') as f: - f.write(content) - - with patch.object(dl, '_fetch_remote_md5', return_value=None): - with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: - dl.get_downloaded_file.cache_clear() - r1 = dl.get_downloaded_file("cached.txt") - r2 = dl.get_downloaded_file("cached.txt") - assert r1 == r2 - mock_dl.assert_called_once() # only one actual download - - class TestGetDownloadedDir: """Tests for get_downloaded_dir.""" @@ -382,6 +585,16 @@ def test_download_metadata_parquet(downloaded_metadata): assert os.path.getsize(downloaded_metadata) > 0 +@pytest.mark.integration +def test_download_creates_meta_file(downloaded_concord): + """After download, a .meta sidecar file should exist.""" + meta_path = downloaded_concord + ".meta" + assert os.path.isfile(meta_path), f"Missing .meta file: {meta_path}" + with open(meta_path) as f: + meta = json.load(f) + assert "last_checked" in meta + + @pytest.mark.integration def test_download_caching_real_files(shared_downloader, downloaded_concord): """Second call returns same path and file is not re-downloaded.""" From d983de561c9c8ab6a7424ce0b427b6cf6577c3ae Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 3 Mar 2026 14:38:42 -0500 Subject: [PATCH 025/123] Added some CURIEs to test. --- tests/data/valid_curies.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/data/valid_curies.txt b/tests/data/valid_curies.txt index 9f2f87c..89a53b3 100644 --- a/tests/data/valid_curies.txt +++ b/tests/data/valid_curies.txt @@ -1,3 +1,5 @@ # Valid CURIEs for integration tests. # Add new CURIEs here to expand test coverage — tests are parametrized over this list. MONDO:0004979 +MONDO:0005044 +NCIT:C55060 From 93086b1e490faeee0881b8fc87a19357e077a579 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 3 Mar 2026 14:49:07 -0500 Subject: [PATCH 026/123] Partially changed --expand to --recurse. --- src/babel_explorer/core/babel_xrefs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 95fda85..012f009 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -116,12 +116,12 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): return xrefs - def get_curie_xrefs(self, curies: list[str], expand: bool = False, ignore_curies_in_expansion: set = set(), label_curies: bool = False): + def get_curie_xrefs(self, curies: list[str], recurse: bool = False, ignore_curies_in_expansion: set = set(), label_curies: bool = False): """ Search for all identifiers that are cross-referenced to the given CURIE. :param curie: A CURIE to search for. - :param expand: Whether to expand the cross-references (i.e. recursively follow all identifiers). + :param recurse: Whether to expand the cross-references (i.e. recursively follow all identifiers). :return: A list of cross-references containing that CURIE. """ @@ -133,11 +133,11 @@ def get_curie_xrefs(self, curies: list[str], expand: bool = False, ignore_curies logging.info(f"Searching for cross-references for {curie}") xrefs.update(self.get_curie_xref(curie, label_curies)) - if expand: + if recurse: # Get a unique set of referenced curies, not including the ones currently queried. new_curies = list(set([curie for xref in xrefs for curie in xref.curies]) - set(curies) - ignore_curies_in_expansion) if new_curies: logging.info(f"Expanding cross-references to {new_curies}") - xrefs.update(self.get_curie_xrefs(new_curies, expand=True, ignore_curies_in_expansion=ignore_curies_in_expansion | set(curies) | set(new_curies), label_curies=label_curies)) + xrefs.update(self.get_curie_xrefs(new_curies, recurse=True, ignore_curies_in_expansion=ignore_curies_in_expansion | set(curies) | set(new_curies), label_curies=label_curies)) return sorted(xrefs) From d3747608a20905cd8827a93478aa8748b021a9bd Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 3 Mar 2026 14:49:47 -0500 Subject: [PATCH 027/123] More fully changed --expand to --recurse. --- src/babel_explorer/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 5acdcac..44955e5 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -26,12 +26,12 @@ def cli(): @click.option("--local-dir", type=str, default="data/2025nov19", help="Local location to save Babel download files to") @click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") @click.option("--nodenorm-url", type=str, default="https://nodenormalization-sri.renci.org/", help="NodeNorm URL to check for concord changes") -@click.option("--expand", is_flag=True, help="Also display xrefs for returned CURIEs") +@click.option("--recurse", is_flag=True, help="Recursively query returned xrefs") @click.option("--labels", is_flag=True, help="Include labels for CURIEs") @click.option("--check-download", type=str, default="3h", show_default=True, help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " "'never' always checks via HTTP HEAD; '0' same as 'never'.") -def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, expand: bool, labels: bool, +def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, recurse: bool, labels: bool, check_download: str): """ Fetches and prints the cross-references (xrefs) for the given CURIEs. @@ -52,7 +52,7 @@ def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, expan freshness = parse_duration(check_download) bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness), NodeNorm(nodenorm_url)) - xrefs = bxref.get_curie_xrefs(curies, expand, label_curies=labels) + xrefs = bxref.get_curie_xrefs(curies, recurse, label_curies=labels) for xref in xrefs: print(xref) From 26b6f9d5afa4d517957f9b60dad862a5f3ffad98 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 3 Mar 2026 14:52:07 -0500 Subject: [PATCH 028/123] Add pytest-xdist for parallel test execution - Add pytest-xdist[psutil] and filelock to dev dependencies - Enable parallel execution by default with addopts = "-n auto" - Switch DuckDB connections to in-memory mode (duckdb.connect()) to eliminate file locking that would deadlock parallel workers - Make test_data_dir teardown worker-aware (only gw0 cleans up) - Wrap download fixtures with FileLock to serialize concurrent downloads - Fix test_babel_xrefs.py: update expand= to recurse= to match renamed param Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 3 ++ src/babel_explorer/core/babel_xrefs.py | 8 ++- tests/conftest.py | 33 ++++++++----- tests/test_babel_xrefs.py | 16 +++--- uv.lock | 68 ++++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eafcfc6..59c1b68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,9 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ + "filelock>=3.16", "pytest>=8.3.5", + "pytest-xdist[psutil]>=3.6", "ruff>=0.11.0", ] @@ -25,6 +27,7 @@ dev = [ babel-explorer = "babel_explorer.cli:cli" [tool.pytest.ini_options] +addopts = "-n auto" markers = [ "integration: tests requiring network access (deselect with '-m \"not integration\"')", "slow: tests downloading very large files 2GB+ (deselect with '-m \"not slow\"')", diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 012f009..de8e661 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -82,9 +82,8 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: identifier_parquet = self.downloader.get_downloaded_file('duckdb/Identifiers.parquet') concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') - # Query the Parquet files using DuckDB. - duckdb_path = self.downloader.get_output_file('output/duckdbs/xrefs.duckdb') - db = duckdb.connect(duckdb_path) + # Query the Parquet files using DuckDB (in-memory; nothing is persisted). + db = duckdb.connect() identifier_table = db.read_parquet(identifier_parquet) result = db.execute(f"SELECT * FROM identifier_table WHERE curie IN $1", [curies]) @@ -96,8 +95,7 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') - duckdb_path = self.downloader.get_output_file('output/duckdbs/xrefs.duckdb') - db = duckdb.connect(duckdb_path) + db = duckdb.connect() concord_table = db.read_parquet(concord_parquet) xref_tuples = db.execute(f"SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", [curie]).fetchall() xrefs = list(map(lambda rec: CrossReference.from_tuple(rec), xref_tuples)) diff --git a/tests/conftest.py b/tests/conftest.py index f3df2fe..fc61599 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,7 @@ import shutil import pytest +from filelock import FileLock from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.babel_xrefs import BabelXRefs @@ -39,20 +40,22 @@ def valid_curies() -> list[str]: @pytest.fixture(scope="session") -def test_data_dir(): +def test_data_dir(request): """ - Provide a clean test data directory for the entire session. + Provide a test data directory for the entire session. Creates the directory before tests, removes it after all tests complete. + When running under pytest-xdist, only the first worker (gw0) performs cleanup. """ - if os.path.exists(TEST_DATA_DIR): - shutil.rmtree(TEST_DATA_DIR) + worker_id = getattr(request.config, "workerinput", {}).get("workerid", "master") os.makedirs(TEST_DATA_DIR, exist_ok=True) yield TEST_DATA_DIR - if os.path.exists(TEST_DATA_DIR): - shutil.rmtree(TEST_DATA_DIR) + # Only the first xdist worker (or a non-xdist run) cleans up the directory. + if worker_id in ("master", "gw0"): + if os.path.exists(TEST_DATA_DIR): + shutil.rmtree(TEST_DATA_DIR) @pytest.fixture(scope="session") @@ -62,15 +65,19 @@ def shared_downloader(test_data_dir) -> BabelDownloader: @pytest.fixture(scope="session") -def downloaded_concord(shared_downloader) -> str: +def downloaded_concord(shared_downloader, test_data_dir) -> str: """Download duckdb/Concord.parquet (~626 MB). Returns the local path.""" - return shared_downloader.get_downloaded_file(CONCORD_FILE) + lock_path = os.path.join(test_data_dir, "concord.lock") + with FileLock(lock_path): + return shared_downloader.get_downloaded_file(CONCORD_FILE) @pytest.fixture(scope="session") -def downloaded_metadata(shared_downloader) -> str: +def downloaded_metadata(shared_downloader, test_data_dir) -> str: """Download duckdb/Metadata.parquet (small). Returns the local path.""" - return shared_downloader.get_downloaded_file(METADATA_FILE) + lock_path = os.path.join(test_data_dir, "metadata.lock") + with FileLock(lock_path): + return shared_downloader.get_downloaded_file(METADATA_FILE) @pytest.fixture(scope="session") @@ -83,9 +90,11 @@ def downloaded_parquet_files(downloaded_concord, downloaded_metadata) -> dict[st @pytest.fixture(scope="session") -def downloaded_identifiers(shared_downloader) -> str: +def downloaded_identifiers(shared_downloader, test_data_dir) -> str: """Download duckdb/Identifiers.parquet (2 GB+). Returns the local path.""" - return shared_downloader.get_downloaded_file(IDENTIFIERS_FILE) + lock_path = os.path.join(test_data_dir, "identifiers.lock") + with FileLock(lock_path): + return shared_downloader.get_downloaded_file(IDENTIFIERS_FILE) @pytest.fixture(scope="session") diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 052d09c..774ccae 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -203,7 +203,7 @@ def test_get_curie_xrefs_no_expand(self, tmp_path): xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") with patch.object(bx, 'get_curie_xref', return_value=[xr]): bx.get_curie_xref.cache_clear() - result = bx.get_curie_xrefs(["A:1"], expand=False) + result = bx.get_curie_xrefs(["A:1"], recurse=False) assert len(result) == 1 assert result[0] == xr @@ -220,7 +220,7 @@ def mock_get_curie_xref(curie, label_curies=False): return [] with patch.object(bx, 'get_curie_xref', side_effect=mock_get_curie_xref): - result = bx.get_curie_xrefs(["A:1"], expand=True) + result = bx.get_curie_xrefs(["A:1"], recurse=True) assert xr1 in result assert xr2 in result @@ -230,7 +230,7 @@ def test_results_are_sorted(self, tmp_path): xr_a = CrossReference(filename="a", subj="A:1", pred="p", obj="B:1") with patch.object(bx, 'get_curie_xref', return_value=[xr_b, xr_a]): - result = bx.get_curie_xrefs(["X:1"], expand=False) + result = bx.get_curie_xrefs(["X:1"], recurse=False) assert result == [xr_a, xr_b] @@ -265,7 +265,7 @@ def test_get_curie_xref_returns_known_xrefs(babel_xrefs, curie): def test_get_curie_xrefs_single_no_expand(babel_xrefs, curie): """get_curie_xrefs without expansion returns sorted, non-empty results.""" babel_xrefs.get_curie_xref.cache_clear() - results = babel_xrefs.get_curie_xrefs([curie], expand=False) + results = babel_xrefs.get_curie_xrefs([curie], recurse=False) assert len(results) > 0 assert results == sorted(results) @@ -275,9 +275,9 @@ def test_get_curie_xrefs_single_no_expand(babel_xrefs, curie): def test_get_curie_xrefs_expansion_finds_more(babel_xrefs, curie): """Expanded results are at least as many as non-expanded.""" babel_xrefs.get_curie_xref.cache_clear() - non_expanded = babel_xrefs.get_curie_xrefs([curie], expand=False) + non_expanded = babel_xrefs.get_curie_xrefs([curie], recurse=False) babel_xrefs.get_curie_xref.cache_clear() - expanded = babel_xrefs.get_curie_xrefs([curie], expand=True) + expanded = babel_xrefs.get_curie_xrefs([curie], recurse=True) assert len(expanded) >= len(non_expanded) @@ -286,9 +286,9 @@ def test_get_curie_xrefs_expansion_finds_more(babel_xrefs, curie): def test_get_curie_xrefs_expanded_includes_original(babel_xrefs, curie): """Non-expanded results are a subset of expanded results.""" babel_xrefs.get_curie_xref.cache_clear() - non_expanded = set(babel_xrefs.get_curie_xrefs([curie], expand=False)) + non_expanded = set(babel_xrefs.get_curie_xrefs([curie], recurse=False)) babel_xrefs.get_curie_xref.cache_clear() - expanded = set(babel_xrefs.get_curie_xrefs([curie], expand=True)) + expanded = set(babel_xrefs.get_curie_xrefs([curie], recurse=True)) assert non_expanded.issubset(expanded) diff --git a/uv.lock b/uv.lock index 56af50a..b8496b5 100644 --- a/uv.lock +++ b/uv.lock @@ -15,7 +15,9 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "filelock" }, { name = "pytest" }, + { name = "pytest-xdist", extra = ["psutil"] }, { name = "ruff" }, ] @@ -29,7 +31,9 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "filelock", specifier = ">=3.16" }, { name = "pytest", specifier = ">=8.3.5" }, + { name = "pytest-xdist", extras = ["psutil"], specifier = ">=3.6" }, { name = "ruff", specifier = ">=0.11.0" }, ] @@ -172,6 +176,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/2d/13e6024e613679d8a489dd922f199ef4b1d08a456a58eadd96dc2f05171f/duckdb-1.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:53cd6423136ab44383ec9955aefe7599b3fb3dd1fe006161e6396d8167e0e0d4", size = 13458633, upload-time = "2026-01-26T11:50:17.657Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + +[[package]] +name = "filelock" +version = "3.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/18/a1fd2231c679dcb9726204645721b12498aeac28e1ad0601038f94b42556/filelock-3.25.0.tar.gz", hash = "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3", size = 40158, upload-time = "2026-03-01T15:08:45.916Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/0b/de6f54d4a8bedfe8645c41497f3c18d749f0bd3218170c667bf4b81d0cdd/filelock-3.25.0-py3-none-any.whl", hash = "sha256:5ccf8069f7948f494968fc0713c10e5c182a9c9d9eef3a636307a20c2490f047", size = 26427, upload-time = "2026-03-01T15:08:44.593Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -208,6 +230,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -233,6 +283,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + +[package.optional-dependencies] +psutil = [ + { name = "psutil" }, +] + [[package]] name = "requests" version = "2.32.5" From bb756cfb26e7bf70036feaddac6a7be4af4e6de1 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 3 Mar 2026 15:06:18 -0500 Subject: [PATCH 029/123] Replace Python recursion in get_curie_xrefs with DuckDB WITH RECURSIVE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recurse=True path previously issued one DuckDB query per CURIE and called itself recursively (O(diameter) queries, Python stack growth). It now delegates to _get_curie_xrefs_recursive, which traverses the full connected component in a single SQL query using WITH RECURSIVE. A bidirectional `edges` CTE (subj→obj and obj→subj) collapses the two traversal directions into one recursive arm; UNION (not UNION ALL) provides automatic cycle detection. ignore_curies_in_expansion is now a no-op on the recurse=True path and emits a DeprecationWarning. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/babel_xrefs.py | 72 ++++++++++++++++++++++---- tests/test_babel_xrefs.py | 48 ++++++++++++++--- 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index de8e661..fba5b61 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -3,6 +3,7 @@ # why we consider two identifiers to be identical. import dataclasses import logging +import warnings import duckdb import functools @@ -114,28 +115,77 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): return xrefs + def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = False): + """Traverse the cross-reference graph in one DuckDB WITH RECURSIVE query.""" + if not curies: + return [] + + concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') + concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') + + db = duckdb.connect() + concord_table = db.read_parquet(concord_parquet) + result = db.execute(""" + WITH RECURSIVE + edges(a, b) AS ( + SELECT subj, obj FROM concord_table + UNION ALL + SELECT obj, subj FROM concord_table + ), + frontier(curie) AS ( + SELECT unnest($1::VARCHAR[]) + UNION + SELECT e.b + FROM edges e + INNER JOIN frontier f ON e.a = f.curie + ) + SELECT DISTINCT c.filename, c.subj, c.pred, c.obj + FROM concord_table c + WHERE c.subj IN (SELECT curie FROM frontier) + OR c.obj IN (SELECT curie FROM frontier) + ORDER BY c.filename, c.subj, c.obj, c.pred + """, [curies]) + + xrefs = [CrossReference.from_tuple(row) for row in result.fetchall()] + + if label_curies: + xrefs = [LabeledCrossReference( + subj=xref.subj, + obj=xref.obj, + filename=xref.filename, + pred=xref.pred, + subj_label=self.nodenorm.get_identifier(xref.subj).label, + subj_biolink_type=self.nodenorm.get_identifier(xref.subj).biolink_type, + obj_label=self.nodenorm.get_identifier(xref.obj).label, + obj_biolink_type=self.nodenorm.get_identifier(xref.obj).biolink_type, + ) for xref in xrefs] + + return xrefs + def get_curie_xrefs(self, curies: list[str], recurse: bool = False, ignore_curies_in_expansion: set = set(), label_curies: bool = False): """ Search for all identifiers that are cross-referenced to the given CURIE. - :param curie: A CURIE to search for. + :param curies: A list of CURIEs to search for. :param recurse: Whether to expand the cross-references (i.e. recursively follow all identifiers). - :return: A list of cross-references containing that CURIE. + :param ignore_curies_in_expansion: Deprecated when recurse=True; has no effect. + :param label_curies: Whether to annotate results with labels from NodeNorm. + :return: A list of cross-references containing those CURIEs. """ - if ignore_curies_in_expansion: - logging.info(f"Ignoring {len(ignore_curies_in_expansion)}: {ignore_curies_in_expansion}") + if recurse: + if ignore_curies_in_expansion: + warnings.warn( + "ignore_curies_in_expansion has no effect when recurse=True; " + "cycle detection is handled automatically by the SQL query.", + DeprecationWarning, + stacklevel=2, + ) + return self._get_curie_xrefs_recursive(curies, label_curies) xrefs = set() for curie in curies: logging.info(f"Searching for cross-references for {curie}") xrefs.update(self.get_curie_xref(curie, label_curies)) - if recurse: - # Get a unique set of referenced curies, not including the ones currently queried. - new_curies = list(set([curie for xref in xrefs for curie in xref.curies]) - set(curies) - ignore_curies_in_expansion) - if new_curies: - logging.info(f"Expanding cross-references to {new_curies}") - xrefs.update(self.get_curie_xrefs(new_curies, recurse=True, ignore_curies_in_expansion=ignore_curies_in_expansion | set(curies) | set(new_curies), label_curies=label_curies)) - return sorted(xrefs) diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 774ccae..42fa6aa 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -212,18 +212,50 @@ def test_get_curie_xrefs_with_expand(self, tmp_path): xr1 = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") xr2 = CrossReference(filename="f", subj="B:2", pred="p", obj="C:3") - def mock_get_curie_xref(curie, label_curies=False): - if curie == "A:1": - return [xr1] - elif curie == "B:2": - return [xr2] - return [] - - with patch.object(bx, 'get_curie_xref', side_effect=mock_get_curie_xref): + with patch.object(bx, '_get_curie_xrefs_recursive', return_value=[xr1, xr2]) as mock_rec: result = bx.get_curie_xrefs(["A:1"], recurse=True) + mock_rec.assert_called_once_with(["A:1"], False) assert xr1 in result assert xr2 in result + def test_get_curie_xrefs_recursive_sql_traversal(self, tmp_path): + """_get_curie_xrefs_recursive uses SQL graph traversal, not Python recursion.""" + import duckdb as real_duckdb + + bx = self._make_bx(tmp_path) + + # Write a tiny Parquet file: graph A-B, B-C, D-E (disconnected from A-B-C) + parquet_path = str(tmp_path / "test_concord.parquet") + setup_db = real_duckdb.connect() + setup_db.execute(f""" + COPY ( + SELECT * FROM (VALUES + ('f1.tsv', 'A:1', 'skos:exactMatch', 'B:2'), + ('f1.tsv', 'B:2', 'skos:exactMatch', 'C:3'), + ('f2.tsv', 'D:4', 'skos:exactMatch', 'E:5') + ) AS t(filename, subj, pred, obj) + ) TO '{parquet_path}' (FORMAT PARQUET) + """) + setup_db.close() + + with patch.object(bx.downloader, 'get_downloaded_file', return_value=parquet_path): + # Starting from A:1 should reach B:2 and C:3 but not the D-E component + result = bx._get_curie_xrefs_recursive(["A:1"]) + pairs = {(xr.subj, xr.obj) for xr in result} + assert ("A:1", "B:2") in pairs + assert ("B:2", "C:3") in pairs + assert ("D:4", "E:5") not in pairs + + # Starting from D:4 should only reach E:5 + result = bx._get_curie_xrefs_recursive(["D:4"]) + pairs = {(xr.subj, xr.obj) for xr in result} + assert ("D:4", "E:5") in pairs + assert ("A:1", "B:2") not in pairs + + # Empty input returns empty list + result = bx._get_curie_xrefs_recursive([]) + assert result == [] + def test_results_are_sorted(self, tmp_path): bx = self._make_bx(tmp_path) xr_b = CrossReference(filename="b", subj="B:1", pred="p", obj="C:1") From e7ca3806b71aba07d40a773d4445792d4ae201c4 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 3 Mar 2026 15:44:56 -0500 Subject: [PATCH 030/123] Fix xdist race condition: skip test-data cleanup in parallel runs When pytest-xdist runs 8 workers, each worker session ends independently. gw0 was deleting data/test/ as soon as it finished its own tests, but other workers were still reading Concord.parquet. This caused sporadic IOException failures on any test that opened a fresh DuckDB connection (e.g. _get_curie_xrefs_recursive) after gw0's teardown deleted the file. Fix: only delete the shared test data directory in a sequential (non-xdist) run where worker_id == "master". In parallel runs the directory persists; BabelDownloader's freshness-window logic re-validates or re-downloads the files on the next run as needed. Co-Authored-By: Claude Sonnet 4.6 --- tests/conftest.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fc61599..f1e0df6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,15 +45,21 @@ def test_data_dir(request): Provide a test data directory for the entire session. Creates the directory before tests, removes it after all tests complete. - When running under pytest-xdist, only the first worker (gw0) performs cleanup. + When running under pytest-xdist, cleanup is skipped: worker sessions end at + unpredictable times and deleting the shared directory from one worker while + others are still reading the same files causes flaky IO errors. The files + are re-used (or re-validated) on the next run via the freshness-window logic + in BabelDownloader.get_downloaded_file. """ worker_id = getattr(request.config, "workerinput", {}).get("workerid", "master") os.makedirs(TEST_DATA_DIR, exist_ok=True) yield TEST_DATA_DIR - # Only the first xdist worker (or a non-xdist run) cleans up the directory. - if worker_id in ("master", "gw0"): + # Only clean up when running without xdist (sequential run). In a parallel + # run each worker session may finish at a different time; gw0 cleaning up + # while gw5 is still reading Concord.parquet causes spurious failures. + if worker_id == "master": if os.path.exists(TEST_DATA_DIR): shutil.rmtree(TEST_DATA_DIR) From 4c3870b28e4d34394af0860b39b308c1ebba53c9 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 3 Mar 2026 15:50:18 -0500 Subject: [PATCH 031/123] Made output a bit prettier. --- src/babel_explorer/core/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 43c3daf..653ccc8 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -282,7 +282,7 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): if meta is not None: # Tier 1: within freshness window — skip all network calls if self._is_within_freshness(meta, self.freshness_seconds): - self.logger.info(f"File within freshness window, skipping check: {local_path_to_download_to}") + self.logger.info(f"File within freshness window ({self.freshness_seconds} seconds), skipping check: {local_path_to_download_to}") return local_path_to_download_to # Tier 2: stale but maybe unchanged — HEAD request From 1cc5c7bfc6db46b435a53c5f57a0ebe877fc284e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 00:41:23 -0400 Subject: [PATCH 032/123] Update src/babel_explorer/core/nodenorm.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/core/nodenorm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index a9c6752..1e1e24e 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -57,11 +57,17 @@ def normalize_curie(self, curie: str, conflate=True, drug_chemical_conflate=True response.raise_for_status() result = response.json() - return result[curie] + try: + return result[curie] + except KeyError: + logging.debug(f"NodeNorm response did not contain CURIE {curie!r}; returning None") + return None @functools.lru_cache(maxsize=None) def get_clique_identifiers(self, curie, **kwargs): result = self.normalize_curie(curie, **kwargs) + if not result: + return None if 'equivalent_identifiers' not in result: return None return list(map(lambda x: Identifier.from_dict(x), result['equivalent_identifiers'])) From d1ec8fe409618c2d639e0f6bd798865f4a20e6c7 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 01:07:11 -0400 Subject: [PATCH 033/123] Simplify babel_xrefs: extract helper, remove dead fetches, fix default arg - Extract _to_labeled_xref() to eliminate duplicated LabeledCrossReference construction in get_curie_xref and _get_curie_xrefs_recursive - Remove unused concord_metadata_parquet fetches from get_curie_ids, get_curie_xref, and _get_curie_xrefs_recursive (Metadata.parquet was downloaded but never queried in any of these methods) - Fix mutable default argument: ignore_curies_in_expansion: set = set() -> set | None = None - Return list (not lazy map) from get_curie_xref for consistency - Update test expectation to match single downloader call (Concord only) Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/babel_xrefs.py | 42 +++++++++++--------------- tests/test_babel_xrefs.py | 4 +-- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index fba5b61..c07c095 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -81,7 +81,6 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: """ identifier_parquet = self.downloader.get_downloaded_file('duckdb/Identifiers.parquet') - concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') # Query the Parquet files using DuckDB (in-memory; nothing is persisted). db = duckdb.connect() @@ -94,34 +93,36 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: @functools.lru_cache(maxsize=None) def get_curie_xref(self, curie: str, label_curies: bool = False): concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') - concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') db = duckdb.connect() concord_table = db.read_parquet(concord_parquet) xref_tuples = db.execute(f"SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", [curie]).fetchall() - xrefs = list(map(lambda rec: CrossReference.from_tuple(rec), xref_tuples)) + xrefs = [CrossReference.from_tuple(rec) for rec in xref_tuples] if label_curies: - xrefs = map(lambda xref: LabeledCrossReference( - subj=xref.subj, - obj=xref.obj, - filename=xref.filename, - pred=xref.pred, - subj_label=self.nodenorm.get_identifier(xref.subj).label, - subj_biolink_type=self.nodenorm.get_identifier(xref.subj).biolink_type, - obj_label=self.nodenorm.get_identifier(xref.obj).label, - obj_biolink_type=self.nodenorm.get_identifier(xref.obj).biolink_type, - ), xrefs) + xrefs = [self._to_labeled_xref(xref) for xref in xrefs] return xrefs + def _to_labeled_xref(self, xref: CrossReference) -> LabeledCrossReference: + """Convert a CrossReference to a LabeledCrossReference using NodeNorm.""" + return LabeledCrossReference( + subj=xref.subj, + obj=xref.obj, + filename=xref.filename, + pred=xref.pred, + subj_label=self.nodenorm.get_identifier(xref.subj).label, + subj_biolink_type=self.nodenorm.get_identifier(xref.subj).biolink_type, + obj_label=self.nodenorm.get_identifier(xref.obj).label, + obj_biolink_type=self.nodenorm.get_identifier(xref.obj).biolink_type, + ) + def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = False): """Traverse the cross-reference graph in one DuckDB WITH RECURSIVE query.""" if not curies: return [] concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') - concord_metadata_parquet = self.downloader.get_downloaded_file('duckdb/Metadata.parquet') db = duckdb.connect() concord_table = db.read_parquet(concord_parquet) @@ -149,20 +150,11 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal xrefs = [CrossReference.from_tuple(row) for row in result.fetchall()] if label_curies: - xrefs = [LabeledCrossReference( - subj=xref.subj, - obj=xref.obj, - filename=xref.filename, - pred=xref.pred, - subj_label=self.nodenorm.get_identifier(xref.subj).label, - subj_biolink_type=self.nodenorm.get_identifier(xref.subj).biolink_type, - obj_label=self.nodenorm.get_identifier(xref.obj).label, - obj_biolink_type=self.nodenorm.get_identifier(xref.obj).biolink_type, - ) for xref in xrefs] + xrefs = [self._to_labeled_xref(xref) for xref in xrefs] return xrefs - def get_curie_xrefs(self, curies: list[str], recurse: bool = False, ignore_curies_in_expansion: set = set(), label_curies: bool = False): + def get_curie_xrefs(self, curies: list[str], recurse: bool = False, ignore_curies_in_expansion: set | None = None, label_curies: bool = False): """ Search for all identifiers that are cross-referenced to the given CURIE. diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 42fa6aa..41ad777 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -192,8 +192,8 @@ def test_get_curie_xref_calls_downloader(self, tmp_path): with patch("babel_explorer.core.babel_xrefs.duckdb.connect", return_value=mock_db): bx.get_curie_xref.cache_clear() result = bx.get_curie_xref("A:1") - # Downloader should be called for Concord and Metadata - assert mock_dl.call_count == 2 + # Downloader should be called for Concord only (Metadata unused here) + assert mock_dl.call_count == 1 result_list = list(result) assert len(result_list) == 1 assert isinstance(result_list[0], CrossReference) From ed1a27df553acd336a50f98e678d10410d3f22e2 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 01:45:18 -0400 Subject: [PATCH 034/123] Fix LabeledCrossReference: make it a frozen dataclass subclass Hand-written __init__ with post-construction setattr raised FrozenInstanceError since CrossReference is frozen=True. Adding @dataclasses.dataclass(frozen=True) lets Python generate the correct __init__ using object.__setattr__ internally. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/babel_xrefs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index c07c095..51727ab 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -29,19 +29,13 @@ def curies(self): def __lt__(self, other): return (self.filename, self.subj, self.obj, self.pred) < (other.filename, other.subj, other.obj, other.pred) +@dataclasses.dataclass(frozen=True) class LabeledCrossReference(CrossReference): subj_label: str subj_biolink_type: str obj_label: str obj_biolink_type: str - def __init__(self, subj: str, pred: str, obj: str, filename: str, subj_label: str, subj_biolink_type: str, obj_label: str, obj_biolink_type: str): - super().__init__(subj=subj, obj=obj, filename=filename, pred=pred) - self.subj_label = subj_label - self.subj_biolink_type = subj_biolink_type - self.obj_label = obj_label - self.obj_biolink_type = obj_biolink_type - def __str__(self): return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", subj_biolink_type="{self.subj_biolink_type}", obj_label="{self.obj_label}", obj_biolink_type="{self.obj_biolink_type}")""" From afd722d4b885e9f9a85ba2594c8e2385cf3e8246 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 01:53:41 -0400 Subject: [PATCH 035/123] Fix BabelDownloader: use tempfile.gettempdir() when local_path is None Replace the fragile TMPDIR-only env var check with tempfile.gettempdir(), which has a cross-platform fallback chain and always returns a valid path. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/downloader.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 653ccc8..57a0911 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -1,6 +1,7 @@ import functools import json import os +import tempfile import urllib.parse import time import requests @@ -22,11 +23,7 @@ def __init__(self, url_base, local_path=None, retries=10, freshness_seconds=3 * self.logger = logging.getLogger(BabelDownloader.__name__) if local_path is None: - # Default to using TMPDIR. - # TODO: replace with a real temporary directory. - tmpdir = os.environ.get("TMPDIR") - if tmpdir: - local_path = tmpdir + local_path = tempfile.gettempdir() # Make sure the local path is an existing directory or that we can create it. if not os.path.exists(local_path): From 26cef003ce389acaedc0301fd5f6da76aad2d2f1 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 02:09:28 -0400 Subject: [PATCH 036/123] Fix test-concord: guard against None from get_clique_identifiers When NodeNorm doesn't recognise a CURIE, get_clique_identifiers returns None, causing a TypeError on iteration. Use (identifiers or []) to skip gracefully. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 44955e5..768add6 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -95,7 +95,7 @@ def test_concord(curies, nodenorm_url): nodenorm = NodeNorm(nodenorm_url) for curie in curies: identifiers = nodenorm.get_clique_identifiers(curie) - for identifier in identifiers: + for identifier in (identifiers or []): if identifier.label: print(f"{curie}\t{identifier.curie}\t{identifier.label}\t{identifier.biolink_type}") else: From bdceca8f708f0f4caf37f600775a08df9499b645 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:10:07 -0400 Subject: [PATCH 037/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/core/nodenorm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index 1e1e24e..b48c4ec 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -30,6 +30,8 @@ def from_dict(d: dict): class NodeNorm: def __init__(self, nodenorm_url: str=""): self.nodenorm_url = nodenorm_url + if self.nodenorm_url and not self.nodenorm_url.endswith("/"): + self.nodenorm_url += "/" @functools.lru_cache(maxsize=None) def get_identifier(self, curie: str): From 2f439bc7449afb7c76bf71112e3a0c6e53ca2e90 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:10:46 -0400 Subject: [PATCH 038/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 768add6..f935b2e 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -30,7 +30,7 @@ def cli(): @click.option("--labels", is_flag=True, help="Include labels for CURIEs") @click.option("--check-download", type=str, default="3h", show_default=True, help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " - "'never' always checks via HTTP HEAD; '0' same as 'never'.") + "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.") def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, recurse: bool, labels: bool, check_download: str): """ @@ -62,7 +62,7 @@ def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, recur @click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") @click.option("--check-download", type=str, default="3h", show_default=True, help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " - "'never' always checks via HTTP HEAD; '0' same as 'never'.") + "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.") def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): """ Fetches and prints the ID records for the given CURIEs, along with Biolink type if provided. From 612e1cef8316ed8fd84cfe470ad41f102b6a703a Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:11:30 -0400 Subject: [PATCH 039/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78fa8e9..00fff8c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ uv sync --group dev uv run babel-explorer xrefs MONDO:0004979 # Get cross-references with expansion (recursive lookup) -uv run babel-explorer xrefs MONDO:0004979 --expand +uv run babel-explorer xrefs MONDO:0004979 --recurse # Get cross-references with labels from NodeNorm uv run babel-explorer xrefs MONDO:0004979 --labels From 52075cad069dd7c1440e0761783b5f21f256571e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:13:26 -0400 Subject: [PATCH 040/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/cli.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index f935b2e..03b87e8 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -9,12 +9,33 @@ def parse_duration(value: str) -> float: """Parse a duration string like '3h', '30m', '1d', '7200', or 'never' → seconds.""" units = {"s": 1, "m": 60, "h": 3600, "d": 86400} - lower = value.lower() + lower = (value or "").strip().lower() + if not lower: + raise click.BadParameter( + "Invalid duration: value cannot be empty. " + "Use an integer number of seconds, optionally followed by 's', 'm', 'h', or 'd', " + "or 'never'." + ) if lower == "never": return float("inf") + # Value with unit suffix (e.g. '3h', '30m') if lower[-1] in units: - return int(lower[:-1]) * units[lower[-1]] - return int(lower) # bare seconds + try: + amount = int(lower[:-1]) + except ValueError: + raise click.BadParameter( + f"Invalid duration {value!r}: expected an integer followed by an optional unit " + "('s', 'm', 'h', or 'd'), or 'never'." + ) + return amount * units[lower[-1]] + # Bare integer seconds + try: + return int(lower) + except ValueError: + raise click.BadParameter( + f"Invalid duration {value!r}: expected an integer number of seconds, optionally " + "followed by 's', 'm', 'h', or 'd', or 'never'." + ) @click.group() From 59c2c3c31a54d5985828733992853b25fcfd6489 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:13:40 -0400 Subject: [PATCH 041/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9fb436f..8f59cfb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ uv run babel-explorer --help uv run babel-explorer xrefs MONDO:0004979 # Get cross-references with expansion (recursive lookup) -uv run babel-explorer xrefs MONDO:0004979 --expand +uv run babel-explorer xrefs MONDO:0004979 --recurse # Get cross-references with labels from NodeNorm uv run babel-explorer xrefs MONDO:0004979 --labels From f2677fd5d2454609eb6a8e27ed69ac0b0e1f0782 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:15:10 -0400 Subject: [PATCH 042/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/core/downloader.py | 50 +++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 57a0911..39ea3fa 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -213,32 +213,32 @@ def _download_with_retry(self, url, local_path, chunk_size): self.logger.info(f"Resuming download from byte {resume_byte_pos}") # Make streaming request with timeout for connection (not total time) - response = requests.get(url, headers=headers, stream=True, timeout=30) - - # Handle different response codes - if response.status_code == 416: - # Range Not Satisfiable - file already complete - self.logger.info(f"File already complete: {local_path}") + with requests.get(url, headers=headers, stream=True, timeout=30) as response: + + # Handle different response codes + if response.status_code == 416: + # Range Not Satisfiable - file already complete + self.logger.info(f"File already complete: {local_path}") + return response.headers + elif response.status_code == 206: + # Partial Content - resume successful + self.logger.info(f"Resuming download (HTTP 206)") + elif response.status_code == 200: + # OK - server doesn't support resume or no Range header was sent + if resume_byte_pos > 0: + self.logger.warning(f"Server doesn't support resume, restarting from beginning") + resume_byte_pos = 0 + # Remove partial file + if os.path.exists(local_path): + os.remove(local_path) + else: + response.raise_for_status() + + # Stream download with progress bar + self._stream_download(response, local_path, resume_byte_pos, chunk_size) + + # Success - exit retry loop return response.headers - elif response.status_code == 206: - # Partial Content - resume successful - self.logger.info(f"Resuming download (HTTP 206)") - elif response.status_code == 200: - # OK - server doesn't support resume or no Range header was sent - if resume_byte_pos > 0: - self.logger.warning(f"Server doesn't support resume, restarting from beginning") - resume_byte_pos = 0 - # Remove partial file - if os.path.exists(local_path): - os.remove(local_path) - else: - response.raise_for_status() - - # Stream download with progress bar - self._stream_download(response, local_path, resume_byte_pos, chunk_size) - - # Success - exit retry loop - return response.headers except (requests.RequestException, IOError) as e: self.logger.warning(f"Download attempt {attempt}/{self.retries} failed: {e}") From f1c2857d428f9c91e0cb0e87bc72d51849925def Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:16:56 -0400 Subject: [PATCH 043/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/core/babel_xrefs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 51727ab..725e1f7 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -86,6 +86,9 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: @functools.lru_cache(maxsize=None) def get_curie_xref(self, curie: str, label_curies: bool = False): + if label_curies and self.nodenorm is None: + raise ValueError("label_curies=True requires a configured NodeNorm instance (nodenorm was None).") + concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') db = duckdb.connect() @@ -113,6 +116,8 @@ def _to_labeled_xref(self, xref: CrossReference) -> LabeledCrossReference: def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = False): """Traverse the cross-reference graph in one DuckDB WITH RECURSIVE query.""" + if label_curies and self.nodenorm is None: + raise ValueError("label_curies=True requires a configured NodeNorm instance (nodenorm was None).") if not curies: return [] From dd194e95292bd7300b717637d86c46415432f92c Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:21:57 -0400 Subject: [PATCH 044/123] Fix DuckDB connection leaks by using context managers Wrap all three duckdb.connect() calls in `with` statements so connections are deterministically closed after each query rather than relying on GC. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/babel_xrefs.py | 30 ++++++++++++-------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 725e1f7..f11f3f4 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -77,12 +77,11 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: identifier_parquet = self.downloader.get_downloaded_file('duckdb/Identifiers.parquet') # Query the Parquet files using DuckDB (in-memory; nothing is persisted). - db = duckdb.connect() - identifier_table = db.read_parquet(identifier_parquet) - result = db.execute(f"SELECT * FROM identifier_table WHERE curie IN $1", [curies]) - - column_names = [desc[0] for desc in result.description] - return [IdentifierRecord.from_row(row, column_names) for row in result.fetchall()] + with duckdb.connect() as db: + identifier_table = db.read_parquet(identifier_parquet) + result = db.execute("SELECT * FROM identifier_table WHERE curie IN $1", [curies]) + column_names = [desc[0] for desc in result.description] + return [IdentifierRecord.from_row(row, column_names) for row in result.fetchall()] @functools.lru_cache(maxsize=None) def get_curie_xref(self, curie: str, label_curies: bool = False): @@ -91,14 +90,13 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') - db = duckdb.connect() - concord_table = db.read_parquet(concord_parquet) - xref_tuples = db.execute(f"SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", [curie]).fetchall() - xrefs = [CrossReference.from_tuple(rec) for rec in xref_tuples] + with duckdb.connect() as db: + concord_table = db.read_parquet(concord_parquet) + xref_tuples = db.execute("SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", [curie]).fetchall() + xrefs = [CrossReference.from_tuple(rec) for rec in xref_tuples] if label_curies: xrefs = [self._to_labeled_xref(xref) for xref in xrefs] - return xrefs def _to_labeled_xref(self, xref: CrossReference) -> LabeledCrossReference: @@ -123,9 +121,9 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') - db = duckdb.connect() - concord_table = db.read_parquet(concord_parquet) - result = db.execute(""" + with duckdb.connect() as db: + concord_table = db.read_parquet(concord_parquet) + rows = db.execute(""" WITH RECURSIVE edges(a, b) AS ( SELECT subj, obj FROM concord_table @@ -144,9 +142,9 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal WHERE c.subj IN (SELECT curie FROM frontier) OR c.obj IN (SELECT curie FROM frontier) ORDER BY c.filename, c.subj, c.obj, c.pred - """, [curies]) + """, [curies]).fetchall() - xrefs = [CrossReference.from_tuple(row) for row in result.fetchall()] + xrefs = [CrossReference.from_tuple(row) for row in rows] if label_curies: xrefs = [self._to_labeled_xref(xref) for xref in xrefs] From cafe3035493242add27d89e5c34c928671b84122 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:36:53 -0400 Subject: [PATCH 045/123] Fix and simplify test mocks for context manager protocol After production code was updated to use `with duckdb.connect()` and `with requests.get()`, the test mocks (plain Mock()) no longer supported the context manager protocol. Updated affected mocks to MagicMock() with __enter__.return_value = self. Also extracted _make_response() helper in TestDownloadWithRetry to eliminate five near-identical 4-line mock setup blocks. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_babel_xrefs.py | 1 + tests/test_downloader.py | 39 +++++++++++++++------------------------ 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 41ad777..75e33cb 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -184,6 +184,7 @@ def test_get_curie_xref_calls_downloader(self, tmp_path): ("concord.tsv", "A:1", "skos:exactMatch", "B:2"), ] mock_db = MagicMock() + mock_db.__enter__.return_value = mock_db mock_db.read_parquet.return_value = "table" mock_db.execute.return_value = mock_result diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 045e402..16a7e9b 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -445,6 +445,16 @@ def fake_download(url, path, chunk_size): class TestDownloadWithRetry: """Tests for _download_with_retry.""" + @staticmethod + def _make_response(status_code, headers=None, content=None): + m = MagicMock() + m.__enter__.return_value = m + m.status_code = status_code + m.headers = headers or {} + if content is not None: + m.iter_content = Mock(return_value=content) + return m + def test_retries_exhausted_raises_runtime_error(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), retries=2) with patch("babel_explorer.core.downloader.requests.get", side_effect=requests.ConnectionError("fail")): @@ -456,11 +466,7 @@ def test_succeeds_on_second_attempt(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), retries=3) out_path = str(tmp_path / "retry_success.bin") - mock_response = Mock() - mock_response.status_code = 200 - mock_response.headers = {'Content-Length': '5'} - mock_response.iter_content = Mock(return_value=[b"hello"]) - + mock_response = self._make_response(200, {'Content-Length': '5'}, [b"hello"]) side_effects = [requests.ConnectionError("first fail"), mock_response] with patch("babel_explorer.core.downloader.requests.get", side_effect=side_effects): @@ -473,11 +479,7 @@ def test_resume_sends_range_header(self, tmp_path): out_path = tmp_path / "partial.bin" out_path.write_bytes(b"partial") # 7 bytes - mock_response = Mock() - mock_response.status_code = 206 - mock_response.headers = {'Content-Length': '3'} - mock_response.iter_content = Mock(return_value=[b"end"]) - + mock_response = self._make_response(206, {'Content-Length': '3'}, [b"end"]) with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response) as mock_get: dl._download_with_retry("https://example.com/file", str(out_path), 1024) _, kwargs = mock_get.call_args @@ -488,10 +490,7 @@ def test_http_416_file_already_complete(self, tmp_path): out_path = tmp_path / "complete.bin" out_path.write_bytes(b"full file") - mock_response = Mock() - mock_response.status_code = 416 - mock_response.headers = {} - + mock_response = self._make_response(416) with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): dl._download_with_retry("https://example.com/file", str(out_path), 1024) # Should return without error @@ -503,11 +502,7 @@ def test_server_no_resume_restarts_download(self, tmp_path): out_path = tmp_path / "no_resume.bin" out_path.write_bytes(b"partial") - mock_response = Mock() - mock_response.status_code = 200 - mock_response.headers = {'Content-Length': '12'} - mock_response.iter_content = Mock(return_value=[b"full content"]) - + mock_response = self._make_response(200, {'Content-Length': '12'}, [b"full content"]) with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): dl._download_with_retry("https://example.com/file", str(out_path), 1024) assert out_path.read_bytes() == b"full content" @@ -517,11 +512,7 @@ def test_returns_response_headers(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) out_path = str(tmp_path / "headers.bin") - mock_response = Mock() - mock_response.status_code = 200 - mock_response.headers = {'Content-Length': '5', 'ETag': '"abc"'} - mock_response.iter_content = Mock(return_value=[b"hello"]) - + mock_response = self._make_response(200, {'Content-Length': '5', 'ETag': '"abc"'}, [b"hello"]) with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): headers = dl._download_with_retry("https://example.com/file", out_path, 1024) assert headers['ETag'] == '"abc"' From 45250ab89bc94bdb621393eac4af772766b9f102 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:41:51 -0400 Subject: [PATCH 046/123] Add configurable HTTP timeout to NodeNorm and BabelDownloader normalize_curie() was calling requests.get() with no timeout, risking an indefinite hang if the NodeNorm service stalls. The downloader had timeout=30 hardcoded in two places with no way to override it. Add timeout: int = 30 to both constructors and thread self.timeout through all three request call sites, making the default consistent and the value overridable without patching. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/downloader.py | 7 ++++--- src/babel_explorer/core/nodenorm.py | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 39ea3fa..7b628d0 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -15,11 +15,12 @@ class BabelDownloader: Class for downloading Babel cross-reference files to a local directory as needed. """ - def __init__(self, url_base, local_path=None, retries=10, freshness_seconds=3 * 3600): + def __init__(self, url_base, local_path=None, retries=10, freshness_seconds=3 * 3600, timeout: int = 30): # We assume the URL base is correct (if not, we can fix it later). self.url_base = url_base self.retries = retries self.freshness_seconds = freshness_seconds + self.timeout = timeout self.logger = logging.getLogger(BabelDownloader.__name__) if local_path is None: @@ -114,7 +115,7 @@ def _etag_matches(self, url, meta): bool: True if remote matches local meta (file is still current) """ try: - response = requests.head(url, timeout=30) + response = requests.head(url, timeout=self.timeout) response.raise_for_status() except requests.RequestException as e: self.logger.warning(f"HEAD request failed for {url}: {e}") @@ -213,7 +214,7 @@ def _download_with_retry(self, url, local_path, chunk_size): self.logger.info(f"Resuming download from byte {resume_byte_pos}") # Make streaming request with timeout for connection (not total time) - with requests.get(url, headers=headers, stream=True, timeout=30) as response: + with requests.get(url, headers=headers, stream=True, timeout=self.timeout) as response: # Handle different response codes if response.status_code == 416: diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index b48c4ec..04a3629 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -28,8 +28,9 @@ def from_dict(d: dict): return identifier class NodeNorm: - def __init__(self, nodenorm_url: str=""): + def __init__(self, nodenorm_url: str = "", timeout: int = 30): self.nodenorm_url = nodenorm_url + self.timeout = timeout if self.nodenorm_url and not self.nodenorm_url.endswith("/"): self.nodenorm_url += "/" @@ -55,7 +56,7 @@ def normalize_curie(self, curie: str, conflate=True, drug_chemical_conflate=True "description": description, "individual_types": individual_types, "include_taxa": include_taxa, - }) + }, timeout=self.timeout) response.raise_for_status() result = response.json() From 7d67f00c90237172be5ae2a380bd4e6a0aa8663b Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:45:18 -0400 Subject: [PATCH 047/123] Fix _etag_matches docstring to match actual behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docstring claimed the method updates last_checked in the .meta file, but it is a pure predicate — the caller (get_downloaded_file) owns that write. Updated the docstring and removed stale inline comments that said the same thing. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/downloader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 7b628d0..016fb01 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -105,7 +105,10 @@ def _is_within_freshness(self, meta, freshness_seconds): def _etag_matches(self, url, meta): """ Do a HEAD request and check if the ETag (or Last-Modified + Content-Length) - matches the stored metadata. If they match, update last_checked in the .meta file. + matches the stored metadata. + + Does not write to disk — the caller is responsible for updating last_checked + when this returns True. Args: url: URL to HEAD @@ -129,9 +132,6 @@ def _etag_matches(self, url, meta): if local_etag and remote_etag: if local_etag == remote_etag: self.logger.info(f"ETag matches ({remote_etag}), file is current") - # Update last_checked in the .meta file - # We need the local_path to update — derive it from URL - # Caller will handle updating; return True return True else: self.logger.info(f"ETag changed: {local_etag!r} → {remote_etag!r}, re-downloading") From 597ab3012966380d5ec26298dd88f037d01774b0 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:51:16 -0400 Subject: [PATCH 048/123] Got rid of ignore_curies_in_expansion, which is no longer used. --- src/babel_explorer/core/babel_xrefs.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index f11f3f4..c017d6b 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -151,25 +151,17 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal return xrefs - def get_curie_xrefs(self, curies: list[str], recurse: bool = False, ignore_curies_in_expansion: set | None = None, label_curies: bool = False): + def get_curie_xrefs(self, curies: list[str], recurse: bool = False, label_curies: bool = False): """ Search for all identifiers that are cross-referenced to the given CURIE. :param curies: A list of CURIEs to search for. :param recurse: Whether to expand the cross-references (i.e. recursively follow all identifiers). - :param ignore_curies_in_expansion: Deprecated when recurse=True; has no effect. :param label_curies: Whether to annotate results with labels from NodeNorm. :return: A list of cross-references containing those CURIEs. """ if recurse: - if ignore_curies_in_expansion: - warnings.warn( - "ignore_curies_in_expansion has no effect when recurse=True; " - "cycle detection is handled automatically by the SQL query.", - DeprecationWarning, - stacklevel=2, - ) return self._get_curie_xrefs_recursive(curies, label_curies) xrefs = set() From ab0aa3375e54cc29fec491107e8295394b8d7650 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 16:56:11 -0400 Subject: [PATCH 049/123] Add ruff CI and fix all lint errors Add .github/workflows/lint.yml to run ruff check and ruff format --check on every pull request. Fix the 7 errors ruff found, and apply ruff format to all files: - Remove unused `import warnings` in babel_xrefs.py (left over after ignore_curies_in_expansion was removed) - Add # noqa: F841 to the three read_parquet() assignments: ruff flags them as unused, but DuckDB resolves SQL table names by matching the Python variable name, so the assignments are load-bearing - Remove spurious f-prefix from two string literals in downloader.py - Drop unused `local_path` variable in test_downloader.py Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/lint.yml | 14 ++ src/babel_explorer/cli.py | 93 +++++++++++--- src/babel_explorer/core/babel_xrefs.py | 64 +++++++--- src/babel_explorer/core/downloader.py | 86 +++++++++---- src/babel_explorer/core/nodenorm.py | 64 ++++++---- tests/conftest.py | 4 +- tests/test_babel_xrefs.py | 71 ++++++++--- tests/test_downloader.py | 169 +++++++++++++++++-------- tests/test_nodenorm.py | 28 ++-- 9 files changed, 427 insertions(+), 166 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..f4771d4 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,14 @@ +name: Lint + +on: + pull_request: + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv sync --group dev + - run: uv run ruff check src/ tests/ + - run: uv run ruff format --check src/ tests/ diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 03b87e8..463b1af 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -42,18 +42,46 @@ def parse_duration(value: str) -> float: def cli(): pass + @cli.command("xrefs") @click.argument("curies", type=str, required=True, nargs=-1) -@click.option("--local-dir", type=str, default="data/2025nov19", help="Local location to save Babel download files to") -@click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") -@click.option("--nodenorm-url", type=str, default="https://nodenormalization-sri.renci.org/", help="NodeNorm URL to check for concord changes") +@click.option( + "--local-dir", + type=str, + default="data/2025nov19", + help="Local location to save Babel download files to", +) +@click.option( + "--babel-url", + type=str, + default="https://stars.renci.org:443/var/babel/2025nov19/", + help="Base URL of the Babel server", +) +@click.option( + "--nodenorm-url", + type=str, + default="https://nodenormalization-sri.renci.org/", + help="NodeNorm URL to check for concord changes", +) @click.option("--recurse", is_flag=True, help="Recursively query returned xrefs") @click.option("--labels", is_flag=True, help="Include labels for CURIEs") -@click.option("--check-download", type=str, default="3h", show_default=True, - help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " - "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.") -def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, recurse: bool, labels: bool, - check_download: str): +@click.option( + "--check-download", + type=str, + default="3h", + show_default=True, + help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " + "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", +) +def xrefs( + curies: list[str], + babel_url: str, + nodenorm_url, + local_dir: str, + recurse: bool, + labels: bool, + check_download: str, +): """ Fetches and prints the cross-references (xrefs) for the given CURIEs. @@ -72,18 +100,37 @@ def xrefs(curies: list[str], babel_url: str, nodenorm_url, local_dir: str, recur logging.basicConfig(level=logging.INFO) freshness = parse_duration(check_download) - bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness), NodeNorm(nodenorm_url)) + bxref = BabelXRefs( + BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness), + NodeNorm(nodenorm_url), + ) xrefs = bxref.get_curie_xrefs(curies, recurse, label_curies=labels) for xref in xrefs: print(xref) + @cli.command("ids") @click.argument("curies", type=str, required=True, nargs=-1) -@click.option("--local-dir", type=str, default="data/2025nov19", help="Local location to save Babel download files to") -@click.option("--babel-url", type=str, default="https://stars.renci.org:443/var/babel/2025nov19/", help="Base URL of the Babel server") -@click.option("--check-download", type=str, default="3h", show_default=True, - help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " - "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.") +@click.option( + "--local-dir", + type=str, + default="data/2025nov19", + help="Local location to save Babel download files to", +) +@click.option( + "--babel-url", + type=str, + default="https://stars.renci.org:443/var/babel/2025nov19/", + help="Base URL of the Babel server", +) +@click.option( + "--check-download", + type=str, + default="3h", + show_default=True, + help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " + "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", +) def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): """ Fetches and prints the ID records for the given CURIEs, along with Biolink type if provided. @@ -101,14 +148,22 @@ def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): logging.basicConfig(level=logging.INFO) freshness = parse_duration(check_download) - bxref = BabelXRefs(BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness)) + bxref = BabelXRefs( + BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness) + ) xrefs = bxref.get_curie_ids(curies) for xref in xrefs: print(xref) + @cli.command("test-concord") @click.argument("curies", type=str, required=True, nargs=-1) -@click.option("--nodenorm-url", type=str, default="https://nodenormalization-sri.renci.org/", help="NodeNorm URL to check for concord changes") +@click.option( + "--nodenorm-url", + type=str, + default="https://nodenormalization-sri.renci.org/", + help="NodeNorm URL to check for concord changes", +) def test_concord(curies, nodenorm_url): # We're trying to answer a simple question here: if the CURIEs we mention were combined, how would the cliques change in NodeNorm? # By definition, this can only combine all the cliques mentioned in the CURIEs. @@ -116,9 +171,11 @@ def test_concord(curies, nodenorm_url): nodenorm = NodeNorm(nodenorm_url) for curie in curies: identifiers = nodenorm.get_clique_identifiers(curie) - for identifier in (identifiers or []): + for identifier in identifiers or []: if identifier.label: - print(f"{curie}\t{identifier.curie}\t{identifier.label}\t{identifier.biolink_type}") + print( + f"{curie}\t{identifier.curie}\t{identifier.label}\t{identifier.biolink_type}" + ) else: print(f"{curie}\t{identifier.curie}\t\t{identifier.biolink_type}") diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index c017d6b..c218761 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -3,7 +3,6 @@ # why we consider two identifiers to be identical. import dataclasses import logging -import warnings import duckdb import functools @@ -20,14 +19,22 @@ class CrossReference: @staticmethod def from_tuple(tuple: tuple[str, str, str, str]): - return CrossReference(filename=tuple[0], subj=tuple[1], pred=tuple[2], obj=tuple[3]) + return CrossReference( + filename=tuple[0], subj=tuple[1], pred=tuple[2], obj=tuple[3] + ) @property def curies(self): return frozenset([self.subj, self.obj]) def __lt__(self, other): - return (self.filename, self.subj, self.obj, self.pred) < (other.filename, other.subj, other.obj, other.pred) + return (self.filename, self.subj, self.obj, self.pred) < ( + other.filename, + other.subj, + other.obj, + other.pred, + ) + @dataclasses.dataclass(frozen=True) class LabeledCrossReference(CrossReference): @@ -39,16 +46,18 @@ class LabeledCrossReference(CrossReference): def __str__(self): return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", subj_biolink_type="{self.subj_biolink_type}", obj_label="{self.obj_label}", obj_biolink_type="{self.obj_biolink_type}")""" + @dataclasses.dataclass(frozen=True) class IdentifierRecord: """A record from the Identifiers.parquet file.""" + curie: str extra_fields: tuple = () @staticmethod def from_row(row: tuple, column_names: list[str]): """Create an IdentifierRecord from a DuckDB result row and its column names.""" - curie_idx = column_names.index('curie') + curie_idx = column_names.index("curie") extra = tuple( (col, row[i]) for i, col in enumerate(column_names) if i != curie_idx ) @@ -74,25 +83,37 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: :return: A list of IdentifierRecords containing those CURIEs. """ - identifier_parquet = self.downloader.get_downloaded_file('duckdb/Identifiers.parquet') + identifier_parquet = self.downloader.get_downloaded_file( + "duckdb/Identifiers.parquet" + ) # Query the Parquet files using DuckDB (in-memory; nothing is persisted). with duckdb.connect() as db: - identifier_table = db.read_parquet(identifier_parquet) - result = db.execute("SELECT * FROM identifier_table WHERE curie IN $1", [curies]) + identifier_table = db.read_parquet(identifier_parquet) # noqa: F841 — DuckDB resolves 'identifier_table' by Python variable name in the SQL query + result = db.execute( + "SELECT * FROM identifier_table WHERE curie IN $1", [curies] + ) column_names = [desc[0] for desc in result.description] - return [IdentifierRecord.from_row(row, column_names) for row in result.fetchall()] + return [ + IdentifierRecord.from_row(row, column_names) + for row in result.fetchall() + ] @functools.lru_cache(maxsize=None) def get_curie_xref(self, curie: str, label_curies: bool = False): if label_curies and self.nodenorm is None: - raise ValueError("label_curies=True requires a configured NodeNorm instance (nodenorm was None).") + raise ValueError( + "label_curies=True requires a configured NodeNorm instance (nodenorm was None)." + ) - concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') + concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") with duckdb.connect() as db: - concord_table = db.read_parquet(concord_parquet) - xref_tuples = db.execute("SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", [curie]).fetchall() + concord_table = db.read_parquet(concord_parquet) # noqa: F841 — DuckDB resolves 'concord_table' by Python variable name in the SQL query + xref_tuples = db.execute( + "SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", + [curie], + ).fetchall() xrefs = [CrossReference.from_tuple(rec) for rec in xref_tuples] if label_curies: @@ -115,15 +136,18 @@ def _to_labeled_xref(self, xref: CrossReference) -> LabeledCrossReference: def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = False): """Traverse the cross-reference graph in one DuckDB WITH RECURSIVE query.""" if label_curies and self.nodenorm is None: - raise ValueError("label_curies=True requires a configured NodeNorm instance (nodenorm was None).") + raise ValueError( + "label_curies=True requires a configured NodeNorm instance (nodenorm was None)." + ) if not curies: return [] - concord_parquet = self.downloader.get_downloaded_file('duckdb/Concord.parquet') + concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") with duckdb.connect() as db: - concord_table = db.read_parquet(concord_parquet) - rows = db.execute(""" + concord_table = db.read_parquet(concord_parquet) # noqa: F841 — DuckDB resolves 'concord_table' by Python variable name in the SQL query + rows = db.execute( + """ WITH RECURSIVE edges(a, b) AS ( SELECT subj, obj FROM concord_table @@ -142,7 +166,9 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal WHERE c.subj IN (SELECT curie FROM frontier) OR c.obj IN (SELECT curie FROM frontier) ORDER BY c.filename, c.subj, c.obj, c.pred - """, [curies]).fetchall() + """, + [curies], + ).fetchall() xrefs = [CrossReference.from_tuple(row) for row in rows] @@ -151,7 +177,9 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal return xrefs - def get_curie_xrefs(self, curies: list[str], recurse: bool = False, label_curies: bool = False): + def get_curie_xrefs( + self, curies: list[str], recurse: bool = False, label_curies: bool = False + ): """ Search for all identifiers that are cross-referenced to the given CURIE. diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 016fb01..6ba9a38 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -15,7 +15,14 @@ class BabelDownloader: Class for downloading Babel cross-reference files to a local directory as needed. """ - def __init__(self, url_base, local_path=None, retries=10, freshness_seconds=3 * 3600, timeout: int = 30): + def __init__( + self, + url_base, + local_path=None, + retries=10, + freshness_seconds=3 * 3600, + timeout: int = 30, + ): # We assume the URL base is correct (if not, we can fix it later). self.url_base = url_base self.retries = retries @@ -33,7 +40,9 @@ def __init__(self, url_base, local_path=None, retries=10, freshness_seconds=3 * elif os.path.exists(local_path) and os.path.isdir(local_path): self.local_path = local_path else: - raise ValueError(f"Invalid local_path (must be an existing directory): '{local_path}'") + raise ValueError( + f"Invalid local_path (must be an existing directory): '{local_path}'" + ) @functools.lru_cache(maxsize=None) def get_output_file(self, filename): @@ -134,7 +143,9 @@ def _etag_matches(self, url, meta): self.logger.info(f"ETag matches ({remote_etag}), file is current") return True else: - self.logger.info(f"ETag changed: {local_etag!r} → {remote_etag!r}, re-downloading") + self.logger.info( + f"ETag changed: {local_etag!r} → {remote_etag!r}, re-downloading" + ) return False # Fallback: Last-Modified + Content-Length @@ -145,10 +156,14 @@ def _etag_matches(self, url, meta): if local_lm and remote_lm and local_lm == remote_lm: if local_cl is None or remote_cl is None or int(remote_cl) == local_cl: - self.logger.info(f"Last-Modified matches ({remote_lm}), file is current") + self.logger.info( + f"Last-Modified matches ({remote_lm}), file is current" + ) return True - self.logger.info("Cannot confirm file is current (no matching ETag or Last-Modified), will re-download") + self.logger.info( + "Cannot confirm file is current (no matching ETag or Last-Modified), will re-download" + ) return False def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): @@ -162,23 +177,23 @@ def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): chunk_size: Size of chunks to read/write """ # Get total size from Content-Length header (may not be present) - content_length = response.headers.get('Content-Length') + content_length = response.headers.get("Content-Length") if content_length: total_size = int(content_length) + resume_byte_pos else: total_size = None # Open file in append mode if resuming, write mode otherwise - mode = 'ab' if resume_byte_pos > 0 else 'wb' + mode = "ab" if resume_byte_pos > 0 else "wb" with open(local_path, mode) as f: with tqdm( total=total_size, initial=resume_byte_pos, - unit='B', + unit="B", unit_scale=True, unit_divisor=1024, - desc=os.path.basename(local_path) + desc=os.path.basename(local_path), ) as progress_bar: for chunk in response.iter_content(chunk_size=chunk_size): if chunk: @@ -210,12 +225,13 @@ def _download_with_retry(self, url, local_path, chunk_size): # Prepare headers for resume headers = {} if resume_byte_pos > 0: - headers['Range'] = f'bytes={resume_byte_pos}-' + headers["Range"] = f"bytes={resume_byte_pos}-" self.logger.info(f"Resuming download from byte {resume_byte_pos}") # Make streaming request with timeout for connection (not total time) - with requests.get(url, headers=headers, stream=True, timeout=self.timeout) as response: - + with requests.get( + url, headers=headers, stream=True, timeout=self.timeout + ) as response: # Handle different response codes if response.status_code == 416: # Range Not Satisfiable - file already complete @@ -223,11 +239,13 @@ def _download_with_retry(self, url, local_path, chunk_size): return response.headers elif response.status_code == 206: # Partial Content - resume successful - self.logger.info(f"Resuming download (HTTP 206)") + self.logger.info("Resuming download (HTTP 206)") elif response.status_code == 200: # OK - server doesn't support resume or no Range header was sent if resume_byte_pos > 0: - self.logger.warning(f"Server doesn't support resume, restarting from beginning") + self.logger.warning( + "Server doesn't support resume, restarting from beginning" + ) resume_byte_pos = 0 # Remove partial file if os.path.exists(local_path): @@ -236,25 +254,31 @@ def _download_with_retry(self, url, local_path, chunk_size): response.raise_for_status() # Stream download with progress bar - self._stream_download(response, local_path, resume_byte_pos, chunk_size) + self._stream_download( + response, local_path, resume_byte_pos, chunk_size + ) # Success - exit retry loop return response.headers except (requests.RequestException, IOError) as e: - self.logger.warning(f"Download attempt {attempt}/{self.retries} failed: {e}") + self.logger.warning( + f"Download attempt {attempt}/{self.retries} failed: {e}" + ) if attempt < self.retries: # Calculate exponential backoff with max of 60 seconds - wait_time = min(2 ** attempt, 60) + wait_time = min(2**attempt, 60) self.logger.info(f"Retrying in {wait_time} seconds...") time.sleep(wait_time) else: # All retries exhausted - raise RuntimeError(f"Failed to download {url} after {self.retries} attempts: {e}") + raise RuntimeError( + f"Failed to download {url} after {self.retries} attempts: {e}" + ) @functools.lru_cache(maxsize=None) - def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): + def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): """ Download a file from the Babel server to local storage with ETag-based caching. @@ -280,7 +304,9 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): if meta is not None: # Tier 1: within freshness window — skip all network calls if self._is_within_freshness(meta, self.freshness_seconds): - self.logger.info(f"File within freshness window ({self.freshness_seconds} seconds), skipping check: {local_path_to_download_to}") + self.logger.info( + f"File within freshness window ({self.freshness_seconds} seconds), skipping check: {local_path_to_download_to}" + ) return local_path_to_download_to # Tier 2: stale but maybe unchanged — HEAD request @@ -290,24 +316,34 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024*1024): meta_path = self._get_meta_path(local_path_to_download_to) with open(meta_path, "w") as f: json.dump(meta, f, indent=2) - self.logger.info(f"ETag matches, using existing file: {local_path_to_download_to}") + self.logger.info( + f"ETag matches, using existing file: {local_path_to_download_to}" + ) return local_path_to_download_to # Tier 3: ETag changed — delete and re-download - self.logger.warning(f"Remote file changed, re-downloading: {local_path_to_download_to}") + self.logger.warning( + f"Remote file changed, re-downloading: {local_path_to_download_to}" + ) os.remove(local_path_to_download_to) - self.logger.info(f"Downloading {url_to_download} to {local_path_to_download_to}") + self.logger.info( + f"Downloading {url_to_download} to {local_path_to_download_to}" + ) # Download with retry logic; get response headers back - response_headers = self._download_with_retry(url_to_download, local_path_to_download_to, chunk_size) + response_headers = self._download_with_retry( + url_to_download, local_path_to_download_to, chunk_size + ) # Save sidecar metadata if response_headers is not None: self._save_meta(local_path_to_download_to, response_headers) bytes_downloaded = os.path.getsize(local_path_to_download_to) - self.logger.info(f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes") + self.logger.info( + f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes" + ) return local_path_to_download_to @functools.lru_cache(maxsize=None) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index 04a3629..ec5b0a8 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -3,6 +3,7 @@ import requests import logging + @dataclasses.dataclass class Identifier: curie: str @@ -16,17 +17,18 @@ def __lt__(self, other): @staticmethod def from_dict(d: dict): - identifier = Identifier(curie=d['identifier']) - if 'label' in d: - identifier.label = d['label'] - if 'taxa' in d: - identifier.taxa = d['taxa'] - if 'description' in d: - identifier.description = d['description'] - if 'type' in d: - identifier.biolink_type = d['type'] + identifier = Identifier(curie=d["identifier"]) + if "label" in d: + identifier.label = d["label"] + if "taxa" in d: + identifier.taxa = d["taxa"] + if "description" in d: + identifier.description = d["description"] + if "type" in d: + identifier.biolink_type = d["type"] return identifier + class NodeNorm: def __init__(self, nodenorm_url: str = "", timeout: int = 30): self.nodenorm_url = nodenorm_url @@ -40,30 +42,44 @@ def get_identifier(self, curie: str): logging.debug(f"Normalizing {curie} with NodeNorm to result: {result}") if not result: return Identifier(curie=curie) - for identifier in result.get('equivalent_identifiers', []): - if identifier['identifier'] == curie: + for identifier in result.get("equivalent_identifiers", []): + if identifier["identifier"] == curie: logging.debug(f"Found exact match for {curie}: {identifier}") return Identifier.from_dict(identifier) return Identifier(curie=curie) @functools.lru_cache(maxsize=None) - def normalize_curie(self, curie: str, conflate=True, drug_chemical_conflate=True, description=True, individual_types=True, include_taxa=True): - response = requests.get(f"{self.nodenorm_url}get_normalized_nodes", params={ - "curie": curie, - "conflate": conflate, - "drug_chemical_conflate": drug_chemical_conflate, - "description": description, - "individual_types": individual_types, - "include_taxa": include_taxa, - }, timeout=self.timeout) + def normalize_curie( + self, + curie: str, + conflate=True, + drug_chemical_conflate=True, + description=True, + individual_types=True, + include_taxa=True, + ): + response = requests.get( + f"{self.nodenorm_url}get_normalized_nodes", + params={ + "curie": curie, + "conflate": conflate, + "drug_chemical_conflate": drug_chemical_conflate, + "description": description, + "individual_types": individual_types, + "include_taxa": include_taxa, + }, + timeout=self.timeout, + ) response.raise_for_status() result = response.json() try: return result[curie] except KeyError: - logging.debug(f"NodeNorm response did not contain CURIE {curie!r}; returning None") + logging.debug( + f"NodeNorm response did not contain CURIE {curie!r}; returning None" + ) return None @functools.lru_cache(maxsize=None) @@ -71,6 +87,8 @@ def get_clique_identifiers(self, curie, **kwargs): result = self.normalize_curie(curie, **kwargs) if not result: return None - if 'equivalent_identifiers' not in result: + if "equivalent_identifiers" not in result: return None - return list(map(lambda x: Identifier.from_dict(x), result['equivalent_identifiers'])) + return list( + map(lambda x: Identifier.from_dict(x), result["equivalent_identifiers"]) + ) diff --git a/tests/conftest.py b/tests/conftest.py index f1e0df6..92ecb06 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,6 +116,8 @@ def babel_xrefs(shared_downloader, downloaded_parquet_files) -> BabelXRefs: @pytest.fixture(scope="session") -def babel_xrefs_with_nodenorm(shared_downloader, nodenorm, downloaded_parquet_files) -> BabelXRefs: +def babel_xrefs_with_nodenorm( + shared_downloader, nodenorm, downloaded_parquet_files +) -> BabelXRefs: """A BabelXRefs instance with NodeNorm, Concord + Metadata already downloaded.""" return BabelXRefs(shared_downloader, nodenorm) diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 75e33cb..d67f81b 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -28,7 +28,9 @@ class TestCrossReference: def test_creation(self): - xr = CrossReference(filename="f.txt", subj="A:1", pred="skos:exactMatch", obj="B:2") + xr = CrossReference( + filename="f.txt", subj="A:1", pred="skos:exactMatch", obj="B:2" + ) assert xr.filename == "f.txt" assert xr.subj == "A:1" assert xr.pred == "skos:exactMatch" @@ -85,9 +87,14 @@ def test_sorting(self): class TestLabeledCrossReference: def test_creation(self): lxr = LabeledCrossReference( - subj="A:1", pred="p", obj="B:2", filename="f", - subj_label="Alpha", subj_biolink_type="biolink:Disease", - obj_label="Beta", obj_biolink_type="biolink:Gene", + subj="A:1", + pred="p", + obj="B:2", + filename="f", + subj_label="Alpha", + subj_biolink_type="biolink:Disease", + obj_label="Beta", + obj_biolink_type="biolink:Gene", ) assert lxr.subj == "A:1" assert lxr.subj_label == "Alpha" @@ -95,23 +102,40 @@ def test_creation(self): def test_inherits_from_cross_reference(self): lxr = LabeledCrossReference( - subj="A:1", pred="p", obj="B:2", filename="f", - subj_label="", subj_biolink_type="", obj_label="", obj_biolink_type="", + subj="A:1", + pred="p", + obj="B:2", + filename="f", + subj_label="", + subj_biolink_type="", + obj_label="", + obj_biolink_type="", ) assert isinstance(lxr, CrossReference) def test_curies_property(self): lxr = LabeledCrossReference( - subj="A:1", pred="p", obj="B:2", filename="f", - subj_label="", subj_biolink_type="", obj_label="", obj_biolink_type="", + subj="A:1", + pred="p", + obj="B:2", + filename="f", + subj_label="", + subj_biolink_type="", + obj_label="", + obj_biolink_type="", ) assert lxr.curies == frozenset({"A:1", "B:2"}) def test_str(self): lxr = LabeledCrossReference( - subj="A:1", pred="p", obj="B:2", filename="f", - subj_label="Alpha", subj_biolink_type="biolink:Disease", - obj_label="Beta", obj_biolink_type="biolink:Gene", + subj="A:1", + pred="p", + obj="B:2", + filename="f", + subj_label="Alpha", + subj_biolink_type="biolink:Disease", + obj_label="Beta", + obj_biolink_type="biolink:Gene", ) s = str(lxr) assert "A:1" in s @@ -188,9 +212,16 @@ def test_get_curie_xref_calls_downloader(self, tmp_path): mock_db.read_parquet.return_value = "table" mock_db.execute.return_value = mock_result - with patch.object(bx.downloader, 'get_downloaded_file', return_value="/fake/path") as mock_dl: - with patch.object(bx.downloader, 'get_output_file', return_value="/fake/db"): - with patch("babel_explorer.core.babel_xrefs.duckdb.connect", return_value=mock_db): + with patch.object( + bx.downloader, "get_downloaded_file", return_value="/fake/path" + ) as mock_dl: + with patch.object( + bx.downloader, "get_output_file", return_value="/fake/db" + ): + with patch( + "babel_explorer.core.babel_xrefs.duckdb.connect", + return_value=mock_db, + ): bx.get_curie_xref.cache_clear() result = bx.get_curie_xref("A:1") # Downloader should be called for Concord only (Metadata unused here) @@ -202,7 +233,7 @@ def test_get_curie_xref_calls_downloader(self, tmp_path): def test_get_curie_xrefs_no_expand(self, tmp_path): bx = self._make_bx(tmp_path) xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") - with patch.object(bx, 'get_curie_xref', return_value=[xr]): + with patch.object(bx, "get_curie_xref", return_value=[xr]): bx.get_curie_xref.cache_clear() result = bx.get_curie_xrefs(["A:1"], recurse=False) assert len(result) == 1 @@ -213,7 +244,9 @@ def test_get_curie_xrefs_with_expand(self, tmp_path): xr1 = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") xr2 = CrossReference(filename="f", subj="B:2", pred="p", obj="C:3") - with patch.object(bx, '_get_curie_xrefs_recursive', return_value=[xr1, xr2]) as mock_rec: + with patch.object( + bx, "_get_curie_xrefs_recursive", return_value=[xr1, xr2] + ) as mock_rec: result = bx.get_curie_xrefs(["A:1"], recurse=True) mock_rec.assert_called_once_with(["A:1"], False) assert xr1 in result @@ -239,7 +272,9 @@ def test_get_curie_xrefs_recursive_sql_traversal(self, tmp_path): """) setup_db.close() - with patch.object(bx.downloader, 'get_downloaded_file', return_value=parquet_path): + with patch.object( + bx.downloader, "get_downloaded_file", return_value=parquet_path + ): # Starting from A:1 should reach B:2 and C:3 but not the D-E component result = bx._get_curie_xrefs_recursive(["A:1"]) pairs = {(xr.subj, xr.obj) for xr in result} @@ -262,7 +297,7 @@ def test_results_are_sorted(self, tmp_path): xr_b = CrossReference(filename="b", subj="B:1", pred="p", obj="C:1") xr_a = CrossReference(filename="a", subj="A:1", pred="p", obj="B:1") - with patch.object(bx, 'get_curie_xref', return_value=[xr_b, xr_a]): + with patch.object(bx, "get_curie_xref", return_value=[xr_b, xr_a]): result = bx.get_curie_xrefs(["X:1"], recurse=False) assert result == [xr_a, xr_b] diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 16a7e9b..9b33e7a 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -39,7 +39,9 @@ def test_creates_directory_if_missing(self, tmp_path): assert dl.local_path == new_dir def test_custom_retries(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), retries=3) + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) assert dl.retries == 3 def test_default_retries(self, tmp_path): @@ -51,7 +53,11 @@ def test_default_freshness_seconds(self, tmp_path): assert dl.freshness_seconds == 3 * 3600 def test_custom_freshness_seconds(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), freshness_seconds=0) + dl = BabelDownloader( + url_base="https://example.com/", + local_path=str(tmp_path), + freshness_seconds=0, + ) assert dl.freshness_seconds == 0 def test_invalid_path_raises_value_error(self): @@ -90,13 +96,15 @@ class TestSaveMeta: """Tests for _save_meta.""" def _make_dl(self, tmp_path): - return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + return BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path) + ) def test_writes_all_fields(self, tmp_path): dl = self._make_dl(tmp_path) file_path = str(tmp_path / "test.parquet") # Create the file so the path is valid - open(file_path, 'wb').close() + open(file_path, "wb").close() headers = { "ETag": '"abc123"', @@ -118,7 +126,7 @@ def test_writes_all_fields(self, tmp_path): def test_last_checked_is_recent_utc(self, tmp_path): dl = self._make_dl(tmp_path) file_path = str(tmp_path / "f.parquet") - open(file_path, 'wb').close() + open(file_path, "wb").close() dl._save_meta(file_path, {"ETag": '"x"'}) @@ -133,7 +141,7 @@ def test_missing_headers_not_written(self, tmp_path): """Headers not present in the response should not appear in .meta.""" dl = self._make_dl(tmp_path) file_path = str(tmp_path / "sparse.parquet") - open(file_path, 'wb').close() + open(file_path, "wb").close() dl._save_meta(file_path, {}) @@ -150,7 +158,9 @@ class TestLoadMeta: """Tests for _load_meta.""" def _make_dl(self, tmp_path): - return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + return BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path) + ) def test_returns_none_if_no_meta_file(self, tmp_path): dl = self._make_dl(tmp_path) @@ -159,7 +169,7 @@ def test_returns_none_if_no_meta_file(self, tmp_path): def test_returns_dict_for_valid_meta(self, tmp_path): dl = self._make_dl(tmp_path) file_path = str(tmp_path / "f.parquet") - open(file_path, 'wb').close() + open(file_path, "wb").close() meta_data = {"etag": '"abc"', "last_checked": "2026-01-01T00:00:00+00:00"} with open(file_path + ".meta", "w") as f: json.dump(meta_data, f) @@ -170,7 +180,7 @@ def test_returns_dict_for_valid_meta(self, tmp_path): def test_returns_none_for_corrupt_meta(self, tmp_path): dl = self._make_dl(tmp_path) file_path = str(tmp_path / "corrupt.parquet") - open(file_path, 'wb').close() + open(file_path, "wb").close() with open(file_path + ".meta", "w") as f: f.write("not valid json {{{") @@ -181,7 +191,9 @@ class TestIsWithinFreshness: """Tests for _is_within_freshness.""" def _make_dl(self, tmp_path): - return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + return BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path) + ) def test_returns_true_when_recent(self, tmp_path): dl = self._make_dl(tmp_path) @@ -217,7 +229,9 @@ class TestEtagMatches: """Tests for _etag_matches.""" def _make_dl(self, tmp_path): - return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + return BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path) + ) def test_returns_true_on_matching_etag(self, tmp_path): dl = self._make_dl(tmp_path) @@ -225,7 +239,9 @@ def test_returns_true_on_matching_etag(self, tmp_path): mock_resp = Mock() mock_resp.headers = {"ETag": '"abc123"'} mock_resp.raise_for_status = Mock() - with patch("babel_explorer.core.downloader.requests.head", return_value=mock_resp): + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_resp + ): assert dl._etag_matches("https://example.com/f.parquet", meta) is True def test_returns_false_on_different_etag(self, tmp_path): @@ -234,7 +250,9 @@ def test_returns_false_on_different_etag(self, tmp_path): mock_resp = Mock() mock_resp.headers = {"ETag": '"new"'} mock_resp.raise_for_status = Mock() - with patch("babel_explorer.core.downloader.requests.head", return_value=mock_resp): + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_resp + ): assert dl._etag_matches("https://example.com/f.parquet", meta) is False def test_fallback_last_modified_match(self, tmp_path): @@ -244,14 +262,18 @@ def test_fallback_last_modified_match(self, tmp_path): mock_resp = Mock() mock_resp.headers = {"Last-Modified": lm, "Content-Length": "100"} mock_resp.raise_for_status = Mock() - with patch("babel_explorer.core.downloader.requests.head", return_value=mock_resp): + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_resp + ): assert dl._etag_matches("https://example.com/f.parquet", meta) is True def test_returns_false_on_request_error(self, tmp_path): dl = self._make_dl(tmp_path) meta = {"etag": '"abc"'} - with patch("babel_explorer.core.downloader.requests.head", - side_effect=requests.ConnectionError("fail")): + with patch( + "babel_explorer.core.downloader.requests.head", + side_effect=requests.ConnectionError("fail"), + ): assert dl._etag_matches("https://example.com/f.parquet", meta) is False @@ -259,8 +281,11 @@ class TestGetDownloadedFileTiers: """Tests for the three-tier logic in get_downloaded_file.""" def _make_dl(self, tmp_path, freshness=3600): - return BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), - freshness_seconds=freshness) + return BabelDownloader( + url_base="https://example.com/", + local_path=str(tmp_path), + freshness_seconds=freshness, + ) # --- Tier 1: within freshness window --- @@ -303,7 +328,9 @@ def test_tier2_head_check_no_redownload(self, tmp_path): mock_head_resp.headers = {"ETag": '"abc"'} mock_head_resp.raise_for_status = Mock() - with patch("babel_explorer.core.downloader.requests.head", return_value=mock_head_resp): + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp + ): with patch("babel_explorer.core.downloader.requests.get") as mock_get: dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) @@ -327,7 +354,9 @@ def test_tier2_updates_last_checked_after_head(self, tmp_path): mock_head_resp.headers = {"ETag": '"abc"'} mock_head_resp.raise_for_status = Mock() - with patch("babel_explorer.core.downloader.requests.head", return_value=mock_head_resp): + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp + ): dl.get_downloaded_file.cache_clear() dl.get_downloaded_file(test_file) @@ -358,16 +387,18 @@ def test_tier3_redownloads_when_etag_changed(self, tmp_path): new_content = b"new data" def fake_download(url, path, chunk_size): - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(new_content) return {"ETag": '"new"', "Content-Length": str(len(new_content))} - with patch("babel_explorer.core.downloader.requests.head", return_value=mock_head_resp): - with patch.object(dl, '_download_with_retry', side_effect=fake_download): + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp + ): + with patch.object(dl, "_download_with_retry", side_effect=fake_download): dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) - assert open(result, 'rb').read() == new_content + assert open(result, "rb").read() == new_content # --- No .meta: fresh download --- @@ -375,22 +406,23 @@ def test_downloads_when_no_meta(self, tmp_path): """No file and no .meta → download happens, .meta is saved.""" dl = self._make_dl(tmp_path) test_file = "duckdb/new.parquet" - local_path = str(tmp_path / "duckdb" / "new.parquet") content = b"fresh download" def fake_download(url, path, chunk_size): os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(content) return {"ETag": '"fresh"', "Content-Length": str(len(content))} - with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: + with patch.object( + dl, "_download_with_retry", side_effect=fake_download + ) as mock_dl: dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) mock_dl.assert_called_once() assert os.path.exists(result) - assert open(result, 'rb').read() == content + assert open(result, "rb").read() == content # .meta should be saved meta_path = result + ".meta" assert os.path.exists(meta_path) @@ -410,16 +442,18 @@ def test_downloads_when_file_exists_but_no_meta(self, tmp_path): new_content = b"refreshed" def fake_download(url, path, chunk_size): - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(new_content) return {"ETag": '"new"'} - with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: + with patch.object( + dl, "_download_with_retry", side_effect=fake_download + ) as mock_dl: dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) mock_dl.assert_called_once() - assert open(result, 'rb').read() == new_content + assert open(result, "rb").read() == new_content class TestGetDownloadedFileCaching: @@ -430,11 +464,13 @@ def test_cache_returns_same_result(self, tmp_path): content = b"cached content" def fake_download(url, path, chunk_size): - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(content) return {} - with patch.object(dl, '_download_with_retry', side_effect=fake_download) as mock_dl: + with patch.object( + dl, "_download_with_retry", side_effect=fake_download + ) as mock_dl: dl.get_downloaded_file.cache_clear() r1 = dl.get_downloaded_file("cached.txt") r2 = dl.get_downloaded_file("cached.txt") @@ -456,20 +492,31 @@ def _make_response(status_code, headers=None, content=None): return m def test_retries_exhausted_raises_runtime_error(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), retries=2) - with patch("babel_explorer.core.downloader.requests.get", side_effect=requests.ConnectionError("fail")): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=2 + ) + with patch( + "babel_explorer.core.downloader.requests.get", + side_effect=requests.ConnectionError("fail"), + ): with patch("babel_explorer.core.downloader.time.sleep"): # skip waiting with pytest.raises(RuntimeError, match="Failed to download"): - dl._download_with_retry("https://example.com/file", str(tmp_path / "f"), 1024) + dl._download_with_retry( + "https://example.com/file", str(tmp_path / "f"), 1024 + ) def test_succeeds_on_second_attempt(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path), retries=3) + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) out_path = str(tmp_path / "retry_success.bin") - mock_response = self._make_response(200, {'Content-Length': '5'}, [b"hello"]) + mock_response = self._make_response(200, {"Content-Length": "5"}, [b"hello"]) side_effects = [requests.ConnectionError("first fail"), mock_response] - with patch("babel_explorer.core.downloader.requests.get", side_effect=side_effects): + with patch( + "babel_explorer.core.downloader.requests.get", side_effect=side_effects + ): with patch("babel_explorer.core.downloader.time.sleep"): dl._download_with_retry("https://example.com/file", out_path, 1024) assert os.path.exists(out_path) @@ -479,11 +526,13 @@ def test_resume_sends_range_header(self, tmp_path): out_path = tmp_path / "partial.bin" out_path.write_bytes(b"partial") # 7 bytes - mock_response = self._make_response(206, {'Content-Length': '3'}, [b"end"]) - with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response) as mock_get: + mock_response = self._make_response(206, {"Content-Length": "3"}, [b"end"]) + with patch( + "babel_explorer.core.downloader.requests.get", return_value=mock_response + ) as mock_get: dl._download_with_retry("https://example.com/file", str(out_path), 1024) _, kwargs = mock_get.call_args - assert kwargs['headers'] == {'Range': 'bytes=7-'} + assert kwargs["headers"] == {"Range": "bytes=7-"} def test_http_416_file_already_complete(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) @@ -491,7 +540,9 @@ def test_http_416_file_already_complete(self, tmp_path): out_path.write_bytes(b"full file") mock_response = self._make_response(416) - with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): + with patch( + "babel_explorer.core.downloader.requests.get", return_value=mock_response + ): dl._download_with_retry("https://example.com/file", str(out_path), 1024) # Should return without error assert out_path.read_bytes() == b"full file" @@ -502,8 +553,12 @@ def test_server_no_resume_restarts_download(self, tmp_path): out_path = tmp_path / "no_resume.bin" out_path.write_bytes(b"partial") - mock_response = self._make_response(200, {'Content-Length': '12'}, [b"full content"]) - with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): + mock_response = self._make_response( + 200, {"Content-Length": "12"}, [b"full content"] + ) + with patch( + "babel_explorer.core.downloader.requests.get", return_value=mock_response + ): dl._download_with_retry("https://example.com/file", str(out_path), 1024) assert out_path.read_bytes() == b"full content" @@ -512,10 +567,16 @@ def test_returns_response_headers(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) out_path = str(tmp_path / "headers.bin") - mock_response = self._make_response(200, {'Content-Length': '5', 'ETag': '"abc"'}, [b"hello"]) - with patch("babel_explorer.core.downloader.requests.get", return_value=mock_response): - headers = dl._download_with_retry("https://example.com/file", out_path, 1024) - assert headers['ETag'] == '"abc"' + mock_response = self._make_response( + 200, {"Content-Length": "5", "ETag": '"abc"'}, [b"hello"] + ) + with patch( + "babel_explorer.core.downloader.requests.get", return_value=mock_response + ): + headers = dl._download_with_retry( + "https://example.com/file", out_path, 1024 + ) + assert headers["ETag"] == '"abc"' class TestStreamDownload: @@ -526,11 +587,11 @@ def test_writes_chunks(self, tmp_path): out_path = str(tmp_path / "stream.bin") mock_response = Mock() - mock_response.headers = {'Content-Length': '10'} + mock_response.headers = {"Content-Length": "10"} mock_response.iter_content = Mock(return_value=[b"hello", b"world"]) dl._stream_download(mock_response, out_path, resume_byte_pos=0, chunk_size=1024) - with open(out_path, 'rb') as f: + with open(out_path, "rb") as f: assert f.read() == b"helloworld" def test_append_mode_on_resume(self, tmp_path): @@ -539,10 +600,12 @@ def test_append_mode_on_resume(self, tmp_path): out_path.write_bytes(b"start") mock_response = Mock() - mock_response.headers = {'Content-Length': '3'} + mock_response.headers = {"Content-Length": "3"} mock_response.iter_content = Mock(return_value=[b"end"]) - dl._stream_download(mock_response, str(out_path), resume_byte_pos=5, chunk_size=1024) + dl._stream_download( + mock_response, str(out_path), resume_byte_pos=5, chunk_size=1024 + ) assert out_path.read_bytes() == b"startend" diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index 2322eef..363d077 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -75,7 +75,11 @@ def test_lt_ordering(self): assert a < b def test_sorting(self): - items = [Identifier(curie="C:3"), Identifier(curie="A:1"), Identifier(curie="B:2")] + items = [ + Identifier(curie="C:3"), + Identifier(curie="A:1"), + Identifier(curie="B:2"), + ] result = sorted(items) assert [x.curie for x in result] == ["A:1", "B:2", "C:3"] @@ -108,7 +112,9 @@ def test_correct_api_endpoint_and_params(self): mock_resp.json.return_value = {"X:1": {"id": {"identifier": "X:1"}}} mock_resp.raise_for_status = Mock() - with patch("babel_explorer.core.nodenorm.requests.get", return_value=mock_resp) as mock_get: + with patch( + "babel_explorer.core.nodenorm.requests.get", return_value=mock_resp + ) as mock_get: nn.normalize_curie("X:1") mock_get.assert_called_once() args, kwargs = mock_get.call_args @@ -132,7 +138,9 @@ def test_lru_caching(self): mock_resp.json.return_value = {"X:1": {"id": "X:1"}} mock_resp.raise_for_status = Mock() - with patch("babel_explorer.core.nodenorm.requests.get", return_value=mock_resp) as mock_get: + with patch( + "babel_explorer.core.nodenorm.requests.get", return_value=mock_resp + ) as mock_get: nn.normalize_curie("X:1") nn.normalize_curie("X:1") mock_get.assert_called_once() @@ -162,7 +170,7 @@ def test_exact_match_found(self): {"identifier": "X:2", "label": "Beta"}, ], } - with patch.object(nn, 'normalize_curie', return_value=api_result): + with patch.object(nn, "normalize_curie", return_value=api_result): ident = nn.get_identifier("X:1") assert ident.curie == "X:1" assert ident.label == "Alpha" @@ -174,14 +182,14 @@ def test_no_match_returns_bare_identifier(self): {"identifier": "X:2", "label": "Beta"}, ], } - with patch.object(nn, 'normalize_curie', return_value=api_result): + with patch.object(nn, "normalize_curie", return_value=api_result): ident = nn.get_identifier("X:1") assert ident.curie == "X:1" assert ident.label == "" def test_falsy_result_returns_bare_identifier(self): nn = self._make_nn() - with patch.object(nn, 'normalize_curie', return_value=None): + with patch.object(nn, "normalize_curie", return_value=None): ident = nn.get_identifier("X:1") assert ident.curie == "X:1" assert ident.label == "" @@ -193,7 +201,7 @@ def test_caching(self): {"identifier": "X:1", "label": "Alpha"}, ], } - with patch.object(nn, 'normalize_curie', return_value=api_result) as mock_norm: + with patch.object(nn, "normalize_curie", return_value=api_result) as mock_norm: nn.get_identifier("X:1") nn.get_identifier("X:1") mock_norm.assert_called_once() @@ -214,7 +222,7 @@ def test_success_returns_list(self): {"identifier": "X:2", "label": "Beta"}, ], } - with patch.object(nn, 'normalize_curie', return_value=api_result): + with patch.object(nn, "normalize_curie", return_value=api_result): result = nn.get_clique_identifiers("X:1") assert len(result) == 2 assert all(isinstance(x, Identifier) for x in result) @@ -222,7 +230,7 @@ def test_success_returns_list(self): def test_missing_key_returns_none(self): nn = self._make_nn() api_result = {"id": {"identifier": "X:1"}} # no equivalent_identifiers - with patch.object(nn, 'normalize_curie', return_value=api_result): + with patch.object(nn, "normalize_curie", return_value=api_result): result = nn.get_clique_identifiers("X:1") assert result is None @@ -231,7 +239,7 @@ def test_caching(self): api_result = { "equivalent_identifiers": [{"identifier": "X:1"}], } - with patch.object(nn, 'normalize_curie', return_value=api_result) as mock_norm: + with patch.object(nn, "normalize_curie", return_value=api_result) as mock_norm: nn.get_clique_identifiers("X:1") nn.get_clique_identifiers("X:1") mock_norm.assert_called_once() From ae8a424caa7ef330b1a4055a71fc48f4e45ec094 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 17:00:36 -0400 Subject: [PATCH 050/123] Rename lint workflow to CI and add unit test job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaced .github/workflows/lint.yml with ci.yml. The new file keeps the existing ruff lint/format job and adds a parallel test job that runs `pytest -v -m "not integration"` (unit tests only — no network required). Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/{lint.yml => ci.yml} | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) rename .github/workflows/{lint.yml => ci.yml} (54%) diff --git a/.github/workflows/lint.yml b/.github/workflows/ci.yml similarity index 54% rename from .github/workflows/lint.yml rename to .github/workflows/ci.yml index f4771d4..8ce9dd1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,10 @@ -name: Lint +name: CI on: pull_request: jobs: - ruff: + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -12,3 +12,11 @@ jobs: - run: uv sync --group dev - run: uv run ruff check src/ tests/ - run: uv run ruff format --check src/ tests/ + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv sync --group dev + - run: uv run pytest -v -m "not integration" From 5a5f00e6ada2cfdf059083ab43c9ea4088e1d5c2 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 17:25:40 -0400 Subject: [PATCH 051/123] Improved documentation. --- src/babel_explorer/cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 463b1af..76090fc 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -85,14 +85,12 @@ def xrefs( """ Fetches and prints the cross-references (xrefs) for the given CURIEs. - This function searches for xrefs associated with the provided CURIEs. - \f :param curies: A list of CURIEs (Compact URI) for which cross-references need to be retrieved. :type curies: list[str] - :param babel_url: Base URL of the Babel server + :param babel_url: Base URL of the Babel server from which to download DuckDB files. :type babel_url: str :return: None From b7a3f93bdbbae75b2bc91c277fa8e368bdf72d6a Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 17:30:45 -0400 Subject: [PATCH 052/123] Add tests for parse_duration() in cli.py Covers all 5 branches: empty input, 'never', unit suffix, bare integer, and invalid values that raise click.BadParameter. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_cli.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/test_cli.py diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..ed12a69 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,52 @@ +""" +Tests for CLI helper functions. + +Unit tests — no network required. +""" + +import pytest +import click + +from babel_explorer.cli import parse_duration + + +# ========================================================================== +# Unit Tests — no network required +# ========================================================================== + + +class TestParseDuration: + """Tests for parse_duration().""" + + @pytest.mark.parametrize( + "value, expected", + [ + ("never", float("inf")), + ("NEVER", float("inf")), + ("3h", 10800), + ("3H", 10800), + ("30m", 1800), + ("1d", 86400), + ("7200s", 7200), + ("7200", 7200), + ("0", 0), + (" 3h ", 10800), + ], + ) + def test_valid_inputs(self, value, expected): + assert parse_duration(value) == expected + + @pytest.mark.parametrize( + "value", + [ + "", + None, + "abc", + "3.5h", + "1.5", + "3x", + ], + ) + def test_invalid_inputs_raise_bad_parameter(self, value): + with pytest.raises(click.BadParameter): + parse_duration(value) From 54b6c5b3309d1b64289010a8ab814ec1dcfe6730 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 16 Mar 2026 17:39:41 -0400 Subject: [PATCH 053/123] Add CliRunner tests for xrefs, ids, and test-concord commands Co-Authored-By: Claude Sonnet 4.6 --- tests/test_cli.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index ed12a69..c8cc924 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,8 +6,10 @@ import pytest import click +from click.testing import CliRunner +from unittest.mock import patch, MagicMock -from babel_explorer.cli import parse_duration +from babel_explorer.cli import parse_duration, cli # ========================================================================== @@ -50,3 +52,95 @@ def test_valid_inputs(self, value, expected): def test_invalid_inputs_raise_bad_parameter(self, value): with pytest.raises(click.BadParameter): parse_duration(value) + + +class TestCliCommands: + """Tests for CLI commands using CliRunner — no network required.""" + + def test_xrefs_happy_path(self): + runner = CliRunner() + mock_xref = MagicMock() + mock_xref.__str__ = lambda self: "A:1 skos:exactMatch B:2" + + with patch("babel_explorer.cli.BabelDownloader"), \ + patch("babel_explorer.cli.BabelXRefs") as mock_bx, \ + patch("babel_explorer.cli.NodeNorm"): + mock_bx.return_value.get_curie_xrefs.return_value = [mock_xref] + result = runner.invoke(cli, ["xrefs", "MONDO:0004979"]) + + assert result.exit_code == 0 + mock_bx.return_value.get_curie_xrefs.assert_called_once_with( + ("MONDO:0004979",), False, label_curies=False + ) + + def test_xrefs_recurse_and_labels_flags(self): + runner = CliRunner() + mock_xref = MagicMock() + mock_xref.__str__ = lambda self: "A:1 skos:exactMatch B:2" + + with patch("babel_explorer.cli.BabelDownloader"), \ + patch("babel_explorer.cli.BabelXRefs") as mock_bx, \ + patch("babel_explorer.cli.NodeNorm"): + mock_bx.return_value.get_curie_xrefs.return_value = [mock_xref] + result = runner.invoke(cli, ["xrefs", "MONDO:0004979", "--recurse", "--labels"]) + + assert result.exit_code == 0 + mock_bx.return_value.get_curie_xrefs.assert_called_once_with( + ("MONDO:0004979",), True, label_curies=True + ) + + def test_xrefs_check_download_option(self): + runner = CliRunner() + + with patch("babel_explorer.cli.BabelDownloader") as mock_dl, \ + patch("babel_explorer.cli.BabelXRefs") as mock_bx, \ + patch("babel_explorer.cli.NodeNorm"): + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke(cli, ["xrefs", "MONDO:0004979", "--check-download", "1h"]) + + assert result.exit_code == 0 + _, kwargs = mock_dl.call_args + assert kwargs.get("freshness_seconds") == 3600 + + def test_ids_happy_path(self): + runner = CliRunner() + mock_id = MagicMock() + mock_id.__str__ = lambda self: "MONDO:0004979 record" + + with patch("babel_explorer.cli.BabelDownloader"), \ + patch("babel_explorer.cli.BabelXRefs") as mock_bx: + mock_bx.return_value.get_curie_ids.return_value = [mock_id] + result = runner.invoke(cli, ["ids", "MONDO:0004979"]) + + assert result.exit_code == 0 + mock_bx.return_value.get_curie_ids.assert_called_once_with(("MONDO:0004979",)) + + def test_test_concord_happy_path(self): + runner = CliRunner() + mock_ident = MagicMock() + mock_ident.curie = "MONDO:0004979" + mock_ident.label = "asthma" + mock_ident.biolink_type = "biolink:Disease" + + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] + result = runner.invoke(cli, ["test-concord", "MONDO:0004979"]) + + assert result.exit_code == 0 + assert "asthma" in result.output + mock_nn.return_value.get_clique_identifiers.assert_called_once_with("MONDO:0004979") + + def test_test_concord_no_label(self): + runner = CliRunner() + mock_ident = MagicMock() + mock_ident.curie = "MONDO:0004979" + mock_ident.label = None + mock_ident.biolink_type = "biolink:Disease" + + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] + result = runner.invoke(cli, ["test-concord", "MONDO:0004979"]) + + assert result.exit_code == 0 + assert "MONDO:0004979" in result.output + assert "biolink:Disease" in result.output From 1ce64286615e740744329f32e808b9e952b71ce6 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Thu, 26 Mar 2026 00:49:42 -0400 Subject: [PATCH 054/123] Reformatted code with ruff. --- tests/test_cli.py | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index c8cc924..09d415d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -62,9 +62,11 @@ def test_xrefs_happy_path(self): mock_xref = MagicMock() mock_xref.__str__ = lambda self: "A:1 skos:exactMatch B:2" - with patch("babel_explorer.cli.BabelDownloader"), \ - patch("babel_explorer.cli.BabelXRefs") as mock_bx, \ - patch("babel_explorer.cli.NodeNorm"): + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): mock_bx.return_value.get_curie_xrefs.return_value = [mock_xref] result = runner.invoke(cli, ["xrefs", "MONDO:0004979"]) @@ -78,11 +80,15 @@ def test_xrefs_recurse_and_labels_flags(self): mock_xref = MagicMock() mock_xref.__str__ = lambda self: "A:1 skos:exactMatch B:2" - with patch("babel_explorer.cli.BabelDownloader"), \ - patch("babel_explorer.cli.BabelXRefs") as mock_bx, \ - patch("babel_explorer.cli.NodeNorm"): + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): mock_bx.return_value.get_curie_xrefs.return_value = [mock_xref] - result = runner.invoke(cli, ["xrefs", "MONDO:0004979", "--recurse", "--labels"]) + result = runner.invoke( + cli, ["xrefs", "MONDO:0004979", "--recurse", "--labels"] + ) assert result.exit_code == 0 mock_bx.return_value.get_curie_xrefs.assert_called_once_with( @@ -92,11 +98,15 @@ def test_xrefs_recurse_and_labels_flags(self): def test_xrefs_check_download_option(self): runner = CliRunner() - with patch("babel_explorer.cli.BabelDownloader") as mock_dl, \ - patch("babel_explorer.cli.BabelXRefs") as mock_bx, \ - patch("babel_explorer.cli.NodeNorm"): + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): mock_bx.return_value.get_curie_xrefs.return_value = [] - result = runner.invoke(cli, ["xrefs", "MONDO:0004979", "--check-download", "1h"]) + result = runner.invoke( + cli, ["xrefs", "MONDO:0004979", "--check-download", "1h"] + ) assert result.exit_code == 0 _, kwargs = mock_dl.call_args @@ -107,8 +117,10 @@ def test_ids_happy_path(self): mock_id = MagicMock() mock_id.__str__ = lambda self: "MONDO:0004979 record" - with patch("babel_explorer.cli.BabelDownloader"), \ - patch("babel_explorer.cli.BabelXRefs") as mock_bx: + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + ): mock_bx.return_value.get_curie_ids.return_value = [mock_id] result = runner.invoke(cli, ["ids", "MONDO:0004979"]) @@ -128,7 +140,9 @@ def test_test_concord_happy_path(self): assert result.exit_code == 0 assert "asthma" in result.output - mock_nn.return_value.get_clique_identifiers.assert_called_once_with("MONDO:0004979") + mock_nn.return_value.get_clique_identifiers.assert_called_once_with( + "MONDO:0004979" + ) def test_test_concord_no_label(self): runner = CliRunner() From f54561a121ccb92aa71261e87ba76bb39e3ed790 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 30 Mar 2026 16:14:01 -0400 Subject: [PATCH 055/123] Update src/babel_explorer/cli.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 76090fc..f2a6b5d 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -61,7 +61,7 @@ def cli(): "--nodenorm-url", type=str, default="https://nodenormalization-sri.renci.org/", - help="NodeNorm URL to check for concord changes", + help="NodeNorm base URL used for node normalization and label enrichment", ) @click.option("--recurse", is_flag=True, help="Recursively query returned xrefs") @click.option("--labels", is_flag=True, help="Include labels for CURIEs") From 44813509c53e18ea335edbbfe2b0d719c2c85505 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 30 Mar 2026 16:28:10 -0400 Subject: [PATCH 056/123] Cache get_identifier() locals in _to_labeled_xref; root-anchor lib/ in .gitignore Both backported from add-nodenorm-frontend: - babel_xrefs.py: avoid calling get_identifier() twice per CURIE in _to_labeled_xref - .gitignore: anchor lib/ and lib64/ to repo root so nested lib dirs aren't ignored Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 5 +++-- src/babel_explorer/core/babel_xrefs.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 67d8b31..55c6b2a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,9 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ +# Python distribution lib directories (not web/src/lib/) +/lib/ +/lib64/ parts/ sdist/ var/ diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index c218761..53dfb28 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -122,15 +122,17 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): def _to_labeled_xref(self, xref: CrossReference) -> LabeledCrossReference: """Convert a CrossReference to a LabeledCrossReference using NodeNorm.""" + subj_ident = self.nodenorm.get_identifier(xref.subj) + obj_ident = self.nodenorm.get_identifier(xref.obj) return LabeledCrossReference( subj=xref.subj, obj=xref.obj, filename=xref.filename, pred=xref.pred, - subj_label=self.nodenorm.get_identifier(xref.subj).label, - subj_biolink_type=self.nodenorm.get_identifier(xref.subj).biolink_type, - obj_label=self.nodenorm.get_identifier(xref.obj).label, - obj_biolink_type=self.nodenorm.get_identifier(xref.obj).biolink_type, + subj_label=subj_ident.label, + subj_biolink_type=subj_ident.biolink_type, + obj_label=obj_ident.label, + obj_biolink_type=obj_ident.biolink_type, ) def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = False): From 43a004d92150204b60fc815e8563b5732984bee7 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 30 Mar 2026 16:47:31 -0400 Subject: [PATCH 057/123] Fix bugs and gaps identified in PR #1 code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - nodenorm.py: Identifier.biolink_type str→list[str] to match NodeNorm API - nodenorm.py: get_clique_identifiers returns [] instead of None; add return type annotation - nodenorm.py: log debug message when get_identifier finds no exact match - cli.py: parse_duration return type int|float; join biolink_type list for display - tests: update assertions for new biolink_type type; add test-concord edge cases (unknown CURIE producing no output, multiple CURIEs queried independently) - ci.yml: add workflow_dispatch trigger and integration-test job Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 10 ++++++++ src/babel_explorer/cli.py | 11 ++++----- src/babel_explorer/core/nodenorm.py | 11 +++++---- tests/test_cli.py | 37 +++++++++++++++++++++++++++-- tests/test_nodenorm.py | 10 ++++---- 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ce9dd1..5ee70c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI on: pull_request: + workflow_dispatch: jobs: lint: @@ -20,3 +21,12 @@ jobs: - uses: astral-sh/setup-uv@v5 - run: uv sync --group dev - run: uv run pytest -v -m "not integration" + + integration-test: + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv sync --group dev + - run: uv run pytest -v -m "integration and not slow" diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index f2a6b5d..11eab26 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -6,7 +6,7 @@ from babel_explorer.core.nodenorm import NodeNorm -def parse_duration(value: str) -> float: +def parse_duration(value: str) -> int | float: """Parse a duration string like '3h', '30m', '1d', '7200', or 'never' → seconds.""" units = {"s": 1, "m": 60, "h": 3600, "d": 86400} lower = (value or "").strip().lower() @@ -169,13 +169,12 @@ def test_concord(curies, nodenorm_url): nodenorm = NodeNorm(nodenorm_url) for curie in curies: identifiers = nodenorm.get_clique_identifiers(curie) - for identifier in identifiers or []: + for identifier in identifiers: + biolink = ", ".join(identifier.biolink_type) if identifier.label: - print( - f"{curie}\t{identifier.curie}\t{identifier.label}\t{identifier.biolink_type}" - ) + print(f"{curie}\t{identifier.curie}\t{identifier.label}\t{biolink}") else: - print(f"{curie}\t{identifier.curie}\t\t{identifier.biolink_type}") + print(f"{curie}\t{identifier.curie}\t\t{biolink}") if __name__ == "__main__": diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index ec5b0a8..f4ead30 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -8,7 +8,7 @@ class Identifier: curie: str label: str = "" - biolink_type: str = "" + biolink_type: list[str] = dataclasses.field(default_factory=list) taxa: list[str] = dataclasses.field(default_factory=list) description: list[str] = dataclasses.field(default_factory=list) @@ -47,6 +47,9 @@ def get_identifier(self, curie: str): logging.debug(f"Found exact match for {curie}: {identifier}") return Identifier.from_dict(identifier) + logging.debug( + f"No exact match for {curie!r} in equivalent_identifiers; returning bare Identifier" + ) return Identifier(curie=curie) @functools.lru_cache(maxsize=None) @@ -83,12 +86,12 @@ def normalize_curie( return None @functools.lru_cache(maxsize=None) - def get_clique_identifiers(self, curie, **kwargs): + def get_clique_identifiers(self, curie, **kwargs) -> list[Identifier]: result = self.normalize_curie(curie, **kwargs) if not result: - return None + return [] if "equivalent_identifiers" not in result: - return None + return [] return list( map(lambda x: Identifier.from_dict(x), result["equivalent_identifiers"]) ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 09d415d..ac75fe6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -132,7 +132,7 @@ def test_test_concord_happy_path(self): mock_ident = MagicMock() mock_ident.curie = "MONDO:0004979" mock_ident.label = "asthma" - mock_ident.biolink_type = "biolink:Disease" + mock_ident.biolink_type = ["biolink:Disease"] with patch("babel_explorer.cli.NodeNorm") as mock_nn: mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] @@ -149,7 +149,7 @@ def test_test_concord_no_label(self): mock_ident = MagicMock() mock_ident.curie = "MONDO:0004979" mock_ident.label = None - mock_ident.biolink_type = "biolink:Disease" + mock_ident.biolink_type = ["biolink:Disease"] with patch("babel_explorer.cli.NodeNorm") as mock_nn: mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] @@ -158,3 +158,36 @@ def test_test_concord_no_label(self): assert result.exit_code == 0 assert "MONDO:0004979" in result.output assert "biolink:Disease" in result.output + + def test_test_concord_unknown_curie_produces_no_output(self): + """When get_clique_identifiers returns [], no output is produced and exit code is 0.""" + runner = CliRunner() + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [] + result = runner.invoke(cli, ["test-concord", "UNKNOWN:9999"]) + assert result.exit_code == 0 + assert result.output.strip() == "" + + def test_test_concord_multiple_curies(self): + """Each CURIE is looked up independently.""" + runner = CliRunner() + mock_a = MagicMock() + mock_a.curie = "A:1" + mock_a.label = "Alpha" + mock_a.biolink_type = ["biolink:Disease"] + mock_b = MagicMock() + mock_b.curie = "B:2" + mock_b.label = "Beta" + mock_b.biolink_type = ["biolink:Gene"] + + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.side_effect = [ + [mock_a], + [mock_b], + ] + result = runner.invoke(cli, ["test-concord", "A:1", "B:2"]) + + assert result.exit_code == 0 + assert mock_nn.return_value.get_clique_identifiers.call_count == 2 + assert "Alpha" in result.output + assert "Beta" in result.output diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index 363d077..71459e7 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -26,7 +26,7 @@ def test_creation_with_defaults(self): ident = Identifier(curie="MONDO:0004979") assert ident.curie == "MONDO:0004979" assert ident.label == "" - assert ident.biolink_type == "" + assert ident.biolink_type == [] assert ident.taxa == [] assert ident.description == [] @@ -34,12 +34,12 @@ def test_full_creation(self): ident = Identifier( curie="MONDO:0004979", label="asthma", - biolink_type="biolink:Disease", + biolink_type=["biolink:Disease"], taxa=["NCBITaxon:9606"], description=["A chronic respiratory disease"], ) assert ident.label == "asthma" - assert ident.biolink_type == "biolink:Disease" + assert ident.biolink_type == ["biolink:Disease"] assert ident.taxa == ["NCBITaxon:9606"] def test_from_dict_minimal(self): @@ -67,7 +67,7 @@ def test_from_dict_partial(self): ident = Identifier.from_dict(d) assert ident.curie == "X:1" assert ident.label == "Beta" - assert ident.biolink_type == "" + assert ident.biolink_type == [] def test_lt_ordering(self): a = Identifier(curie="A:1") @@ -232,7 +232,7 @@ def test_missing_key_returns_none(self): api_result = {"id": {"identifier": "X:1"}} # no equivalent_identifiers with patch.object(nn, "normalize_curie", return_value=api_result): result = nn.get_clique_identifiers("X:1") - assert result is None + assert result == [] def test_caching(self): nn = self._make_nn() From 6d2eb1f5f2d9571ba8bc44b4c90e0d3f420243b2 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 30 Mar 2026 16:55:35 -0400 Subject: [PATCH 058/123] Run integration tests on push to master and weekly on Tuesdays - Add push trigger for master branch (fires when PRs are merged) - Add schedule trigger: Tuesdays at 17:00 UTC (12pm EST / 1pm EDT) - Change integration-test job condition to run on all non-PR events Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee70c1..7f712ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,10 @@ name: CI on: pull_request: + push: + branches: [master] + schedule: + - cron: "0 17 * * 2" # Tuesdays at 12pm EST (17:00 UTC); 1pm during EDT workflow_dispatch: jobs: @@ -24,7 +28,7 @@ jobs: integration-test: runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' + if: github.event_name != 'pull_request' steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 From e3d3434df9de080ca69691a1bf6ee2b6b7228e0d Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 30 Mar 2026 18:09:17 -0400 Subject: [PATCH 059/123] Add module, class, and method docstrings to new files in PR #1 Source files: - nodenorm.py: module, Identifier class/from_dict, NodeNorm class/__init__/ get_identifier/normalize_curie/get_clique_identifiers - babel_xrefs.py: convert # comment to module docstring; CrossReference class/ from_tuple/curies property; LabeledCrossReference class; IdentifierRecord.__str__; BabelXRefs class/__init__/get_curie_xref - downloader.py: module, BabelDownloader.__init__, get_output_file - cli.py: cli() group, test_concord() command Test files (class docstrings only): - test_babel_xrefs.py: TestCrossReference, TestLabeledCrossReference, TestIdentifierRecord, TestBabelXRefsInit - test_nodenorm.py: TestIdentifier, TestNodeNormInit, TestNormalizeCurieMocked, TestGetIdentifierMocked, TestGetCliqueIdentifiersMocked Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/cli.py | 6 ++++ src/babel_explorer/core/babel_xrefs.py | 40 ++++++++++++++++++++++++-- src/babel_explorer/core/downloader.py | 14 +++++++++ src/babel_explorer/core/nodenorm.py | 33 ++++++++++++++++++++- tests/test_babel_xrefs.py | 8 ++++++ tests/test_nodenorm.py | 10 +++++++ 6 files changed, 107 insertions(+), 4 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 11eab26..5c4c703 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -40,6 +40,7 @@ def parse_duration(value: str) -> int | float: @click.group() def cli(): + """babel-explorer: query and explore Babel intermediate files.""" pass @@ -163,6 +164,11 @@ def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): help="NodeNorm URL to check for concord changes", ) def test_concord(curies, nodenorm_url): + """For each CURIE, print the current NodeNorm clique (all equivalent identifiers, labels, and Biolink types). + + Useful for inspecting how a potential Babel concordance change would affect NodeNorm: + run before and after a Babel rebuild to see how cliques would shift. + """ # We're trying to answer a simple question here: if the CURIEs we mention were combined, how would the cliques change in NodeNorm? # By definition, this can only combine all the cliques mentioned in the CURIEs. diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 53dfb28..e89acb1 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -1,6 +1,10 @@ -# Babel XRefs is a tool for accessing and querying the intermediate files -# that we make available with Babel builds. This allows you to find out -# why we consider two identifiers to be identical. +"""Query engine for Babel cross-reference intermediate files. + +Provides access to Concord.parquet and Identifiers.parquet via DuckDB, +allowing callers to discover why two biological/chemical identifiers are +considered identical in a Babel build. +""" + import dataclasses import logging import duckdb @@ -12,6 +16,8 @@ @dataclasses.dataclass(frozen=True) class CrossReference: + """A single cross-reference edge read from Concord.parquet.""" + filename: str subj: str pred: str @@ -19,12 +25,14 @@ class CrossReference: @staticmethod def from_tuple(tuple: tuple[str, str, str, str]): + """Construct from a ``(filename, subj, pred, obj)`` database row tuple.""" return CrossReference( filename=tuple[0], subj=tuple[1], pred=tuple[2], obj=tuple[3] ) @property def curies(self): + """The frozenset of both CURIEs in this edge (subject and object).""" return frozenset([self.subj, self.obj]) def __lt__(self, other): @@ -38,6 +46,8 @@ def __lt__(self, other): @dataclasses.dataclass(frozen=True) class LabeledCrossReference(CrossReference): + """A CrossReference enriched with human-readable labels and Biolink types from NodeNorm.""" + subj_label: str subj_biolink_type: str obj_label: str @@ -64,6 +74,7 @@ def from_row(row: tuple, column_names: list[str]): return IdentifierRecord(curie=row[curie_idx], extra_fields=extra) def __str__(self): + """Return a ``key=value`` string of the CURIE and all extra fields.""" parts = [f"curie={self.curie!r}"] for name, value in self.extra_fields: parts.append(f"{name}={value!r}") @@ -71,7 +82,20 @@ def __str__(self): class BabelXRefs: + """Query engine for Babel cross-reference and identifier Parquet files. + + Uses DuckDB for in-memory SQL queries against Concord.parquet and + Identifiers.parquet. NodeNorm is optional and only required when + ``label_curies=True`` is passed to enrichment-aware methods. + """ + def __init__(self, downloader: BabelDownloader, nodenorm: NodeNorm = None): + """ + :param downloader: A configured ``BabelDownloader`` that provides local paths + to the required Parquet files, downloading them on first access. + :param nodenorm: Optional ``NodeNorm`` client. Required only when callers pass + ``label_curies=True``; may be ``None`` for label-free queries. + """ self.downloader = downloader self.nodenorm = nodenorm @@ -101,6 +125,16 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: @functools.lru_cache(maxsize=None) def get_curie_xref(self, curie: str, label_curies: bool = False): + """Return all cross-references in Concord.parquet where *curie* is the subject or object. + + Results are LRU-cached per ``(curie, label_curies)`` pair. + + :param curie: The CURIE to look up. + :param label_curies: If ``True``, annotate each result with NodeNorm labels and + Biolink types. Requires a NodeNorm instance to have been passed to ``__init__``. + :raises ValueError: If ``label_curies=True`` but no NodeNorm instance is available. + :return: A list of ``CrossReference`` (or ``LabeledCrossReference``) objects. + """ if label_curies and self.nodenorm is None: raise ValueError( "label_curies=True requires a configured NodeNorm instance (nodenorm was None)." diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 6ba9a38..f314c3d 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -1,3 +1,5 @@ +"""HTTP downloader for Babel Parquet files with ETag-based freshness checking.""" + import functools import json import os @@ -23,6 +25,17 @@ def __init__( freshness_seconds=3 * 3600, timeout: int = 30, ): + """ + :param url_base: Base URL of the Babel server (must end with ``/``). + :param local_path: Directory for cached downloads. Defaults to + ``tempfile.gettempdir()`` if ``None``; created automatically if it + does not exist. + :param retries: Maximum number of download retry attempts on failure. + :param freshness_seconds: How long a local file is considered fresh without + re-checking the server. Use ``float('inf')`` to never re-check, or ``0`` + to always issue a HEAD request. Defaults to 3 hours. + :param timeout: HTTP request timeout in seconds. + """ # We assume the URL base is correct (if not, we can fix it later). self.url_base = url_base self.retries = retries @@ -46,6 +59,7 @@ def __init__( @functools.lru_cache(maxsize=None) def get_output_file(self, filename): + """Return (and create) the local filesystem path for a given relative filename.""" filepath = os.path.join(self.local_path, filename) os.makedirs(os.path.dirname(filepath), exist_ok=True) return filepath diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index f4ead30..fae0a57 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -1,3 +1,5 @@ +"""NodeNorm API client for identifier normalisation and label enrichment.""" + import dataclasses import functools import requests @@ -6,6 +8,8 @@ @dataclasses.dataclass class Identifier: + """Normalised identifier record returned by the NodeNorm API.""" + curie: str label: str = "" biolink_type: list[str] = dataclasses.field(default_factory=list) @@ -17,6 +21,7 @@ def __lt__(self, other): @staticmethod def from_dict(d: dict): + """Parse an identifier entry from a NodeNorm API response dict.""" identifier = Identifier(curie=d["identifier"]) if "label" in d: identifier.label = d["label"] @@ -30,14 +35,29 @@ def from_dict(d: dict): class NodeNorm: + """Client for the NodeNormalization API (https://nodenormalization-sri.renci.org/).""" + def __init__(self, nodenorm_url: str = "", timeout: int = 30): + """ + :param nodenorm_url: Base URL of the NodeNorm service. Pass an empty string (default) + to skip all network calls and have every lookup return a bare ``Identifier``. + :param timeout: HTTP request timeout in seconds. + """ self.nodenorm_url = nodenorm_url self.timeout = timeout if self.nodenorm_url and not self.nodenorm_url.endswith("/"): self.nodenorm_url += "/" @functools.lru_cache(maxsize=None) - def get_identifier(self, curie: str): + def get_identifier(self, curie: str) -> "Identifier": + """Return the ``Identifier`` for *curie* by looking it up in its NodeNorm clique. + + Searches ``equivalent_identifiers`` for an entry whose ``identifier`` field matches + *curie* exactly. Falls back to a bare ``Identifier(curie=curie)`` (empty label and + type) if NodeNorm does not recognise the CURIE or it is not listed in the clique. + + Results are LRU-cached so repeated calls for the same CURIE are free. + """ result = self.normalize_curie(curie) logging.debug(f"Normalizing {curie} with NodeNorm to result: {result}") if not result: @@ -62,6 +82,12 @@ def normalize_curie( individual_types=True, include_taxa=True, ): + """Call ``get_normalized_nodes`` and return the per-CURIE result dict. + + :return: The normalisation dict for *curie* (contains ``id``, ``equivalent_identifiers``, + ``type``, etc.), or ``None`` if the CURIE is not recognised by NodeNorm. + :raises requests.HTTPError: If the API returns a non-2xx status code. + """ response = requests.get( f"{self.nodenorm_url}get_normalized_nodes", params={ @@ -87,6 +113,11 @@ def normalize_curie( @functools.lru_cache(maxsize=None) def get_clique_identifiers(self, curie, **kwargs) -> list[Identifier]: + """Return all ``Identifier`` objects in the NodeNorm clique for *curie*. + + :return: A list of ``Identifier`` objects (one per entry in ``equivalent_identifiers``), + or an empty list if the CURIE is unknown or has no equivalents. + """ result = self.normalize_curie(curie, **kwargs) if not result: return [] diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index d67f81b..7c48935 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -27,6 +27,8 @@ class TestCrossReference: + """Tests for the CrossReference frozen dataclass.""" + def test_creation(self): xr = CrossReference( filename="f.txt", subj="A:1", pred="skos:exactMatch", obj="B:2" @@ -85,6 +87,8 @@ def test_sorting(self): class TestLabeledCrossReference: + """Tests for the LabeledCrossReference frozen dataclass.""" + def test_creation(self): lxr = LabeledCrossReference( subj="A:1", @@ -149,6 +153,8 @@ def test_str(self): class TestIdentifierRecord: + """Tests for the IdentifierRecord frozen dataclass.""" + def test_creation(self): rec = IdentifierRecord(curie="MONDO:0004979") assert rec.curie == "MONDO:0004979" @@ -181,6 +187,8 @@ def test_str(self): class TestBabelXRefsInit: + """Tests for BabelXRefs constructor.""" + def test_init_without_nodenorm(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) bx = BabelXRefs(dl) diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index 71459e7..5fb088d 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -22,6 +22,8 @@ class TestIdentifier: + """Tests for the Identifier dataclass.""" + def test_creation_with_defaults(self): ident = Identifier(curie="MONDO:0004979") assert ident.curie == "MONDO:0004979" @@ -90,6 +92,8 @@ def test_sorting(self): class TestNodeNormInit: + """Tests for NodeNorm constructor and URL normalisation.""" + def test_default_url(self): nn = NodeNorm() assert nn.nodenorm_url == "" @@ -100,6 +104,8 @@ def test_custom_url(self): class TestNormalizeCurieMocked: + """Unit tests for NodeNorm.normalize_curie() with mocked HTTP responses.""" + def _make_nn(self): nn = NodeNorm(nodenorm_url="https://example.com/") nn.normalize_curie.cache_clear() @@ -156,6 +162,8 @@ def test_http_error_raises(self): class TestGetIdentifierMocked: + """Unit tests for NodeNorm.get_identifier() with mocked normalize_curie.""" + def _make_nn(self): nn = NodeNorm(nodenorm_url="https://example.com/") nn.normalize_curie.cache_clear() @@ -208,6 +216,8 @@ def test_caching(self): class TestGetCliqueIdentifiersMocked: + """Unit tests for NodeNorm.get_clique_identifiers() with mocked normalize_curie.""" + def _make_nn(self): nn = NodeNorm(nodenorm_url="https://example.com/") nn.normalize_curie.cache_clear() From 807bfb2d91a31065d3e7d89609749e5fe0cf806d Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 1 Apr 2026 01:40:43 -0400 Subject: [PATCH 060/123] Fix LabeledCrossReference biolink_type fields to list[str]; simplify map to listcomp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - babel_xrefs.py: subj_biolink_type/obj_biolink_type str→list[str] to match Identifier.biolink_type after the nodenorm.py type change - nodenorm.py: replace map(lambda) with list comprehension in get_clique_identifiers - tests: update LabeledCrossReference construction to use list values Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/babel_xrefs.py | 4 ++-- src/babel_explorer/core/nodenorm.py | 4 +--- tests/test_babel_xrefs.py | 18 +++++++++--------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 53dfb28..2df9ac6 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -39,9 +39,9 @@ def __lt__(self, other): @dataclasses.dataclass(frozen=True) class LabeledCrossReference(CrossReference): subj_label: str - subj_biolink_type: str + subj_biolink_type: list[str] obj_label: str - obj_biolink_type: str + obj_biolink_type: list[str] def __str__(self): return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", subj_biolink_type="{self.subj_biolink_type}", obj_label="{self.obj_label}", obj_biolink_type="{self.obj_biolink_type}")""" diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index f4ead30..f83a5d4 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -92,6 +92,4 @@ def get_clique_identifiers(self, curie, **kwargs) -> list[Identifier]: return [] if "equivalent_identifiers" not in result: return [] - return list( - map(lambda x: Identifier.from_dict(x), result["equivalent_identifiers"]) - ) + return [Identifier.from_dict(x) for x in result["equivalent_identifiers"]] diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index d67f81b..409c6fe 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -92,13 +92,13 @@ def test_creation(self): obj="B:2", filename="f", subj_label="Alpha", - subj_biolink_type="biolink:Disease", + subj_biolink_type=["biolink:Disease"], obj_label="Beta", - obj_biolink_type="biolink:Gene", + obj_biolink_type=["biolink:Gene"], ) assert lxr.subj == "A:1" assert lxr.subj_label == "Alpha" - assert lxr.obj_biolink_type == "biolink:Gene" + assert lxr.obj_biolink_type == ["biolink:Gene"] def test_inherits_from_cross_reference(self): lxr = LabeledCrossReference( @@ -107,9 +107,9 @@ def test_inherits_from_cross_reference(self): obj="B:2", filename="f", subj_label="", - subj_biolink_type="", + subj_biolink_type=[], obj_label="", - obj_biolink_type="", + obj_biolink_type=[], ) assert isinstance(lxr, CrossReference) @@ -120,9 +120,9 @@ def test_curies_property(self): obj="B:2", filename="f", subj_label="", - subj_biolink_type="", + subj_biolink_type=[], obj_label="", - obj_biolink_type="", + obj_biolink_type=[], ) assert lxr.curies == frozenset({"A:1", "B:2"}) @@ -133,9 +133,9 @@ def test_str(self): obj="B:2", filename="f", subj_label="Alpha", - subj_biolink_type="biolink:Disease", + subj_biolink_type=["biolink:Disease"], obj_label="Beta", - obj_biolink_type="biolink:Gene", + obj_biolink_type=["biolink:Gene"], ) s = str(lxr) assert "A:1" in s From 06c3833d06521e898178dcf7c39edac7ed7b0257 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 1 Apr 2026 02:52:54 -0400 Subject: [PATCH 061/123] Address PR #1 review: frozen Identifier, atomic rename, fail-open HEAD, type fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - nodenorm.py: Identifier is now frozen=True; rewrite from_dict as one-shot constructor to avoid post-construction mutation of lru_cache'd objects - nodenorm.py: remove **kwargs from get_clique_identifiers — unhashable and unused, would raise TypeError if any kwarg was ever passed - downloader.py: download to .tmp then os.replace() so the final file is never partially written; clean up .tmp on failure - downloader.py: _etag_matches returns True (fail open) on HEAD network error instead of False, avoiding spurious 2GB re-downloads on transient failures - cli.py: add nodenorm_url: str annotation in xrefs and test_concord; move test_concord inline comment to docstring - tests: update test_returns_false_on_request_error → test_returns_true_on_request_error - FUTURE.md: track CLI option deduplication refactor Co-Authored-By: Claude Sonnet 4.6 --- FUTURE.md | 7 +++++++ src/babel_explorer/cli.py | 10 ++++++---- src/babel_explorer/core/downloader.py | 25 +++++++++++++++++-------- src/babel_explorer/core/nodenorm.py | 25 +++++++++++-------------- tests/test_downloader.py | 5 +++-- 5 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 FUTURE.md diff --git a/FUTURE.md b/FUTURE.md new file mode 100644 index 0000000..a2bb3fb --- /dev/null +++ b/FUTURE.md @@ -0,0 +1,7 @@ +# Future Work + +## Deduplicate CLI option blocks + +`--local-dir`, `--babel-url`, and `--check-download` are copy-pasted between the +`xrefs` and `ids` commands in `cli.py`. Extract a `@common_babel_options` Click +decorator so defaults are defined in one place and can't drift. diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 11eab26..2361693 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -76,7 +76,7 @@ def cli(): def xrefs( curies: list[str], babel_url: str, - nodenorm_url, + nodenorm_url: str, local_dir: str, recurse: bool, labels: bool, @@ -162,10 +162,12 @@ def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): default="https://nodenormalization-sri.renci.org/", help="NodeNorm URL to check for concord changes", ) -def test_concord(curies, nodenorm_url): - # We're trying to answer a simple question here: if the CURIEs we mention were combined, how would the cliques change in NodeNorm? - # By definition, this can only combine all the cliques mentioned in the CURIEs. +def test_concord(curies: tuple[str, ...], nodenorm_url: str): + """ + For each input CURIE, show what clique NodeNorm currently maps it to. + Answers: if these CURIEs were merged in Babel, which NodeNorm cliques would combine? + """ nodenorm = NodeNorm(nodenorm_url) for curie in curies: identifiers = nodenorm.get_clique_identifiers(curie) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 6ba9a38..63e6826 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -130,8 +130,10 @@ def _etag_matches(self, url, meta): response = requests.head(url, timeout=self.timeout) response.raise_for_status() except requests.RequestException as e: - self.logger.warning(f"HEAD request failed for {url}: {e}") - return False + self.logger.warning( + f"HEAD request failed for {url}: {e}; assuming file is current" + ) + return True remote_headers = response.headers @@ -321,20 +323,27 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): ) return local_path_to_download_to - # Tier 3: ETag changed — delete and re-download + # Tier 3: ETag changed — re-download self.logger.warning( f"Remote file changed, re-downloading: {local_path_to_download_to}" ) - os.remove(local_path_to_download_to) self.logger.info( f"Downloading {url_to_download} to {local_path_to_download_to}" ) - # Download with retry logic; get response headers back - response_headers = self._download_with_retry( - url_to_download, local_path_to_download_to, chunk_size - ) + # Download to a sibling .tmp file, then atomically replace the final destination. + # This ensures the final file is never partially written. + tmp_path = local_path_to_download_to + ".tmp" + try: + response_headers = self._download_with_retry( + url_to_download, tmp_path, chunk_size + ) + os.replace(tmp_path, local_path_to_download_to) + except Exception: + if os.path.exists(tmp_path): + os.remove(tmp_path) + raise # Save sidecar metadata if response_headers is not None: diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index f83a5d4..42845fb 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -4,7 +4,7 @@ import logging -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Identifier: curie: str label: str = "" @@ -16,17 +16,14 @@ def __lt__(self, other): return self.curie < other.curie @staticmethod - def from_dict(d: dict): - identifier = Identifier(curie=d["identifier"]) - if "label" in d: - identifier.label = d["label"] - if "taxa" in d: - identifier.taxa = d["taxa"] - if "description" in d: - identifier.description = d["description"] - if "type" in d: - identifier.biolink_type = d["type"] - return identifier + def from_dict(d: dict) -> "Identifier": + return Identifier( + curie=d["identifier"], + label=d.get("label", ""), + biolink_type=d.get("type", []), + taxa=d.get("taxa", []), + description=d.get("description", []), + ) class NodeNorm: @@ -86,8 +83,8 @@ def normalize_curie( return None @functools.lru_cache(maxsize=None) - def get_clique_identifiers(self, curie, **kwargs) -> list[Identifier]: - result = self.normalize_curie(curie, **kwargs) + def get_clique_identifiers(self, curie: str) -> list[Identifier]: + result = self.normalize_curie(curie) if not result: return [] if "equivalent_identifiers" not in result: diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 9b33e7a..d23b323 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -267,14 +267,15 @@ def test_fallback_last_modified_match(self, tmp_path): ): assert dl._etag_matches("https://example.com/f.parquet", meta) is True - def test_returns_false_on_request_error(self, tmp_path): + def test_returns_true_on_request_error(self, tmp_path): + """Network errors are treated as 'assume still fresh' to avoid triggering large re-downloads.""" dl = self._make_dl(tmp_path) meta = {"etag": '"abc"'} with patch( "babel_explorer.core.downloader.requests.head", side_effect=requests.ConnectionError("fail"), ): - assert dl._etag_matches("https://example.com/f.parquet", meta) is False + assert dl._etag_matches("https://example.com/f.parquet", meta) is True class TestGetDownloadedFileTiers: From 14fe2ebe2847da1320936207bb2fbee1358e7d3d Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 1 Apr 2026 02:58:46 -0400 Subject: [PATCH 062/123] Sync CLAUDE.md with current code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix --expand → --recurse (the actual flag name) in Data Flow and Key Design Patterns - BabelXRefs: remove false claim about writing DuckDB databases to disk; all connections are in-memory (duckdb.connect() with no path) - Remove 'Generated DuckDB databases' entry from File Locations (nothing on disk) - Update test count table: numbers were stale and test_cli.py was missing entirely - Add Identifier to Key Dataclasses (now frozen=True as of recent fix) Co-Authored-By: Claude Sonnet 4.6 --- CLAUDE.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8f59cfb..cc0a558 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,9 +79,9 @@ uv run ruff format 2. **BabelXRefs** (`src/babel_explorer/core/babel_xrefs.py`): - Main query engine for cross-references - - Uses DuckDB to query Parquet files (`Concord.parquet`, `Identifiers.parquet`, `Metadata.parquet`) - - Supports recursive expansion of cross-references - - Creates ephemeral DuckDB databases in `data//output/duckdbs/` + - Uses DuckDB to query Parquet files (`Concord.parquet`, `Identifiers.parquet`) + - Supports recursive expansion of cross-references via a single `WITH RECURSIVE` query + - Uses ephemeral in-memory DuckDB connections (nothing written to disk) 3. **NodeNorm** (`src/babel_explorer/core/nodenorm.py`): - Integration with NodeNormalization API (https://nodenormalization-sri.renci.org/) @@ -98,14 +98,14 @@ uv run ruff format 1. User provides CURIEs via CLI 2. BabelDownloader ensures required Parquet files are downloaded 3. BabelXRefs queries files using DuckDB -4. If `--labels` or `--expand` flags are set, NodeNorm is queried for additional metadata +4. If `--labels` or `--recurse` flags are set, NodeNorm is queried for additional metadata 5. Results are printed to stdout ### Key Design Patterns - **Lazy downloading**: Files are only downloaded when first accessed - **LRU caching**: Heavy use of `@functools.lru_cache` to avoid redundant downloads and API calls -- **Recursive expansion**: The `--expand` flag recursively follows all cross-references to build complete graphs +- **Recursive expansion**: The `--recurse` flag recursively follows all cross-references to build complete graphs - **DuckDB for querying**: In-memory SQL queries against Parquet files for fast lookups ## Testing @@ -119,9 +119,10 @@ Tests live in `tests/` and are split into fast **unit tests** (mocked, no networ | File | Unit | Integration | Slow | Total | |------|------|-------------|------|-------| -| `tests/test_downloader.py` | 22 | 3 | 1 | 26 | -| `tests/test_babel_xrefs.py` | 22 | 8 | 1 | 31 | -| `tests/test_nodenorm.py` | 18 | 5 | 0 | 23 | +| `tests/test_downloader.py` | 41 | 4 | 1 | 46 | +| `tests/test_babel_xrefs.py` | 23 | 20 | 3 | 46 | +| `tests/test_nodenorm.py` | 20 | 13 | 0 | 33 | +| `tests/test_cli.py` | 24 | 0 | 0 | 24 | ### Test Infrastructure @@ -131,6 +132,7 @@ Tests live in `tests/` and are split into fast **unit tests** (mocked, no networ ### Key Dataclasses +- **`Identifier`** — Frozen dataclass for a normalized NodeNorm entry (curie, label, biolink_type, taxa, description). Returned by `NodeNorm.get_identifier()` and `get_clique_identifiers()`. - **`CrossReference`** — Frozen dataclass for Concord.parquet rows (filename, subj, pred, obj) - **`LabeledCrossReference`** — Extends CrossReference with labels and biolink types from NodeNorm - **`IdentifierRecord`** — Frozen dataclass for Identifiers.parquet rows (curie + dynamic extra fields). Returned by `BabelXRefs.get_curie_ids()`. @@ -146,5 +148,4 @@ Tests live in `tests/` and are split into fast **unit tests** (mocked, no networ - Tests: `tests/` - Test CURIEs: `tests/data/valid_curies.txt` - Downloaded Babel files: `data//duckdb/*.parquet` -- Generated DuckDB databases: `data//output/duckdbs/` - Entry point: `src/babel_explorer/cli.py` From 7d8faa887cba92870ecec748dc5706cf5179362e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Wed, 1 Apr 2026 18:49:09 -0400 Subject: [PATCH 063/123] Address PR #1 review: fix six correctness and quality issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - list→tuple on Identifier and LabeledCrossReference fields so frozen dataclasses are hashable (was a TypeError crash in get_curie_xrefs) - NodeNorm(''): add early return in normalize_curie so empty URL truly skips all network calls as documented - BabelDownloader: auto-append trailing slash to url_base so urljoin can't silently drop path segments - CI: fix push trigger branch master → main - Remove dead get_downloaded_dir method (lru_cache + NotImplementedError) - parse_duration: reject negative values with a clear BadParameter error Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 2 +- src/babel_explorer/cli.py | 11 ++++++++- src/babel_explorer/core/babel_xrefs.py | 4 ++-- src/babel_explorer/core/downloader.py | 19 ++-------------- src/babel_explorer/core/nodenorm.py | 14 +++++++----- tests/test_babel_xrefs.py | 18 +++++++-------- tests/test_cli.py | 2 ++ tests/test_downloader.py | 23 ++++++++++--------- tests/test_nodenorm.py | 31 +++++++++++++++++--------- 9 files changed, 67 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f712ae..c3ef4c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: pull_request: push: - branches: [master] + branches: [main] schedule: - cron: "0 17 * * 2" # Tuesdays at 12pm EST (17:00 UTC); 1pm during EDT workflow_dispatch: diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 58c7f8a..bc32418 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -27,15 +27,24 @@ def parse_duration(value: str) -> int | float: f"Invalid duration {value!r}: expected an integer followed by an optional unit " "('s', 'm', 'h', or 'd'), or 'never'." ) + if amount < 0: + raise click.BadParameter( + f"Invalid duration {value!r}: duration must be non-negative." + ) return amount * units[lower[-1]] # Bare integer seconds try: - return int(lower) + result = int(lower) except ValueError: raise click.BadParameter( f"Invalid duration {value!r}: expected an integer number of seconds, optionally " "followed by 's', 'm', 'h', or 'd', or 'never'." ) + if result < 0: + raise click.BadParameter( + f"Invalid duration {value!r}: duration must be non-negative." + ) + return result @click.group() diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index a8b94b3..0c94074 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -49,9 +49,9 @@ class LabeledCrossReference(CrossReference): """A CrossReference enriched with human-readable labels and Biolink types from NodeNorm.""" subj_label: str - subj_biolink_type: list[str] + subj_biolink_type: tuple[str, ...] obj_label: str - obj_biolink_type: list[str] + obj_biolink_type: tuple[str, ...] def __str__(self): return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", subj_biolink_type="{self.subj_biolink_type}", obj_label="{self.obj_label}", obj_biolink_type="{self.obj_biolink_type}")""" diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index c5dcada..4d2f9a2 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -36,7 +36,8 @@ def __init__( to always issue a HEAD request. Defaults to 3 hours. :param timeout: HTTP request timeout in seconds. """ - # We assume the URL base is correct (if not, we can fix it later). + if not url_base.endswith("/"): + url_base += "/" self.url_base = url_base self.retries = retries self.freshness_seconds = freshness_seconds @@ -368,19 +369,3 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes" ) return local_path_to_download_to - - @functools.lru_cache(maxsize=None) - def get_downloaded_dir(self, dirpath: str): - """ - Download a directory recursively. - - NOTE: This method is not implemented in the Python-based downloader. - Use get_downloaded_file() for individual files instead. - - Raises: - NotImplementedError: This method is not implemented - """ - raise NotImplementedError( - "Recursive directory downloads are not supported. " - "Use get_downloaded_file() for individual files." - ) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index 11a7da9..9ce916d 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -12,9 +12,9 @@ class Identifier: curie: str label: str = "" - biolink_type: list[str] = dataclasses.field(default_factory=list) - taxa: list[str] = dataclasses.field(default_factory=list) - description: list[str] = dataclasses.field(default_factory=list) + biolink_type: tuple[str, ...] = () + taxa: tuple[str, ...] = () + description: tuple[str, ...] = () def __lt__(self, other): return self.curie < other.curie @@ -24,9 +24,9 @@ def from_dict(d: dict) -> "Identifier": return Identifier( curie=d["identifier"], label=d.get("label", ""), - biolink_type=d.get("type", []), - taxa=d.get("taxa", []), - description=d.get("description", []), + biolink_type=tuple(d.get("type", [])), + taxa=tuple(d.get("taxa", [])), + description=tuple(d.get("description", [])), ) @@ -84,6 +84,8 @@ def normalize_curie( ``type``, etc.), or ``None`` if the CURIE is not recognised by NodeNorm. :raises requests.HTTPError: If the API returns a non-2xx status code. """ + if not self.nodenorm_url: + return None response = requests.get( f"{self.nodenorm_url}get_normalized_nodes", params={ diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 9fdb5da..114d269 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -96,13 +96,13 @@ def test_creation(self): obj="B:2", filename="f", subj_label="Alpha", - subj_biolink_type=["biolink:Disease"], + subj_biolink_type=("biolink:Disease",), obj_label="Beta", - obj_biolink_type=["biolink:Gene"], + obj_biolink_type=("biolink:Gene",), ) assert lxr.subj == "A:1" assert lxr.subj_label == "Alpha" - assert lxr.obj_biolink_type == ["biolink:Gene"] + assert lxr.obj_biolink_type == ("biolink:Gene",) def test_inherits_from_cross_reference(self): lxr = LabeledCrossReference( @@ -111,9 +111,9 @@ def test_inherits_from_cross_reference(self): obj="B:2", filename="f", subj_label="", - subj_biolink_type=[], + subj_biolink_type=(), obj_label="", - obj_biolink_type=[], + obj_biolink_type=(), ) assert isinstance(lxr, CrossReference) @@ -124,9 +124,9 @@ def test_curies_property(self): obj="B:2", filename="f", subj_label="", - subj_biolink_type=[], + subj_biolink_type=(), obj_label="", - obj_biolink_type=[], + obj_biolink_type=(), ) assert lxr.curies == frozenset({"A:1", "B:2"}) @@ -137,9 +137,9 @@ def test_str(self): obj="B:2", filename="f", subj_label="Alpha", - subj_biolink_type=["biolink:Disease"], + subj_biolink_type=("biolink:Disease",), obj_label="Beta", - obj_biolink_type=["biolink:Gene"], + obj_biolink_type=("biolink:Gene",), ) s = str(lxr) assert "A:1" in s diff --git a/tests/test_cli.py b/tests/test_cli.py index ac75fe6..3d73e55 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -47,6 +47,8 @@ def test_valid_inputs(self, value, expected): "3.5h", "1.5", "3x", + "-5", + "-5h", ], ) def test_invalid_inputs_raise_bad_parameter(self, value): diff --git a/tests/test_downloader.py b/tests/test_downloader.py index d23b323..7fe8609 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -60,6 +60,19 @@ def test_custom_freshness_seconds(self, tmp_path): ) assert dl.freshness_seconds == 0 + def test_url_base_trailing_slash_added(self, tmp_path): + """url_base without trailing slash gets one appended automatically.""" + dl = BabelDownloader( + url_base="https://example.com/path", local_path=str(tmp_path) + ) + assert dl.url_base == "https://example.com/path/" + + def test_url_base_with_trailing_slash_unchanged(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/path/", local_path=str(tmp_path) + ) + assert dl.url_base == "https://example.com/path/" + def test_invalid_path_raises_value_error(self): """Using a file path (not a directory) should raise ValueError.""" with tempfile.NamedTemporaryFile(delete=False) as f: @@ -610,16 +623,6 @@ def test_append_mode_on_resume(self, tmp_path): assert out_path.read_bytes() == b"startend" -class TestGetDownloadedDir: - """Tests for get_downloaded_dir.""" - - def test_raises_not_implemented(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - dl.get_downloaded_dir.cache_clear() - with pytest.raises(NotImplementedError): - dl.get_downloaded_dir("some/dir") - - # ========================================================================== # Integration Tests — require network access # ========================================================================== diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index 5fb088d..8b30fcd 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -28,21 +28,21 @@ def test_creation_with_defaults(self): ident = Identifier(curie="MONDO:0004979") assert ident.curie == "MONDO:0004979" assert ident.label == "" - assert ident.biolink_type == [] - assert ident.taxa == [] - assert ident.description == [] + assert ident.biolink_type == () + assert ident.taxa == () + assert ident.description == () def test_full_creation(self): ident = Identifier( curie="MONDO:0004979", label="asthma", - biolink_type=["biolink:Disease"], - taxa=["NCBITaxon:9606"], - description=["A chronic respiratory disease"], + biolink_type=("biolink:Disease",), + taxa=("NCBITaxon:9606",), + description=("A chronic respiratory disease",), ) assert ident.label == "asthma" - assert ident.biolink_type == ["biolink:Disease"] - assert ident.taxa == ["NCBITaxon:9606"] + assert ident.biolink_type == ("biolink:Disease",) + assert ident.taxa == ("NCBITaxon:9606",) def test_from_dict_minimal(self): d = {"identifier": "X:1"} @@ -61,15 +61,15 @@ def test_from_dict_full(self): ident = Identifier.from_dict(d) assert ident.curie == "X:1" assert ident.label == "Alpha" - assert ident.biolink_type == ["biolink:NamedThing"] - assert ident.taxa == ["NCBITaxon:9606"] + assert ident.biolink_type == ("biolink:NamedThing",) + assert ident.taxa == ("NCBITaxon:9606",) def test_from_dict_partial(self): d = {"identifier": "X:1", "label": "Beta"} ident = Identifier.from_dict(d) assert ident.curie == "X:1" assert ident.label == "Beta" - assert ident.biolink_type == [] + assert ident.biolink_type == () def test_lt_ordering(self): a = Identifier(curie="A:1") @@ -102,6 +102,15 @@ def test_custom_url(self): nn = NodeNorm(nodenorm_url="https://custom.api/") assert nn.nodenorm_url == "https://custom.api/" + def test_empty_url_normalize_curie_returns_none_without_network(self): + """NodeNorm('') must not make any HTTP calls and must return None.""" + nn = NodeNorm("") + nn.normalize_curie.cache_clear() + with patch("babel_explorer.core.nodenorm.requests.get") as mock_get: + result = nn.normalize_curie("MONDO:0004979") + mock_get.assert_not_called() + assert result is None + class TestNormalizeCurieMocked: """Unit tests for NodeNorm.normalize_curie() with mocked HTTP responses.""" From 0607e00ddd12bf0220617155ba52f478f0168afb Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Thu, 9 Apr 2026 11:45:33 -0600 Subject: [PATCH 064/123] Add --format [text|json|tsv|csv] option to all CLI commands Introduces a central formatting.py module (write_records + _record_to_dict) that serialises any dataclass to text, JSON, TSV, or CSV without touching domain objects. A format_option decorator wires --format and --json-indent onto xrefs, ids, and test-concord. test-concord injects a query_curie column for non-text formats. 30 new unit tests in test_formatting.py; 7 CLI format tests added to test_cli.py. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/cli.py | 56 ++++++-- src/babel_explorer/formatting.py | 71 ++++++++++ tests/test_cli.py | 138 +++++++++++++++++- tests/test_formatting.py | 233 +++++++++++++++++++++++++++++++ 4 files changed, 483 insertions(+), 15 deletions(-) create mode 100644 src/babel_explorer/formatting.py create mode 100644 tests/test_formatting.py diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index bc32418..eb7c821 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -4,6 +4,26 @@ from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.babel_xrefs import BabelXRefs from babel_explorer.core.nodenorm import NodeNorm +from babel_explorer.formatting import write_records, _record_to_dict + + +def format_option(f): + """Decorator adding --format and --json-indent options to a command.""" + f = click.option( + "--format", + "fmt", + default="text", + type=click.Choice(["text", "json", "tsv", "csv"]), + show_default=True, + help="Output format", + )(f) + f = click.option( + "--json-indent", + default=2, + show_default=True, + help="Indentation depth for JSON output", + )(f) + return f def parse_duration(value: str) -> int | float: @@ -83,6 +103,7 @@ def cli(): help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", ) +@format_option def xrefs( curies: list[str], babel_url: str, @@ -91,6 +112,8 @@ def xrefs( recurse: bool, labels: bool, check_download: str, + fmt: str, + json_indent: int, ): """ Fetches and prints the cross-references (xrefs) for the given CURIEs. @@ -113,8 +136,7 @@ def xrefs( NodeNorm(nodenorm_url), ) xrefs = bxref.get_curie_xrefs(curies, recurse, label_curies=labels) - for xref in xrefs: - print(xref) + write_records(xrefs, fmt=fmt, indent=json_indent) @cli.command("ids") @@ -139,7 +161,8 @@ def xrefs( help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", ) -def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): +@format_option +def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str, fmt: str, json_indent: int): """ Fetches and prints the ID records for the given CURIEs, along with Biolink type if provided. @@ -160,8 +183,7 @@ def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness) ) xrefs = bxref.get_curie_ids(curies) - for xref in xrefs: - print(xref) + write_records(xrefs, fmt=fmt, indent=json_indent) @cli.command("test-concord") @@ -172,21 +194,27 @@ def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str): default="https://nodenormalization-sri.renci.org/", help="NodeNorm URL to check for concord changes", ) -def test_concord(curies, nodenorm_url): +@format_option +def test_concord(curies, nodenorm_url, fmt, json_indent): """For each CURIE, print the current NodeNorm clique (all equivalent identifiers, labels, and Biolink types). Useful for inspecting how a potential Babel concordance change would affect NodeNorm: run before and after a Babel rebuild to see how cliques would shift. """ nodenorm = NodeNorm(nodenorm_url) - for curie in curies: - identifiers = nodenorm.get_clique_identifiers(curie) - for identifier in identifiers: - biolink = ", ".join(identifier.biolink_type) - if identifier.label: - print(f"{curie}\t{identifier.curie}\t{identifier.label}\t{biolink}") - else: - print(f"{curie}\t{identifier.curie}\t\t{biolink}") + if fmt == "text": + for curie in curies: + for identifier in nodenorm.get_clique_identifiers(curie): + biolink = ", ".join(identifier.biolink_type) + label = identifier.label or "" + print(f"{curie}\t{identifier.curie}\t{label}\t{biolink}") + else: + rows = [ + {"query_curie": curie, **_record_to_dict(ident)} + for curie in curies + for ident in nodenorm.get_clique_identifiers(curie) + ] + write_records(rows, fmt=fmt, indent=json_indent) if __name__ == "__main__": diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py new file mode 100644 index 0000000..7e7701e --- /dev/null +++ b/src/babel_explorer/formatting.py @@ -0,0 +1,71 @@ +"""Output formatting for babel-explorer CLI commands. + +Provides write_records() to render any list of dataclass records (or plain +dicts) as text, JSON, TSV, or CSV. +""" + +import csv +import dataclasses +import json +import sys +from typing import Any + + +def _record_to_dict(record) -> dict[str, Any]: + """Convert a dataclass (or plain dict) to a flat dict. + + Handles IdentifierRecord's extra_fields, which asdict() returns as a + list of [col, val] pairs rather than a nested dict. + """ + if isinstance(record, dict): + return record + d = dataclasses.asdict(record) + if "extra_fields" in d: + for col, val in d.pop("extra_fields"): + d[col] = val + return d + + +def _flatten_for_tabular(row: dict) -> dict: + """Convert list/tuple fields to pipe-joined strings for TSV/CSV output.""" + return {k: "|".join(v) if isinstance(v, (list, tuple)) else v for k, v in row.items()} + + +def write_records(records, fmt: str, indent: int = 2, file=None): + """Write an iterable of dataclass records (or dicts) in the requested format. + + :param records: Iterable of dataclass instances or plain dicts. + :param fmt: One of "text", "json", "tsv", "csv". + :param indent: JSON indentation depth (ignored for other formats). + :param file: Output file-like object; defaults to sys.stdout. + :raises ValueError: If fmt is not a recognised format. + """ + if file is None: + file = sys.stdout + records = list(records) + + if fmt == "text": + for r in records: + print(r, file=file) + + elif fmt == "json": + rows = [_record_to_dict(r) for r in records] + json.dump(rows, file, indent=indent, default=str) + print(file=file) # trailing newline + + elif fmt in ("tsv", "csv"): + if not records: + return + rows = [_flatten_for_tabular(_record_to_dict(r)) for r in records] + delimiter = "\t" if fmt == "tsv" else "," + writer = csv.DictWriter( + file, + fieldnames=list(rows[0].keys()), + delimiter=delimiter, + lineterminator="\n", + ) + writer.writeheader() + writer.writerows(rows) + + else: + raise ValueError(f"Unknown format: {fmt!r}") diff --git a/tests/test_cli.py b/tests/test_cli.py index 3d73e55..62112d4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,12 +4,16 @@ Unit tests — no network required. """ -import pytest +import json + import click +import pytest from click.testing import CliRunner from unittest.mock import patch, MagicMock from babel_explorer.cli import parse_duration, cli +from babel_explorer.core.babel_xrefs import CrossReference, IdentifierRecord +from babel_explorer.core.nodenorm import Identifier # ========================================================================== @@ -193,3 +197,135 @@ def test_test_concord_multiple_curies(self): assert mock_nn.return_value.get_clique_identifiers.call_count == 2 assert "Alpha" in result.output assert "Beta" in result.output + + +class TestOutputFormats: + """Tests for --format option on all commands.""" + + # Shared real dataclass instances (no mocking needed for formatting logic) + _xref = CrossReference(filename="Concord.parquet", subj="A:1", pred="skos:exactMatch", obj="B:2") + _id_record = IdentifierRecord(curie="A:1", extra_fields=(("type", "gene"), ("label", "Alpha"))) + _identifier = Identifier( + curie="MONDO:0004979", label="asthma", + biolink_type=("biolink:Disease",), taxa=(), description=(), + ) + + # -- xrefs -- + + def test_xrefs_format_json(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "json"]) + + assert result.exit_code == 0 + data = json.loads(result.output) + assert isinstance(data, list) + assert data[0]["subj"] == "A:1" + assert data[0]["obj"] == "B:2" + + def test_xrefs_format_tsv(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "tsv"]) + + assert result.exit_code == 0 + lines = result.output.splitlines() + assert lines[0] == "filename\tsubj\tpred\tobj" + assert "A:1" in lines[1] + + def test_xrefs_format_csv(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "csv"]) + + assert result.exit_code == 0 + lines = result.output.splitlines() + assert lines[0] == "filename,subj,pred,obj" + assert "A:1" in lines[1] + + # -- ids -- + + def test_ids_format_json_expands_extra_fields(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + ): + mock_bx.return_value.get_curie_ids.return_value = [self._id_record] + result = runner.invoke(cli, ["ids", "A:1", "--format", "json"]) + + assert result.exit_code == 0 + data = json.loads(result.output) + assert data[0]["curie"] == "A:1" + assert data[0]["type"] == "gene" + assert data[0]["label"] == "Alpha" + assert "extra_fields" not in data[0] + + def test_ids_format_tsv_expands_extra_fields(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + ): + mock_bx.return_value.get_curie_ids.return_value = [self._id_record] + result = runner.invoke(cli, ["ids", "A:1", "--format", "tsv"]) + + assert result.exit_code == 0 + lines = result.output.splitlines() + assert "type" in lines[0] + assert "label" in lines[0] + assert "gene" in lines[1] + + # -- test-concord -- + + def test_test_concord_format_json_includes_query_curie(self): + runner = CliRunner() + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [self._identifier] + result = runner.invoke(cli, ["test-concord", "MONDO:0004979", "--format", "json"]) + + assert result.exit_code == 0 + data = json.loads(result.output) + assert data[0]["query_curie"] == "MONDO:0004979" + assert data[0]["curie"] == "MONDO:0004979" + assert data[0]["label"] == "asthma" + assert data[0]["biolink_type"] == ["biolink:Disease"] + + def test_test_concord_format_tsv(self): + runner = CliRunner() + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [self._identifier] + result = runner.invoke(cli, ["test-concord", "MONDO:0004979", "--format", "tsv"]) + + assert result.exit_code == 0 + lines = result.output.splitlines() + assert "query_curie" in lines[0] + assert "MONDO:0004979" in lines[1] + + # -- format validation -- + + def test_invalid_format_rejected_by_click(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs"), + patch("babel_explorer.cli.NodeNorm"), + ): + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "xml"]) + + assert result.exit_code != 0 diff --git a/tests/test_formatting.py b/tests/test_formatting.py new file mode 100644 index 0000000..9b9aa90 --- /dev/null +++ b/tests/test_formatting.py @@ -0,0 +1,233 @@ +""" +Unit tests for formatting.py — no network, no mocking required. +""" + +import io +import json + +import pytest + +from babel_explorer.core.babel_xrefs import CrossReference, LabeledCrossReference, IdentifierRecord +from babel_explorer.core.nodenorm import Identifier +from babel_explorer.formatting import _record_to_dict, write_records + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def xref(): + return CrossReference(filename="Concord.parquet", subj="A:1", pred="skos:exactMatch", obj="B:2") + + +@pytest.fixture +def labeled_xref(): + return LabeledCrossReference( + filename="Concord.parquet", + subj="A:1", + pred="skos:exactMatch", + obj="B:2", + subj_label="Alpha", + subj_biolink_type=("biolink:Disease",), + obj_label="Beta", + obj_biolink_type=("biolink:Gene", "biolink:NamedThing"), + ) + + +@pytest.fixture +def id_record(): + return IdentifierRecord( + curie="A:1", + extra_fields=(("type", "gene"), ("label", "Alpha")), + ) + + +@pytest.fixture +def identifier(): + return Identifier( + curie="MONDO:0004979", + label="asthma", + biolink_type=("biolink:Disease",), + taxa=("NCBITaxon:9606",), + description=("A chronic inflammatory disease",), + ) + + +# --------------------------------------------------------------------------- +# Tests for _record_to_dict +# --------------------------------------------------------------------------- + + +class TestRecordToDict: + def test_cross_reference(self, xref): + d = _record_to_dict(xref) + assert d == {"filename": "Concord.parquet", "subj": "A:1", "pred": "skos:exactMatch", "obj": "B:2"} + + def test_labeled_cross_reference_has_all_eight_fields(self, labeled_xref): + d = _record_to_dict(labeled_xref) + assert set(d.keys()) == { + "filename", "subj", "pred", "obj", + "subj_label", "subj_biolink_type", "obj_label", "obj_biolink_type", + } + # dataclasses.asdict() preserves tuple types + assert d["subj_biolink_type"] == ("biolink:Disease",) + assert d["obj_biolink_type"] == ("biolink:Gene", "biolink:NamedThing") + + def test_identifier_record_extra_fields_expanded(self, id_record): + d = _record_to_dict(id_record) + assert "extra_fields" not in d + assert d["curie"] == "A:1" + assert d["type"] == "gene" + assert d["label"] == "Alpha" + + def test_identifier_record_no_extra_fields(self): + rec = IdentifierRecord(curie="X:1") + d = _record_to_dict(rec) + assert d == {"curie": "X:1"} + + def test_plain_dict_passthrough(self): + data = {"a": 1, "b": "hello"} + assert _record_to_dict(data) is data + + def test_identifier_dataclass(self, identifier): + d = _record_to_dict(identifier) + assert d["curie"] == "MONDO:0004979" + assert d["label"] == "asthma" + # dataclasses.asdict() preserves tuple types + assert d["biolink_type"] == ("biolink:Disease",) + assert d["taxa"] == ("NCBITaxon:9606",) + + +# --------------------------------------------------------------------------- +# Tests for write_records +# --------------------------------------------------------------------------- + + +class TestWriteRecords: + + # -- text format -- + + def test_text_uses_str(self, xref): + out = io.StringIO() + write_records([xref], "text", file=out) + assert out.getvalue().strip() == str(xref) + + def test_text_empty_no_output(self): + out = io.StringIO() + write_records([], "text", file=out) + assert out.getvalue() == "" + + def test_text_multiple_records(self, xref): + out = io.StringIO() + write_records([xref, xref], "text", file=out) + lines = out.getvalue().strip().splitlines() + assert len(lines) == 2 + + # -- json format -- + + def test_json_is_valid_list(self, xref): + out = io.StringIO() + write_records([xref], "json", file=out) + data = json.loads(out.getvalue()) + assert isinstance(data, list) + assert len(data) == 1 + assert data[0]["subj"] == "A:1" + + def test_json_empty_list(self): + out = io.StringIO() + write_records([], "json", file=out) + assert json.loads(out.getvalue()) == [] + + def test_json_indent_controls_formatting(self, xref): + out_pretty = io.StringIO() + write_records([xref], "json", indent=2, file=out_pretty) + + out_compact = io.StringIO() + write_records([xref], "json", indent=None, file=out_compact) + + # Pretty-printed output has more lines (has newlines per field) + assert out_pretty.getvalue().count("\n") > out_compact.getvalue().count("\n") + + def test_json_tuple_fields_serialized_as_arrays(self, labeled_xref): + # json.dump converts tuples to JSON arrays, so json.loads gives back lists + out = io.StringIO() + write_records([labeled_xref], "json", file=out) + data = json.loads(out.getvalue()) + assert isinstance(data[0]["subj_biolink_type"], list) + assert data[0]["obj_biolink_type"] == ["biolink:Gene", "biolink:NamedThing"] + + def test_json_plain_dict(self): + out = io.StringIO() + write_records([{"a": 1, "b": "x"}], "json", file=out) + assert json.loads(out.getvalue()) == [{"a": 1, "b": "x"}] + + # -- tsv format -- + + def test_tsv_has_header_row(self, xref): + out = io.StringIO() + write_records([xref], "tsv", file=out) + lines = out.getvalue().splitlines() + assert lines[0] == "filename\tsubj\tpred\tobj" + + def test_tsv_data_row(self, xref): + out = io.StringIO() + write_records([xref], "tsv", file=out) + lines = out.getvalue().splitlines() + assert lines[1] == "Concord.parquet\tA:1\tskos:exactMatch\tB:2" + + def test_tsv_tuple_fields_pipe_joined(self, labeled_xref): + out = io.StringIO() + write_records([labeled_xref], "tsv", file=out) + lines = out.getvalue().splitlines() + # Header row + assert "subj_biolink_type" in lines[0] + # Data row: multi-value tuple joined with pipe + assert "biolink:Gene|biolink:NamedThing" in lines[1] + + def test_tsv_empty_no_output(self): + out = io.StringIO() + write_records([], "tsv", file=out) + assert out.getvalue() == "" + + def test_tsv_identifier_record_extra_fields_expanded(self, id_record): + out = io.StringIO() + write_records([id_record], "tsv", file=out) + lines = out.getvalue().splitlines() + assert "curie" in lines[0] + assert "type" in lines[0] + assert "label" in lines[0] + assert "A:1" in lines[1] + + # -- csv format -- + + def test_csv_has_header_row(self, xref): + out = io.StringIO() + write_records([xref], "csv", file=out) + lines = out.getvalue().splitlines() + assert lines[0] == "filename,subj,pred,obj" + + def test_csv_data_row(self, xref): + out = io.StringIO() + write_records([xref], "csv", file=out) + lines = out.getvalue().splitlines() + assert lines[1] == "Concord.parquet,A:1,skos:exactMatch,B:2" + + def test_csv_empty_no_output(self): + out = io.StringIO() + write_records([], "csv", file=out) + assert out.getvalue() == "" + + def test_csv_tuple_fields_pipe_joined(self, labeled_xref): + out = io.StringIO() + write_records([labeled_xref], "csv", file=out) + lines = out.getvalue().splitlines() + assert "biolink:Gene|biolink:NamedThing" in lines[1] + + # -- invalid format -- + + def test_invalid_format_raises_value_error(self, xref): + out = io.StringIO() + with pytest.raises(ValueError, match="Unknown format"): + write_records([xref], "xml", file=out) From 07610333f2af8fbd579065dd6490308a2cb22a30 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Thu, 9 Apr 2026 12:10:12 -0600 Subject: [PATCH 065/123] Add console format with rich color highlighting; replace text default Replaces the 'text' default format with 'console', backed by the rich library. xrefs and test-concord highlight query CURIEs in bold cyan wherever they appear as subject or object; rich auto-strips markup when output is piped. ids uses console.print(str(record)) for TTY-aware plain output. formatting.py gains make_console() and hl_curie() utilities for new commands to reuse. LabeledCrossReference labels appear in parentheses next to CURIEs in console output. Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 1 + src/babel_explorer/cli.py | 51 ++++++++++++++++---- src/babel_explorer/formatting.py | 32 +++++++++--- tests/test_cli.py | 83 ++++++++++++++++++++++++++++++-- tests/test_formatting.py | 80 ++++++++++++++++++++++-------- uv.lock | 36 ++++++++++++++ 6 files changed, 241 insertions(+), 42 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 59c1b68..34a0be3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "click>=8.3.1", "duckdb>=1.4.2", "requests>=2.32.5", + "rich>=13", "tqdm>=4.67.0", ] diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index eb7c821..58f1e5e 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -4,7 +4,9 @@ from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.babel_xrefs import BabelXRefs from babel_explorer.core.nodenorm import NodeNorm -from babel_explorer.formatting import write_records, _record_to_dict +from babel_explorer.core.babel_xrefs import LabeledCrossReference +from babel_explorer.formatting import write_records, _record_to_dict, make_console, hl_curie +from rich.markup import escape def format_option(f): @@ -12,8 +14,8 @@ def format_option(f): f = click.option( "--format", "fmt", - default="text", - type=click.Choice(["text", "json", "tsv", "csv"]), + default="console", + type=click.Choice(["console", "json", "tsv", "csv"]), show_default=True, help="Output format", )(f) @@ -136,7 +138,24 @@ def xrefs( NodeNorm(nodenorm_url), ) xrefs = bxref.get_curie_xrefs(curies, recurse, label_curies=labels) - write_records(xrefs, fmt=fmt, indent=json_indent) + + if fmt == "console": + console = make_console() + query_set = set(curies) + for xref in xrefs: + subj_str = hl_curie(xref.subj, xref.subj in query_set) + obj_str = hl_curie(xref.obj, xref.obj in query_set) + if isinstance(xref, LabeledCrossReference): + if xref.subj_label: + subj_str += f" ({escape(xref.subj_label)})" + if xref.obj_label: + obj_str += f" ({escape(xref.obj_label)})" + console.print( + f"{subj_str} [dim]{escape(xref.pred)}[/dim] " + f"{obj_str} [dim italic]{escape(xref.filename)}[/dim italic]" + ) + else: + write_records(xrefs, fmt=fmt, indent=json_indent) @cli.command("ids") @@ -183,7 +202,13 @@ def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str, BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness) ) xrefs = bxref.get_curie_ids(curies) - write_records(xrefs, fmt=fmt, indent=json_indent) + + if fmt == "console": + console = make_console() + for record in xrefs: + console.print(str(record)) + else: + write_records(xrefs, fmt=fmt, indent=json_indent) @cli.command("test-concord") @@ -202,12 +227,18 @@ def test_concord(curies, nodenorm_url, fmt, json_indent): run before and after a Babel rebuild to see how cliques would shift. """ nodenorm = NodeNorm(nodenorm_url) - if fmt == "text": + if fmt == "console": + console = make_console() + query_set = set(curies) for curie in curies: - for identifier in nodenorm.get_clique_identifiers(curie): - biolink = ", ".join(identifier.biolink_type) - label = identifier.label or "" - print(f"{curie}\t{identifier.curie}\t{label}\t{biolink}") + for ident in nodenorm.get_clique_identifiers(curie): + biolink = ", ".join(ident.biolink_type) + console.print( + f"{hl_curie(curie, True)} " + f"{hl_curie(ident.curie, ident.curie in query_set)} " + f"{escape(ident.label or '-')} " + f"[dim]{escape(biolink)}[/dim]" + ) else: rows = [ {"query_curie": curie, **_record_to_dict(ident)} diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index 7e7701e..191f1cd 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -1,7 +1,8 @@ """Output formatting for babel-explorer CLI commands. -Provides write_records() to render any list of dataclass records (or plain -dicts) as text, JSON, TSV, or CSV. +Provides: +- write_records() for machine-readable output (json, tsv, csv) +- make_console() and hl_curie() for rich console output """ import csv @@ -10,6 +11,9 @@ import sys from typing import Any +from rich.console import Console +from rich.markup import escape + def _record_to_dict(record) -> dict[str, Any]: """Convert a dataclass (or plain dict) to a flat dict. @@ -31,11 +35,27 @@ def _flatten_for_tabular(row: dict) -> dict: return {k: "|".join(v) if isinstance(v, (list, tuple)) else v for k, v in row.items()} +def make_console(file=None) -> Console: + """Create a rich Console with babel-explorer defaults. + + Auto-detects TTY and NO_COLOR; strips markup when output is piped. + highlight=False prevents rich from auto-highlighting numbers and strings. + """ + return Console(file=file, highlight=False) + + +def hl_curie(curie: str, highlight: bool) -> str: + """Return rich markup for a CURIE — bold cyan if it is a query CURIE.""" + escaped = escape(curie) + return f"[bold cyan]{escaped}[/bold cyan]" if highlight else escaped + + def write_records(records, fmt: str, indent: int = 2, file=None): """Write an iterable of dataclass records (or dicts) in the requested format. :param records: Iterable of dataclass instances or plain dicts. - :param fmt: One of "text", "json", "tsv", "csv". + :param fmt: One of "json", "tsv", "csv". (Console output is handled by + make_console/hl_curie in the CLI layer.) :param indent: JSON indentation depth (ignored for other formats). :param file: Output file-like object; defaults to sys.stdout. :raises ValueError: If fmt is not a recognised format. @@ -44,11 +64,7 @@ def write_records(records, fmt: str, indent: int = 2, file=None): file = sys.stdout records = list(records) - if fmt == "text": - for r in records: - print(r, file=file) - - elif fmt == "json": + if fmt == "json": rows = [_record_to_dict(r) for r in records] json.dump(rows, file, indent=indent, default=str) print(file=file) # trailing newline diff --git a/tests/test_cli.py b/tests/test_cli.py index 62112d4..f4f6dbb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -67,6 +67,10 @@ def test_xrefs_happy_path(self): runner = CliRunner() mock_xref = MagicMock() mock_xref.__str__ = lambda self: "A:1 skos:exactMatch B:2" + mock_xref.subj = "A:1" + mock_xref.obj = "B:2" + mock_xref.pred = "skos:exactMatch" + mock_xref.filename = "test.parquet" with ( patch("babel_explorer.cli.BabelDownloader"), @@ -84,7 +88,10 @@ def test_xrefs_happy_path(self): def test_xrefs_recurse_and_labels_flags(self): runner = CliRunner() mock_xref = MagicMock() - mock_xref.__str__ = lambda self: "A:1 skos:exactMatch B:2" + mock_xref.subj = "A:1" + mock_xref.obj = "B:2" + mock_xref.pred = "skos:exactMatch" + mock_xref.filename = "test.parquet" with ( patch("babel_explorer.cli.BabelDownloader"), @@ -210,7 +217,65 @@ class TestOutputFormats: biolink_type=("biolink:Disease",), taxa=(), description=(), ) - # -- xrefs -- + # -- console format (default) -- + + def test_xrefs_default_format_is_console(self): + """Default format is console — output contains the CURIEs as plain text (no TTY in runner).""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1"]) + + assert result.exit_code == 0 + # Rich strips markup on non-TTY; plain CURIEs and predicate appear + assert "A:1" in result.output + assert "B:2" in result.output + assert "skos:exactMatch" in result.output + + def test_xrefs_console_shows_query_curie(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "console"]) + + assert result.exit_code == 0 + assert "A:1" in result.output + + def test_test_concord_console_format(self): + runner = CliRunner() + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [self._identifier] + result = runner.invoke(cli, ["test-concord", "MONDO:0004979", "--format", "console"]) + + assert result.exit_code == 0 + assert "MONDO:0004979" in result.output + assert "asthma" in result.output + assert "biolink:Disease" in result.output + + def test_test_concord_console_no_label_shows_dash(self): + """Identifiers with no label display '-' in console format.""" + runner = CliRunner() + mock_ident = MagicMock() + mock_ident.curie = "MONDO:0004979" + mock_ident.label = None + mock_ident.biolink_type = ["biolink:Disease"] + + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] + result = runner.invoke(cli, ["test-concord", "MONDO:0004979", "--format", "console"]) + + assert result.exit_code == 0 + assert "-" in result.output + + # -- json format -- def test_xrefs_format_json(self): runner = CliRunner() @@ -291,7 +356,7 @@ def test_ids_format_tsv_expands_extra_fields(self): assert "label" in lines[0] assert "gene" in lines[1] - # -- test-concord -- + # -- test-concord structured formats -- def test_test_concord_format_json_includes_query_curie(self): runner = CliRunner() @@ -329,3 +394,15 @@ def test_invalid_format_rejected_by_click(self): result = runner.invoke(cli, ["xrefs", "A:1", "--format", "xml"]) assert result.exit_code != 0 + + def test_text_format_rejected_by_click(self): + """'text' was removed; it is no longer a valid choice.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs"), + patch("babel_explorer.cli.NodeNorm"), + ): + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "text"]) + + assert result.exit_code != 0 diff --git a/tests/test_formatting.py b/tests/test_formatting.py index 9b9aa90..2d402db 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -6,10 +6,11 @@ import json import pytest +from rich.console import Console from babel_explorer.core.babel_xrefs import CrossReference, LabeledCrossReference, IdentifierRecord from babel_explorer.core.nodenorm import Identifier -from babel_explorer.formatting import _record_to_dict, write_records +from babel_explorer.formatting import _record_to_dict, write_records, make_console, hl_curie # --------------------------------------------------------------------------- @@ -55,6 +56,50 @@ def identifier(): ) +# --------------------------------------------------------------------------- +# Tests for make_console and hl_curie +# --------------------------------------------------------------------------- + + +class TestConsoleUtilities: + def test_make_console_returns_console(self): + console = make_console() + assert isinstance(console, Console) + + def test_make_console_accepts_file(self): + out = io.StringIO() + console = make_console(file=out) + assert isinstance(console, Console) + console.print("hello") + assert "hello" in out.getvalue() + + def test_hl_curie_highlighted_contains_markup(self): + result = hl_curie("HGNC:1100", highlight=True) + assert "bold cyan" in result + assert "HGNC:1100" in result + + def test_hl_curie_not_highlighted_is_plain(self): + result = hl_curie("HGNC:1100", highlight=False) + assert result == "HGNC:1100" + assert "[" not in result + + def test_hl_curie_highlighted_renders_correctly(self): + """Markup renders to plain text on a non-TTY console.""" + out = io.StringIO() + console = Console(file=out, highlight=False, no_color=True) + console.print(hl_curie("HGNC:1100", highlight=True)) + assert "HGNC:1100" in out.getvalue() + + def test_hl_curie_highlighted_renders_with_color(self): + """On a forced-TTY console, ANSI codes are emitted.""" + out = io.StringIO() + console = Console(file=out, highlight=False, force_terminal=True) + console.print(hl_curie("HGNC:1100", highlight=True)) + output = out.getvalue() + assert "HGNC:1100" in output + assert "\x1b[" in output # ANSI escape present + + # --------------------------------------------------------------------------- # Tests for _record_to_dict # --------------------------------------------------------------------------- @@ -107,24 +152,6 @@ def test_identifier_dataclass(self, identifier): class TestWriteRecords: - # -- text format -- - - def test_text_uses_str(self, xref): - out = io.StringIO() - write_records([xref], "text", file=out) - assert out.getvalue().strip() == str(xref) - - def test_text_empty_no_output(self): - out = io.StringIO() - write_records([], "text", file=out) - assert out.getvalue() == "" - - def test_text_multiple_records(self, xref): - out = io.StringIO() - write_records([xref, xref], "text", file=out) - lines = out.getvalue().strip().splitlines() - assert len(lines) == 2 - # -- json format -- def test_json_is_valid_list(self, xref): @@ -225,9 +252,20 @@ def test_csv_tuple_fields_pipe_joined(self, labeled_xref): lines = out.getvalue().splitlines() assert "biolink:Gene|biolink:NamedThing" in lines[1] - # -- invalid format -- + # -- invalid formats (including console, which is handled at CLI layer) -- + + def test_text_format_raises_value_error(self, xref): + out = io.StringIO() + with pytest.raises(ValueError, match="Unknown format"): + write_records([xref], "text", file=out) + + def test_console_format_raises_value_error(self, xref): + """Console format is handled by the CLI, not write_records.""" + out = io.StringIO() + with pytest.raises(ValueError, match="Unknown format"): + write_records([xref], "console", file=out) - def test_invalid_format_raises_value_error(self, xref): + def test_unknown_format_raises_value_error(self, xref): out = io.StringIO() with pytest.raises(ValueError, match="Unknown format"): write_records([xref], "xml", file=out) diff --git a/uv.lock b/uv.lock index b8496b5..7b201e4 100644 --- a/uv.lock +++ b/uv.lock @@ -10,6 +10,7 @@ dependencies = [ { name = "click" }, { name = "duckdb" }, { name = "requests" }, + { name = "rich" }, { name = "tqdm" }, ] @@ -26,6 +27,7 @@ requires-dist = [ { name = "click", specifier = ">=8.3.1" }, { name = "duckdb", specifier = ">=1.4.2" }, { name = "requests", specifier = ">=2.32.5" }, + { name = "rich", specifier = ">=13" }, { name = "tqdm", specifier = ">=4.67.0" }, ] @@ -212,6 +214,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -316,6 +339,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "rich" +version = "14.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, +] + [[package]] name = "ruff" version = "0.15.2" From 859c2792d6f5f86fcf96b6c6fabdd4140a7fdb22 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Thu, 9 Apr 2026 13:41:39 -0600 Subject: [PATCH 066/123] Fix Identifier.from_dict splitting string fields into characters tuple() on a bare string iterates its characters, so biolink_type, taxa, and description would become ('b','i','o',...) when NodeNorm returns them as strings rather than lists. _to_tuple() now wraps a bare string in a 1-tuple. Four new unit tests cover the string case for each field. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/nodenorm.py | 12 ++++++--- tests/test_nodenorm.py | 38 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index 9ce916d..4f0f6d1 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -21,12 +21,18 @@ def __lt__(self, other): @staticmethod def from_dict(d: dict) -> "Identifier": + def _to_tuple(val) -> tuple[str, ...]: + """Coerce a string or list to a tuple — guards against iterating string chars.""" + if not val: + return () + return (val,) if isinstance(val, str) else tuple(val) + return Identifier( curie=d["identifier"], label=d.get("label", ""), - biolink_type=tuple(d.get("type", [])), - taxa=tuple(d.get("taxa", [])), - description=tuple(d.get("description", [])), + biolink_type=_to_tuple(d.get("type")), + taxa=_to_tuple(d.get("taxa")), + description=_to_tuple(d.get("description")), ) diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index 8b30fcd..57b6dab 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -71,6 +71,44 @@ def test_from_dict_partial(self): assert ident.label == "Beta" assert ident.biolink_type == () + def test_from_dict_type_as_string(self): + """NodeNorm may return 'type' as a bare string for individual identifiers.""" + d = {"identifier": "X:1", "type": "biolink:Disease"} + ident = Identifier.from_dict(d) + assert ident.biolink_type == ("biolink:Disease",), ( + "biolink_type should be a 1-tuple, not a tuple of characters" + ) + + def test_from_dict_description_as_string(self): + """NodeNorm may return 'description' as a bare string.""" + d = {"identifier": "X:1", "description": "A chronic disease"} + ident = Identifier.from_dict(d) + assert ident.description == ("A chronic disease",), ( + "description should be a 1-tuple, not a tuple of characters" + ) + + def test_from_dict_taxa_as_string(self): + """NodeNorm may return 'taxa' as a bare string.""" + d = {"identifier": "X:1", "taxa": "NCBITaxon:9606"} + ident = Identifier.from_dict(d) + assert ident.taxa == ("NCBITaxon:9606",), ( + "taxa should be a 1-tuple, not a tuple of characters" + ) + + def test_from_dict_all_fields_as_strings(self): + """All three tuple fields as strings produce correct single-element tuples.""" + d = { + "identifier": "X:1", + "label": "Alpha", + "type": "biolink:NamedThing", + "taxa": "NCBITaxon:9606", + "description": "Some description", + } + ident = Identifier.from_dict(d) + assert ident.biolink_type == ("biolink:NamedThing",) + assert ident.taxa == ("NCBITaxon:9606",) + assert ident.description == ("Some description",) + def test_lt_ordering(self): a = Identifier(curie="A:1") b = Identifier(curie="B:2") From 23532d5b8e54d3fb2b89224c70658f93dff9a766 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 14:44:01 -0400 Subject: [PATCH 067/123] Deduplicate CLI options and centralise logging setup Extract babel_options decorator covering --local-dir, --babel-url, and --check-download so the three options are defined once instead of being copy-pasted into xrefs and ids. Move logging.basicConfig into the cli group callback so it fires once rather than inside each subcommand. Replace the file-level # comment with a proper module docstring. Update FUTURE.md: mark CLI deduplication resolved; add two new items for batch NodeNorm lookups and DuckDB connection reuse. Co-Authored-By: Claude Sonnet 4.6 --- FUTURE.md | 16 ++++++--- src/babel_explorer/cli.py | 75 +++++++++++++++------------------------ 2 files changed, 41 insertions(+), 50 deletions(-) diff --git a/FUTURE.md b/FUTURE.md index a2bb3fb..b49144c 100644 --- a/FUTURE.md +++ b/FUTURE.md @@ -1,7 +1,15 @@ # Future Work -## Deduplicate CLI option blocks +## Batch NodeNorm lookups -`--local-dir`, `--babel-url`, and `--check-download` are copy-pasted between the -`xrefs` and `ids` commands in `cli.py`. Extract a `@common_babel_options` Click -decorator so defaults are defined in one place and can't drift. +`normalize_curie` makes one HTTP round-trip per CURIE. NodeNorm's +`/get_normalized_nodes` endpoint accepts repeated `curie=` parameters in one +request. Adding a `normalize_curies(curies)` batch method that pre-warms the +`normalize_curie` cache would collapse N serial round-trips into one when +`--labels` is set. + +## Reuse a single DuckDB connection per BabelXRefs instance + +`get_curie_xref` opens a fresh DuckDB connection and re-reads Concord.parquet +for each unique CURIE. Opening one connection at first use and reusing it across +all queries would eliminate the per-CURIE open/scan overhead. diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 58f1e5e..2d75248 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -1,4 +1,4 @@ -# Command line interface for babel-explorer +"""Command-line interface for babel-explorer.""" import click import logging from babel_explorer.core.downloader import BabelDownloader @@ -9,6 +9,31 @@ from rich.markup import escape +def babel_options(f): + """Decorator adding --local-dir, --babel-url, and --check-download options to a command.""" + f = click.option( + "--check-download", + type=str, + default="3h", + show_default=True, + help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " + "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", + )(f) + f = click.option( + "--babel-url", + type=str, + default="https://stars.renci.org:443/var/babel/2025nov19/", + help="Base URL of the Babel server", + )(f) + f = click.option( + "--local-dir", + type=str, + default="data/2025nov19", + help="Local location to save Babel download files to", + )(f) + return f + + def format_option(f): """Decorator adding --format and --json-indent options to a command.""" f = click.option( @@ -72,23 +97,12 @@ def parse_duration(value: str) -> int | float: @click.group() def cli(): """babel-explorer: query and explore Babel intermediate files.""" - pass + logging.basicConfig(level=logging.INFO) @cli.command("xrefs") @click.argument("curies", type=str, required=True, nargs=-1) -@click.option( - "--local-dir", - type=str, - default="data/2025nov19", - help="Local location to save Babel download files to", -) -@click.option( - "--babel-url", - type=str, - default="https://stars.renci.org:443/var/babel/2025nov19/", - help="Base URL of the Babel server", -) +@babel_options @click.option( "--nodenorm-url", type=str, @@ -97,14 +111,6 @@ def cli(): ) @click.option("--recurse", is_flag=True, help="Recursively query returned xrefs") @click.option("--labels", is_flag=True, help="Include labels for CURIEs") -@click.option( - "--check-download", - type=str, - default="3h", - show_default=True, - help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " - "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", -) @format_option def xrefs( curies: list[str], @@ -130,8 +136,6 @@ def xrefs( :return: None """ - logging.basicConfig(level=logging.INFO) - freshness = parse_duration(check_download) bxref = BabelXRefs( BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness), @@ -160,26 +164,7 @@ def xrefs( @cli.command("ids") @click.argument("curies", type=str, required=True, nargs=-1) -@click.option( - "--local-dir", - type=str, - default="data/2025nov19", - help="Local location to save Babel download files to", -) -@click.option( - "--babel-url", - type=str, - default="https://stars.renci.org:443/var/babel/2025nov19/", - help="Base URL of the Babel server", -) -@click.option( - "--check-download", - type=str, - default="3h", - show_default=True, - help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " - "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", -) +@babel_options @format_option def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str, fmt: str, json_indent: int): """ @@ -195,8 +180,6 @@ def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str, :return: None """ - logging.basicConfig(level=logging.INFO) - freshness = parse_duration(check_download) bxref = BabelXRefs( BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness) From 92f48e861b686000678be21a866201888d045796 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 14:45:08 -0400 Subject: [PATCH 068/123] Simplify core query and normalisation logic babel_xrefs: extract _require_nodenorm() to replace the identical guard duplicated in get_curie_xref and _get_curie_xrefs_recursive. Delete LabeledCrossReference.__str__; the dataclass-generated __repr__ already includes all fields. nodenorm: drop the five unused keyword parameters from normalize_curie (no caller ever varies them; values hardcoded inline). Collapse the sequential double-guard in get_clique_identifiers into a single condition. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/babel_xrefs.py | 21 ++++++++++----------- src/babel_explorer/core/nodenorm.py | 24 +++++++----------------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 0c94074..dcf84ed 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -53,9 +53,6 @@ class LabeledCrossReference(CrossReference): obj_label: str obj_biolink_type: tuple[str, ...] - def __str__(self): - return f"""LabeledCrossReference(subj="{self.subj}", pred="{self.pred}", obj="{self.obj}", subj_label="{self.subj_label}", subj_biolink_type="{self.subj_biolink_type}", obj_label="{self.obj_label}", obj_biolink_type="{self.obj_biolink_type}")""" - @dataclasses.dataclass(frozen=True) class IdentifierRecord: @@ -99,6 +96,12 @@ def __init__(self, downloader: BabelDownloader, nodenorm: NodeNorm = None): self.downloader = downloader self.nodenorm = nodenorm + def _require_nodenorm(self): + if self.nodenorm is None: + raise ValueError( + "label_curies=True requires a configured NodeNorm instance (nodenorm was None)." + ) + def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: """ Search for all identifiers in the /ids/ files for a particular CURIE. @@ -135,10 +138,8 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): :raises ValueError: If ``label_curies=True`` but no NodeNorm instance is available. :return: A list of ``CrossReference`` (or ``LabeledCrossReference``) objects. """ - if label_curies and self.nodenorm is None: - raise ValueError( - "label_curies=True requires a configured NodeNorm instance (nodenorm was None)." - ) + if label_curies: + self._require_nodenorm() concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") @@ -171,10 +172,8 @@ def _to_labeled_xref(self, xref: CrossReference) -> LabeledCrossReference: def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = False): """Traverse the cross-reference graph in one DuckDB WITH RECURSIVE query.""" - if label_curies and self.nodenorm is None: - raise ValueError( - "label_curies=True requires a configured NodeNorm instance (nodenorm was None)." - ) + if label_curies: + self._require_nodenorm() if not curies: return [] diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index 4f0f6d1..c98c085 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -75,15 +75,7 @@ def get_identifier(self, curie: str) -> "Identifier": return Identifier(curie=curie) @functools.lru_cache(maxsize=None) - def normalize_curie( - self, - curie: str, - conflate=True, - drug_chemical_conflate=True, - description=True, - individual_types=True, - include_taxa=True, - ): + def normalize_curie(self, curie: str): """Call ``get_normalized_nodes`` and return the per-CURIE result dict. :return: The normalisation dict for *curie* (contains ``id``, ``equivalent_identifiers``, @@ -96,11 +88,11 @@ def normalize_curie( f"{self.nodenorm_url}get_normalized_nodes", params={ "curie": curie, - "conflate": conflate, - "drug_chemical_conflate": drug_chemical_conflate, - "description": description, - "individual_types": individual_types, - "include_taxa": include_taxa, + "conflate": True, + "drug_chemical_conflate": True, + "description": True, + "individual_types": True, + "include_taxa": True, }, timeout=self.timeout, ) @@ -123,8 +115,6 @@ def get_clique_identifiers(self, curie: str) -> list[Identifier]: or an empty list if the CURIE is unknown or has no equivalents. """ result = self.normalize_curie(curie) - if not result: - return [] - if "equivalent_identifiers" not in result: + if not result or "equivalent_identifiers" not in result: return [] return [Identifier.from_dict(x) for x in result["equivalent_identifiers"]] From be2d86543d3ac8e1d24efad577c08bb9c5caac3b Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 14:45:49 -0400 Subject: [PATCH 069/123] Clean up downloader: fix redundant check, remove noisy comments Drop the redundant os.path.exists() call in the elif branch of __init__ (the if-not-exists branch above guarantees it is True at that point; only the isdir check is needed). Delete self-evident comments from _stream_download and _download_with_retry; keep the non-obvious notes about connection-only timeout and the three-tier freshness logic. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/downloader.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 4d2f9a2..aa073ee 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -47,11 +47,10 @@ def __init__( if local_path is None: local_path = tempfile.gettempdir() - # Make sure the local path is an existing directory or that we can create it. if not os.path.exists(local_path): os.makedirs(local_path, exist_ok=True) self.local_path = local_path - elif os.path.exists(local_path) and os.path.isdir(local_path): + elif os.path.isdir(local_path): self.local_path = local_path else: raise ValueError( @@ -193,14 +192,12 @@ def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): resume_byte_pos: Starting byte position (for resume) chunk_size: Size of chunks to read/write """ - # Get total size from Content-Length header (may not be present) content_length = response.headers.get("Content-Length") if content_length: total_size = int(content_length) + resume_byte_pos else: total_size = None - # Open file in append mode if resuming, write mode otherwise mode = "ab" if resume_byte_pos > 0 else "wb" with open(local_path, mode) as f: @@ -234,48 +231,38 @@ def _download_with_retry(self, url, local_path, chunk_size): """ for attempt in range(1, self.retries + 1): try: - # Check if we're resuming a partial download resume_byte_pos = 0 if os.path.exists(local_path): resume_byte_pos = os.path.getsize(local_path) - # Prepare headers for resume headers = {} if resume_byte_pos > 0: headers["Range"] = f"bytes={resume_byte_pos}-" self.logger.info(f"Resuming download from byte {resume_byte_pos}") - # Make streaming request with timeout for connection (not total time) + # timeout applies to connection only, not total transfer time with requests.get( url, headers=headers, stream=True, timeout=self.timeout ) as response: - # Handle different response codes if response.status_code == 416: - # Range Not Satisfiable - file already complete self.logger.info(f"File already complete: {local_path}") return response.headers elif response.status_code == 206: - # Partial Content - resume successful self.logger.info("Resuming download (HTTP 206)") elif response.status_code == 200: - # OK - server doesn't support resume or no Range header was sent if resume_byte_pos > 0: self.logger.warning( "Server doesn't support resume, restarting from beginning" ) resume_byte_pos = 0 - # Remove partial file if os.path.exists(local_path): os.remove(local_path) else: response.raise_for_status() - # Stream download with progress bar self._stream_download( response, local_path, resume_byte_pos, chunk_size ) - - # Success - exit retry loop return response.headers except (requests.RequestException, IOError) as e: @@ -284,12 +271,10 @@ def _download_with_retry(self, url, local_path, chunk_size): ) if attempt < self.retries: - # Calculate exponential backoff with max of 60 seconds wait_time = min(2**attempt, 60) self.logger.info(f"Retrying in {wait_time} seconds...") time.sleep(wait_time) else: - # All retries exhausted raise RuntimeError( f"Failed to download {url} after {self.retries} attempts: {e}" ) From d614af10db62e06eb09887cfc11297ef9ca5c661 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 17:32:01 -0400 Subject: [PATCH 070/123] Fix lru_cache memory leak and IN-parameter query in BabelXRefs Replace @functools.lru_cache on the get_curie_xref instance method with an instance-level dict cache (self._xref_cache). lru_cache on instance methods holds a strong reference to self via the cache key, preventing garbage collection for the process lifetime. Also change the get_curie_ids IN-clause from `IN $1` to `IN (SELECT unnest($1::VARCHAR[]))`, consistent with the pattern already used in the recursive query and guaranteed to work across DuckDB versions. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/babel_xrefs.py | 13 +++++++++---- tests/test_babel_xrefs.py | 2 -- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index dcf84ed..80bf1a1 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -8,7 +8,6 @@ import dataclasses import logging import duckdb -import functools from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.nodenorm import NodeNorm @@ -95,6 +94,7 @@ def __init__(self, downloader: BabelDownloader, nodenorm: NodeNorm = None): """ self.downloader = downloader self.nodenorm = nodenorm + self._xref_cache: dict = {} def _require_nodenorm(self): if self.nodenorm is None: @@ -118,7 +118,8 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: with duckdb.connect() as db: identifier_table = db.read_parquet(identifier_parquet) # noqa: F841 — DuckDB resolves 'identifier_table' by Python variable name in the SQL query result = db.execute( - "SELECT * FROM identifier_table WHERE curie IN $1", [curies] + "SELECT * FROM identifier_table WHERE curie IN (SELECT unnest($1::VARCHAR[]))", + [list(curies)], ) column_names = [desc[0] for desc in result.description] return [ @@ -126,11 +127,10 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: for row in result.fetchall() ] - @functools.lru_cache(maxsize=None) def get_curie_xref(self, curie: str, label_curies: bool = False): """Return all cross-references in Concord.parquet where *curie* is the subject or object. - Results are LRU-cached per ``(curie, label_curies)`` pair. + Results are cached per ``(curie, label_curies)`` pair on this instance. :param curie: The CURIE to look up. :param label_curies: If ``True``, annotate each result with NodeNorm labels and @@ -138,6 +138,10 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): :raises ValueError: If ``label_curies=True`` but no NodeNorm instance is available. :return: A list of ``CrossReference`` (or ``LabeledCrossReference``) objects. """ + cache_key = (curie, label_curies) + if cache_key in self._xref_cache: + return self._xref_cache[cache_key] + if label_curies: self._require_nodenorm() @@ -153,6 +157,7 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): xrefs = [CrossReference.from_tuple(rec) for rec in xref_tuples] if label_curies: xrefs = [self._to_labeled_xref(xref) for xref in xrefs] + self._xref_cache[cache_key] = xrefs return xrefs def _to_labeled_xref(self, xref: CrossReference) -> LabeledCrossReference: diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 114d269..f4304f7 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -230,7 +230,6 @@ def test_get_curie_xref_calls_downloader(self, tmp_path): "babel_explorer.core.babel_xrefs.duckdb.connect", return_value=mock_db, ): - bx.get_curie_xref.cache_clear() result = bx.get_curie_xref("A:1") # Downloader should be called for Concord only (Metadata unused here) assert mock_dl.call_count == 1 @@ -242,7 +241,6 @@ def test_get_curie_xrefs_no_expand(self, tmp_path): bx = self._make_bx(tmp_path) xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") with patch.object(bx, "get_curie_xref", return_value=[xr]): - bx.get_curie_xref.cache_clear() result = bx.get_curie_xrefs(["A:1"], recurse=False) assert len(result) == 1 assert result[0] == xr From f1a85322b9ee883bc776bf3e0a73e7f29a09d692 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 19:57:09 -0400 Subject: [PATCH 071/123] Fix urljoin silently stripping path segments in BabelDownloader urllib.parse.urljoin resolves relative paths against the base URL's last path segment, not the full path, so a multi-level url_base like https://host/a/b/ + 'duckdb/file' would produce https://host/a/duckdb/file. Replace with simple string concatenation (url_base + dirpath), which is safe and unambiguous because the constructor already enforces a trailing /. Also drop the now-unused urllib.parse import. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/downloader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index aa073ee..fd7f6e8 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -4,7 +4,6 @@ import json import os import tempfile -import urllib.parse import time import requests from datetime import datetime, timezone @@ -299,7 +298,7 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): local_path_to_download_to = os.path.join(self.local_path, dirpath) os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) - url_to_download = urllib.parse.urljoin(self.url_base, dirpath) + url_to_download = self.url_base + dirpath if os.path.exists(local_path_to_download_to): meta = self._load_meta(local_path_to_download_to) From e899a21bd78cfe5f23d6a3a22d26a4401729daa7 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 19:57:43 -0400 Subject: [PATCH 072/123] Delete unintentional src/__init__.py Making src/ itself a Python package can interfere with import resolution when the project is installed via hatchling (which uses src layout). Only src/babel_explorer/__init__.py and src/babel_explorer/core/__init__.py are needed. Co-Authored-By: Claude Sonnet 4.6 --- src/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/__init__.py diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29..0000000 From c63c47acbe2c918ebf77d35f0d6b85161f3e01ac Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 19:57:55 -0400 Subject: [PATCH 073/123] =?UTF-8?q?Housekeeping:=20trailing=20newline,=20e?= =?UTF-8?q?mpty=20test=20init,=20FUTURE.md=20=E2=86=92=20issues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README.md: add missing trailing newline - tests/__init__.py: remove stray comment (test __init__ files should be empty) - FUTURE.md: replace duplicated text with links to GitHub issues #12 and #13 where the two known performance improvements are now tracked Co-Authored-By: Claude Sonnet 4.6 --- FUTURE.md | 15 +++------------ README.md | 2 +- tests/__init__.py | 1 - 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/FUTURE.md b/FUTURE.md index b49144c..d6612f7 100644 --- a/FUTURE.md +++ b/FUTURE.md @@ -1,15 +1,6 @@ # Future Work -## Batch NodeNorm lookups +These items are tracked as GitHub issues: -`normalize_curie` makes one HTTP round-trip per CURIE. NodeNorm's -`/get_normalized_nodes` endpoint accepts repeated `curie=` parameters in one -request. Adding a `normalize_curies(curies)` batch method that pre-warms the -`normalize_curie` cache would collapse N serial round-trips into one when -`--labels` is set. - -## Reuse a single DuckDB connection per BabelXRefs instance - -`get_curie_xref` opens a fresh DuckDB connection and re-reads Concord.parquet -for each unique CURIE. Opening one connection at first use and reusing it across -all queries would eliminate the per-CURIE open/scan overhead. +- [#12](https://github.com/TranslatorSRI/babel-explorer/issues/12) — Batch NodeNorm lookups to reduce N round-trips when `--labels` is set +- [#13](https://github.com/TranslatorSRI/babel-explorer/issues/13) — Reuse a single DuckDB connection per `BabelXRefs` instance diff --git a/README.md b/README.md index 00fff8c..6d2d934 100644 --- a/README.md +++ b/README.md @@ -53,4 +53,4 @@ Integration tests are parametrized over the CURIEs listed in `tests/data/valid_c # tests/data/valid_curies.txt MONDO:0004979 HP:0000001 -``` \ No newline at end of file +``` diff --git a/tests/__init__.py b/tests/__init__.py index 588fec0..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ -# Tests for babel-explorer From dda05492bb9acc39095035d4e4678fe45429550e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 19:58:09 -0400 Subject: [PATCH 074/123] Apply ruff formatting to cli, formatting, and test files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No logic changes — ruff wrapped long import lists and long lines to meet the project's line-length limit. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/cli.py | 17 +++++++++++-- src/babel_explorer/formatting.py | 4 ++- tests/test_cli.py | 43 ++++++++++++++++++++++++-------- tests/test_formatting.py | 35 ++++++++++++++++++++------ 4 files changed, 78 insertions(+), 21 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 2d75248..6e23f82 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -1,11 +1,17 @@ """Command-line interface for babel-explorer.""" + import click import logging from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.babel_xrefs import BabelXRefs from babel_explorer.core.nodenorm import NodeNorm from babel_explorer.core.babel_xrefs import LabeledCrossReference -from babel_explorer.formatting import write_records, _record_to_dict, make_console, hl_curie +from babel_explorer.formatting import ( + write_records, + _record_to_dict, + make_console, + hl_curie, +) from rich.markup import escape @@ -166,7 +172,14 @@ def xrefs( @click.argument("curies", type=str, required=True, nargs=-1) @babel_options @format_option -def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str, fmt: str, json_indent: int): +def ids( + curies: list[str], + babel_url: str, + local_dir: str, + check_download: str, + fmt: str, + json_indent: int, +): """ Fetches and prints the ID records for the given CURIEs, along with Biolink type if provided. diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index 191f1cd..84a44a2 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -32,7 +32,9 @@ def _record_to_dict(record) -> dict[str, Any]: def _flatten_for_tabular(row: dict) -> dict: """Convert list/tuple fields to pipe-joined strings for TSV/CSV output.""" - return {k: "|".join(v) if isinstance(v, (list, tuple)) else v for k, v in row.items()} + return { + k: "|".join(v) if isinstance(v, (list, tuple)) else v for k, v in row.items() + } def make_console(file=None) -> Console: diff --git a/tests/test_cli.py b/tests/test_cli.py index f4f6dbb..c4b9e95 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -210,11 +210,18 @@ class TestOutputFormats: """Tests for --format option on all commands.""" # Shared real dataclass instances (no mocking needed for formatting logic) - _xref = CrossReference(filename="Concord.parquet", subj="A:1", pred="skos:exactMatch", obj="B:2") - _id_record = IdentifierRecord(curie="A:1", extra_fields=(("type", "gene"), ("label", "Alpha"))) + _xref = CrossReference( + filename="Concord.parquet", subj="A:1", pred="skos:exactMatch", obj="B:2" + ) + _id_record = IdentifierRecord( + curie="A:1", extra_fields=(("type", "gene"), ("label", "Alpha")) + ) _identifier = Identifier( - curie="MONDO:0004979", label="asthma", - biolink_type=("biolink:Disease",), taxa=(), description=(), + curie="MONDO:0004979", + label="asthma", + biolink_type=("biolink:Disease",), + taxa=(), + description=(), ) # -- console format (default) -- @@ -252,8 +259,12 @@ def test_xrefs_console_shows_query_curie(self): def test_test_concord_console_format(self): runner = CliRunner() with patch("babel_explorer.cli.NodeNorm") as mock_nn: - mock_nn.return_value.get_clique_identifiers.return_value = [self._identifier] - result = runner.invoke(cli, ["test-concord", "MONDO:0004979", "--format", "console"]) + mock_nn.return_value.get_clique_identifiers.return_value = [ + self._identifier + ] + result = runner.invoke( + cli, ["test-concord", "MONDO:0004979", "--format", "console"] + ) assert result.exit_code == 0 assert "MONDO:0004979" in result.output @@ -270,7 +281,9 @@ def test_test_concord_console_no_label_shows_dash(self): with patch("babel_explorer.cli.NodeNorm") as mock_nn: mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] - result = runner.invoke(cli, ["test-concord", "MONDO:0004979", "--format", "console"]) + result = runner.invoke( + cli, ["test-concord", "MONDO:0004979", "--format", "console"] + ) assert result.exit_code == 0 assert "-" in result.output @@ -361,8 +374,12 @@ def test_ids_format_tsv_expands_extra_fields(self): def test_test_concord_format_json_includes_query_curie(self): runner = CliRunner() with patch("babel_explorer.cli.NodeNorm") as mock_nn: - mock_nn.return_value.get_clique_identifiers.return_value = [self._identifier] - result = runner.invoke(cli, ["test-concord", "MONDO:0004979", "--format", "json"]) + mock_nn.return_value.get_clique_identifiers.return_value = [ + self._identifier + ] + result = runner.invoke( + cli, ["test-concord", "MONDO:0004979", "--format", "json"] + ) assert result.exit_code == 0 data = json.loads(result.output) @@ -374,8 +391,12 @@ def test_test_concord_format_json_includes_query_curie(self): def test_test_concord_format_tsv(self): runner = CliRunner() with patch("babel_explorer.cli.NodeNorm") as mock_nn: - mock_nn.return_value.get_clique_identifiers.return_value = [self._identifier] - result = runner.invoke(cli, ["test-concord", "MONDO:0004979", "--format", "tsv"]) + mock_nn.return_value.get_clique_identifiers.return_value = [ + self._identifier + ] + result = runner.invoke( + cli, ["test-concord", "MONDO:0004979", "--format", "tsv"] + ) assert result.exit_code == 0 lines = result.output.splitlines() diff --git a/tests/test_formatting.py b/tests/test_formatting.py index 2d402db..643544c 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -8,9 +8,18 @@ import pytest from rich.console import Console -from babel_explorer.core.babel_xrefs import CrossReference, LabeledCrossReference, IdentifierRecord +from babel_explorer.core.babel_xrefs import ( + CrossReference, + LabeledCrossReference, + IdentifierRecord, +) from babel_explorer.core.nodenorm import Identifier -from babel_explorer.formatting import _record_to_dict, write_records, make_console, hl_curie +from babel_explorer.formatting import ( + _record_to_dict, + write_records, + make_console, + hl_curie, +) # --------------------------------------------------------------------------- @@ -20,7 +29,9 @@ @pytest.fixture def xref(): - return CrossReference(filename="Concord.parquet", subj="A:1", pred="skos:exactMatch", obj="B:2") + return CrossReference( + filename="Concord.parquet", subj="A:1", pred="skos:exactMatch", obj="B:2" + ) @pytest.fixture @@ -108,13 +119,24 @@ def test_hl_curie_highlighted_renders_with_color(self): class TestRecordToDict: def test_cross_reference(self, xref): d = _record_to_dict(xref) - assert d == {"filename": "Concord.parquet", "subj": "A:1", "pred": "skos:exactMatch", "obj": "B:2"} + assert d == { + "filename": "Concord.parquet", + "subj": "A:1", + "pred": "skos:exactMatch", + "obj": "B:2", + } def test_labeled_cross_reference_has_all_eight_fields(self, labeled_xref): d = _record_to_dict(labeled_xref) assert set(d.keys()) == { - "filename", "subj", "pred", "obj", - "subj_label", "subj_biolink_type", "obj_label", "obj_biolink_type", + "filename", + "subj", + "pred", + "obj", + "subj_label", + "subj_biolink_type", + "obj_label", + "obj_biolink_type", } # dataclasses.asdict() preserves tuple types assert d["subj_biolink_type"] == ("biolink:Disease",) @@ -151,7 +173,6 @@ def test_identifier_dataclass(self, identifier): class TestWriteRecords: - # -- json format -- def test_json_is_valid_list(self, xref): From 6576fa7fdca9587174b42fa596f0f03773c085d5 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 23:14:21 -0400 Subject: [PATCH 075/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 6e23f82..1c07d6f 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -28,7 +28,7 @@ def babel_options(f): f = click.option( "--babel-url", type=str, - default="https://stars.renci.org:443/var/babel/2025nov19/", + default="https://stars.renci.org/var/babel/2025nov19/", help="Base URL of the Babel server", )(f) f = click.option( From 8a49cad51fa050d99db1ae939ae5438a708a0a1a Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 23:14:56 -0400 Subject: [PATCH 076/123] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/babel_explorer/core/babel_xrefs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 80bf1a1..8d09256 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -23,10 +23,10 @@ class CrossReference: obj: str @staticmethod - def from_tuple(tuple: tuple[str, str, str, str]): + def from_tuple(row: tuple[str, str, str, str]): """Construct from a ``(filename, subj, pred, obj)`` database row tuple.""" return CrossReference( - filename=tuple[0], subj=tuple[1], pred=tuple[2], obj=tuple[3] + filename=row[0], subj=row[1], pred=row[2], obj=row[3] ) @property From d1badf4be0b6261e238d9f0679c61c5d3a26766a Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 23:52:59 -0400 Subject: [PATCH 077/123] Use read_parquet() inline in SQL instead of magic variable-name resolution Replaces the implicit DuckDB pattern (assign a relation to a Python variable, then reference that name in a SQL string) with explicit read_parquet($N) parameterised calls. The recursive query wraps the scan in a MATERIALIZED CTE so the parquet is read once regardless of how many CTEs reference it. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/babel_xrefs.py | 28 ++++++++++++-------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 8d09256..0f7a038 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -25,9 +25,7 @@ class CrossReference: @staticmethod def from_tuple(row: tuple[str, str, str, str]): """Construct from a ``(filename, subj, pred, obj)`` database row tuple.""" - return CrossReference( - filename=row[0], subj=row[1], pred=row[2], obj=row[3] - ) + return CrossReference(filename=row[0], subj=row[1], pred=row[2], obj=row[3]) @property def curies(self): @@ -116,10 +114,9 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: # Query the Parquet files using DuckDB (in-memory; nothing is persisted). with duckdb.connect() as db: - identifier_table = db.read_parquet(identifier_parquet) # noqa: F841 — DuckDB resolves 'identifier_table' by Python variable name in the SQL query result = db.execute( - "SELECT * FROM identifier_table WHERE curie IN (SELECT unnest($1::VARCHAR[]))", - [list(curies)], + "SELECT * FROM read_parquet($1) WHERE curie IN (SELECT unnest($2::VARCHAR[]))", + [identifier_parquet, list(curies)], ) column_names = [desc[0] for desc in result.description] return [ @@ -148,10 +145,9 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") with duckdb.connect() as db: - concord_table = db.read_parquet(concord_parquet) # noqa: F841 — DuckDB resolves 'concord_table' by Python variable name in the SQL query xref_tuples = db.execute( - "SELECT filename, subj, pred, obj FROM concord_table WHERE subj=$1 OR obj=$1", - [curie], + "SELECT filename, subj, pred, obj FROM read_parquet($1) WHERE subj=$2 OR obj=$2", + [concord_parquet, curie], ).fetchall() xrefs = [CrossReference.from_tuple(rec) for rec in xref_tuples] @@ -185,29 +181,31 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") with duckdb.connect() as db: - concord_table = db.read_parquet(concord_parquet) # noqa: F841 — DuckDB resolves 'concord_table' by Python variable name in the SQL query rows = db.execute( """ WITH RECURSIVE + concord AS MATERIALIZED ( + SELECT filename, subj, pred, obj FROM read_parquet($1) + ), edges(a, b) AS ( - SELECT subj, obj FROM concord_table + SELECT subj, obj FROM concord UNION ALL - SELECT obj, subj FROM concord_table + SELECT obj, subj FROM concord ), frontier(curie) AS ( - SELECT unnest($1::VARCHAR[]) + SELECT unnest($2::VARCHAR[]) UNION SELECT e.b FROM edges e INNER JOIN frontier f ON e.a = f.curie ) SELECT DISTINCT c.filename, c.subj, c.pred, c.obj - FROM concord_table c + FROM concord c WHERE c.subj IN (SELECT curie FROM frontier) OR c.obj IN (SELECT curie FROM frontier) ORDER BY c.filename, c.subj, c.obj, c.pred """, - [curies], + [concord_parquet, curies], ).fetchall() xrefs = [CrossReference.from_tuple(row) for row in rows] From 5ca6a2621fd37afdc9d071a588981b6ecbee1532 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 23:53:09 -0400 Subject: [PATCH 078/123] Remove lru_cache from get_downloaded_file; fix timeout comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lru_cache on get_downloaded_file caused a latent bug: once it cached a local path, subsequent calls bypassed the os.path.exists() guard — returning a stale path if the file was deleted mid-run. The three-tier freshness logic (meta + ETag) already prevents redundant network calls, so the cache adds no benefit and only introduces this risk. Also corrects the misleading "connection only" comment on the requests timeout: it is a per-read idle timeout, not a total-transfer limit. Tests updated to remove cache_clear() calls and rename the caching test to reflect that the freshness window is now the deduplication mechanism. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/downloader.py | 3 +-- tests/test_downloader.py | 13 +++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index fd7f6e8..6382ad8 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -239,7 +239,7 @@ def _download_with_retry(self, url, local_path, chunk_size): headers["Range"] = f"bytes={resume_byte_pos}-" self.logger.info(f"Resuming download from byte {resume_byte_pos}") - # timeout applies to connection only, not total transfer time + # timeout is per-read (seconds without receiving bytes), not a total time limit. with requests.get( url, headers=headers, stream=True, timeout=self.timeout ) as response: @@ -278,7 +278,6 @@ def _download_with_retry(self, url, local_path, chunk_size): f"Failed to download {url} after {self.retries} attempts: {e}" ) - @functools.lru_cache(maxsize=None) def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): """ Download a file from the Babel server to local storage with ETag-based caching. diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 7fe8609..db2e44b 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -317,7 +317,6 @@ def test_tier1_returns_immediately_no_http(self, tmp_path): with patch("babel_explorer.core.downloader.requests.head") as mock_head: with patch("babel_explorer.core.downloader.requests.get") as mock_get: - dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) mock_head.assert_not_called() mock_get.assert_not_called() @@ -346,7 +345,6 @@ def test_tier2_head_check_no_redownload(self, tmp_path): "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp ): with patch("babel_explorer.core.downloader.requests.get") as mock_get: - dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) mock_get.assert_not_called() assert result == str(local) @@ -371,7 +369,6 @@ def test_tier2_updates_last_checked_after_head(self, tmp_path): with patch( "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp ): - dl.get_downloaded_file.cache_clear() dl.get_downloaded_file(test_file) with open(str(local) + ".meta") as f: @@ -409,7 +406,6 @@ def fake_download(url, path, chunk_size): "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp ): with patch.object(dl, "_download_with_retry", side_effect=fake_download): - dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) assert open(result, "rb").read() == new_content @@ -431,7 +427,6 @@ def fake_download(url, path, chunk_size): with patch.object( dl, "_download_with_retry", side_effect=fake_download ) as mock_dl: - dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) mock_dl.assert_called_once() @@ -463,7 +458,6 @@ def fake_download(url, path, chunk_size): with patch.object( dl, "_download_with_retry", side_effect=fake_download ) as mock_dl: - dl.get_downloaded_file.cache_clear() result = dl.get_downloaded_file(test_file) mock_dl.assert_called_once() @@ -471,9 +465,9 @@ def fake_download(url, path, chunk_size): class TestGetDownloadedFileCaching: - """Tests for get_downloaded_file LRU caching.""" + """Tests that repeated calls within the freshness window avoid redundant downloads.""" - def test_cache_returns_same_result(self, tmp_path): + def test_second_call_within_freshness_skips_download(self, tmp_path): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) content = b"cached content" @@ -485,11 +479,10 @@ def fake_download(url, path, chunk_size): with patch.object( dl, "_download_with_retry", side_effect=fake_download ) as mock_dl: - dl.get_downloaded_file.cache_clear() r1 = dl.get_downloaded_file("cached.txt") r2 = dl.get_downloaded_file("cached.txt") assert r1 == r2 - mock_dl.assert_called_once() # only one actual download + mock_dl.assert_called_once() # freshness window prevents second download class TestDownloadWithRetry: From aa1612e7762f98f0a1473a193dcee44d59604288 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 17 May 2026 23:58:05 -0400 Subject: [PATCH 079/123] Replace lru_cache with instance-level dict caches in NodeNorm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #14. lru_cache on instance methods holds a reference to self in the cache key, preventing garbage collection of NodeNorm instances for the lifetime of the process. Each of the three methods now checks and populates a dedicated dict (_normalize_cache, _identifier_cache, _clique_cache) on the instance. This makes caching scope explicit: the cache lives and dies with the object, and callers who need fresh results simply instantiate a new NodeNorm. HTTP errors in normalize_curie are intentionally not cached so a transient failure does not permanently suppress retries. Tests updated to remove cache_clear() calls — unit tests already construct a fresh NodeNorm per test case via _make_nn(), and integration tests are parametrized per-CURIE so cached results do not interfere. Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/core/nodenorm.py | 72 +++++++++++++++++++++-------- tests/test_nodenorm.py | 25 ++-------- 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index c98c085..bfb0c27 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -1,7 +1,6 @@ """NodeNorm API client for identifier normalisation and label enrichment.""" import dataclasses -import functools import requests import logging @@ -37,7 +36,11 @@ def _to_tuple(val) -> tuple[str, ...]: class NodeNorm: - """Client for the NodeNormalization API (https://nodenormalization-sri.renci.org/).""" + """Client for the NodeNormalization API (https://nodenormalization-sri.renci.org/). + + Results are cached per instance. To get uncached results, instantiate a new + NodeNorm object. + """ def __init__(self, nodenorm_url: str = "", timeout: int = 30): """ @@ -49,8 +52,10 @@ def __init__(self, nodenorm_url: str = "", timeout: int = 30): self.timeout = timeout if self.nodenorm_url and not self.nodenorm_url.endswith("/"): self.nodenorm_url += "/" + self._normalize_cache: dict[str, dict | None] = {} + self._identifier_cache: dict[str, Identifier] = {} + self._clique_cache: dict[str, list[Identifier]] = {} - @functools.lru_cache(maxsize=None) def get_identifier(self, curie: str) -> "Identifier": """Return the ``Identifier`` for *curie* by looking it up in its NodeNorm clique. @@ -58,32 +63,46 @@ def get_identifier(self, curie: str) -> "Identifier": *curie* exactly. Falls back to a bare ``Identifier(curie=curie)`` (empty label and type) if NodeNorm does not recognise the CURIE or it is not listed in the clique. - Results are LRU-cached so repeated calls for the same CURIE are free. + Results are cached per instance. """ + if curie in self._identifier_cache: + return self._identifier_cache[curie] + result = self.normalize_curie(curie) logging.debug(f"Normalizing {curie} with NodeNorm to result: {result}") if not result: - return Identifier(curie=curie) - for identifier in result.get("equivalent_identifiers", []): - if identifier["identifier"] == curie: - logging.debug(f"Found exact match for {curie}: {identifier}") - return Identifier.from_dict(identifier) - - logging.debug( - f"No exact match for {curie!r} in equivalent_identifiers; returning bare Identifier" - ) - return Identifier(curie=curie) + ident = Identifier(curie=curie) + else: + for identifier in result.get("equivalent_identifiers", []): + if identifier["identifier"] == curie: + logging.debug(f"Found exact match for {curie}: {identifier}") + ident = Identifier.from_dict(identifier) + break + else: + logging.debug( + f"No exact match for {curie!r} in equivalent_identifiers; returning bare Identifier" + ) + ident = Identifier(curie=curie) + + self._identifier_cache[curie] = ident + return ident - @functools.lru_cache(maxsize=None) def normalize_curie(self, curie: str): """Call ``get_normalized_nodes`` and return the per-CURIE result dict. :return: The normalisation dict for *curie* (contains ``id``, ``equivalent_identifiers``, ``type``, etc.), or ``None`` if the CURIE is not recognised by NodeNorm. :raises requests.HTTPError: If the API returns a non-2xx status code. + + Results are cached per instance. HTTP errors are not cached. """ + if curie in self._normalize_cache: + return self._normalize_cache[curie] + if not self.nodenorm_url: + self._normalize_cache[curie] = None return None + response = requests.get( f"{self.nodenorm_url}get_normalized_nodes", params={ @@ -100,21 +119,34 @@ def normalize_curie(self, curie: str): result = response.json() try: - return result[curie] + value = result[curie] except KeyError: logging.debug( f"NodeNorm response did not contain CURIE {curie!r}; returning None" ) - return None + value = None + + self._normalize_cache[curie] = value + return value - @functools.lru_cache(maxsize=None) def get_clique_identifiers(self, curie: str) -> list[Identifier]: """Return all ``Identifier`` objects in the NodeNorm clique for *curie*. :return: A list of ``Identifier`` objects (one per entry in ``equivalent_identifiers``), or an empty list if the CURIE is unknown or has no equivalents. + + Results are cached per instance. """ + if curie in self._clique_cache: + return self._clique_cache[curie] + result = self.normalize_curie(curie) if not result or "equivalent_identifiers" not in result: - return [] - return [Identifier.from_dict(x) for x in result["equivalent_identifiers"]] + identifiers = [] + else: + identifiers = [ + Identifier.from_dict(x) for x in result["equivalent_identifiers"] + ] + + self._clique_cache[curie] = identifiers + return identifiers diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index 57b6dab..b0bc921 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -143,7 +143,6 @@ def test_custom_url(self): def test_empty_url_normalize_curie_returns_none_without_network(self): """NodeNorm('') must not make any HTTP calls and must return None.""" nn = NodeNorm("") - nn.normalize_curie.cache_clear() with patch("babel_explorer.core.nodenorm.requests.get") as mock_get: result = nn.normalize_curie("MONDO:0004979") mock_get.assert_not_called() @@ -154,9 +153,7 @@ class TestNormalizeCurieMocked: """Unit tests for NodeNorm.normalize_curie() with mocked HTTP responses.""" def _make_nn(self): - nn = NodeNorm(nodenorm_url="https://example.com/") - nn.normalize_curie.cache_clear() - return nn + return NodeNorm(nodenorm_url="https://example.com/") def test_correct_api_endpoint_and_params(self): nn = self._make_nn() @@ -185,7 +182,7 @@ def test_returns_result_for_curie(self): result = nn.normalize_curie("X:1") assert result == expected - def test_lru_caching(self): + def test_caching(self): nn = self._make_nn() mock_resp = Mock() mock_resp.json.return_value = {"X:1": {"id": "X:1"}} @@ -212,10 +209,7 @@ class TestGetIdentifierMocked: """Unit tests for NodeNorm.get_identifier() with mocked normalize_curie.""" def _make_nn(self): - nn = NodeNorm(nodenorm_url="https://example.com/") - nn.normalize_curie.cache_clear() - nn.get_identifier.cache_clear() - return nn + return NodeNorm(nodenorm_url="https://example.com/") def test_exact_match_found(self): nn = self._make_nn() @@ -266,10 +260,7 @@ class TestGetCliqueIdentifiersMocked: """Unit tests for NodeNorm.get_clique_identifiers() with mocked normalize_curie.""" def _make_nn(self): - nn = NodeNorm(nodenorm_url="https://example.com/") - nn.normalize_curie.cache_clear() - nn.get_clique_identifiers.cache_clear() - return nn + return NodeNorm(nodenorm_url="https://example.com/") def test_success_returns_list(self): nn = self._make_nn() @@ -311,7 +302,6 @@ def test_caching(self): @pytest.mark.parametrize("curie", VALID_CURIES) def test_normalize_curie_real_api(nodenorm, curie): """normalize_curie returns a dict with expected keys.""" - nodenorm.normalize_curie.cache_clear() result = nodenorm.normalize_curie(curie) assert isinstance(result, dict) assert "id" in result @@ -323,8 +313,6 @@ def test_normalize_curie_real_api(nodenorm, curie): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_identifier_real_api(nodenorm, curie): """get_identifier returns an Identifier with non-empty label and biolink_type.""" - nodenorm.normalize_curie.cache_clear() - nodenorm.get_identifier.cache_clear() ident = nodenorm.get_identifier(curie) assert isinstance(ident, Identifier) assert ident.curie == curie @@ -335,8 +323,6 @@ def test_get_identifier_real_api(nodenorm, curie): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_clique_identifiers_real_api(nodenorm, curie): """get_clique_identifiers returns a non-empty list of Identifiers.""" - nodenorm.normalize_curie.cache_clear() - nodenorm.get_clique_identifiers.cache_clear() result = nodenorm.get_clique_identifiers(curie) assert result is not None assert len(result) > 0 @@ -347,8 +333,6 @@ def test_get_clique_identifiers_real_api(nodenorm, curie): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_clique_identifiers_has_known_ids(nodenorm, curie): """At least one equivalent identifier is returned.""" - nodenorm.normalize_curie.cache_clear() - nodenorm.get_clique_identifiers.cache_clear() result = nodenorm.get_clique_identifiers(curie) assert len(result) >= 1 @@ -356,6 +340,5 @@ def test_get_clique_identifiers_has_known_ids(nodenorm, curie): @pytest.mark.integration def test_normalize_curie_nonexistent(nodenorm): """A made-up CURIE returns None.""" - nodenorm.normalize_curie.cache_clear() result = nodenorm.normalize_curie("FAKENS:9999999999") assert result is None From 7f0dfb3740c7c5703caf491abd1f072b1fd89242 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sat, 30 May 2026 19:15:10 -0400 Subject: [PATCH 080/123] Add --paths flag and depth-based coloring for recursive xrefs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --paths finds and displays the shortest path between each pair of query CURIEs (implies --recurse), making indirect clique connections immediately readable. --recurse output now colors each CURIE by BFS distance from the nearest query term (bold cyan → bold yellow → yellow → green → dim). Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/cli.py | 140 ++++++++++++++++++++++--- src/babel_explorer/core/babel_xrefs.py | 51 +++++++++ src/babel_explorer/formatting.py | 23 ++++ 3 files changed, 198 insertions(+), 16 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 1c07d6f..c5a526f 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -2,8 +2,10 @@ import click import logging +from itertools import combinations + from babel_explorer.core.downloader import BabelDownloader -from babel_explorer.core.babel_xrefs import BabelXRefs +from babel_explorer.core.babel_xrefs import BabelXRefs, build_depth_map, find_shortest_path from babel_explorer.core.nodenorm import NodeNorm from babel_explorer.core.babel_xrefs import LabeledCrossReference from babel_explorer.formatting import ( @@ -11,6 +13,7 @@ _record_to_dict, make_console, hl_curie, + hl_curie_at_depth, ) from rich.markup import escape @@ -100,6 +103,95 @@ def parse_duration(value: str) -> int | float: return result +def _curie_str(curie: str, is_query: bool, depth: int | None, label: str | None) -> str: + """Build a Rich-marked-up CURIE string with optional label.""" + if is_query: + s = hl_curie(curie, True) + else: + s = hl_curie_at_depth(curie, depth) + if label: + s += f" ({escape(label)})" + return s + + +def _print_paths(console, curies, xrefs_list, labels: bool) -> None: + """Print the shortest path between every pair of query CURIEs.""" + curie_list = list(curies) + if len(curie_list) < 2: + console.print("[yellow]--paths requires at least two CURIEs.[/yellow]") + return + + query_set = set(curie_list) + + for from_c, to_c in combinations(curie_list, 2): + path = find_shortest_path(from_c, to_c, xrefs_list) + header_from = hl_curie(from_c, True) + header_to = hl_curie(to_c, True) + + if path is None: + console.print( + f"[bold]Path:[/bold] {header_from} [dim]→[/dim] {header_to}" + f" [red]no path found[/red]" + ) + console.print() + continue + + if len(path) == 0: + console.print( + f"[bold]Path:[/bold] {header_from} [dim]=[/dim] {header_to}" + f" [dim](same node)[/dim]" + ) + console.print() + continue + + # Reconstruct ordered node list from the edge sequence. + nodes = [from_c] + for edge in path: + prev = nodes[-1] + nodes.append(edge.obj if edge.subj == prev else edge.subj) + + # Header: node1 → node2 → … → nodeN + node_strs = [] + for i, node in enumerate(nodes): + if node in query_set: + node_strs.append(hl_curie(node, True)) + else: + depth = i # position along path == depth from from_c + node_strs.append(hl_curie_at_depth(node, depth)) + n_steps = len(path) + step_word = "step" if n_steps == 1 else "steps" + console.print( + f"[bold]Path ({n_steps} {step_word}):[/bold] " + + f" [dim]→[/dim] ".join(node_strs) + ) + + # Edge details, indented, oriented in traversal direction. + for i, edge in enumerate(path): + from_node = nodes[i] + to_node = nodes[i + 1] + subj_node = edge.subj if edge.subj == from_node else edge.obj + obj_node = edge.obj if edge.subj == from_node else edge.subj + + subj_label = obj_label = None + if labels and isinstance(edge, LabeledCrossReference): + if edge.subj == subj_node: + subj_label = edge.subj_label + obj_label = edge.obj_label + else: + subj_label = edge.obj_label + obj_label = edge.subj_label + + subj_str = _curie_str(subj_node, subj_node in query_set, i, subj_label) + obj_str = _curie_str(obj_node, obj_node in query_set, i + 1, obj_label) + + console.print( + f" {subj_str} [dim]{escape(edge.pred)}[/dim] " + f"{obj_str} [dim italic]{escape(edge.filename)}[/dim italic]" + ) + + console.print() + + @click.group() def cli(): """babel-explorer: query and explore Babel intermediate files.""" @@ -117,6 +209,11 @@ def cli(): ) @click.option("--recurse", is_flag=True, help="Recursively query returned xrefs") @click.option("--labels", is_flag=True, help="Include labels for CURIEs") +@click.option( + "--paths", + is_flag=True, + help="Show shortest path(s) connecting the given CURIEs (implies --recurse)", +) @format_option def xrefs( curies: list[str], @@ -125,6 +222,7 @@ def xrefs( local_dir: str, recurse: bool, labels: bool, + paths: bool, check_download: str, fmt: str, json_indent: int, @@ -147,25 +245,35 @@ def xrefs( BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness), NodeNorm(nodenorm_url), ) - xrefs = bxref.get_curie_xrefs(curies, recurse, label_curies=labels) + if paths: + recurse = True + xref_list = bxref.get_curie_xrefs(curies, recurse, label_curies=labels) if fmt == "console": console = make_console() - query_set = set(curies) - for xref in xrefs: - subj_str = hl_curie(xref.subj, xref.subj in query_set) - obj_str = hl_curie(xref.obj, xref.obj in query_set) - if isinstance(xref, LabeledCrossReference): - if xref.subj_label: - subj_str += f" ({escape(xref.subj_label)})" - if xref.obj_label: - obj_str += f" ({escape(xref.obj_label)})" - console.print( - f"{subj_str} [dim]{escape(xref.pred)}[/dim] " - f"{obj_str} [dim italic]{escape(xref.filename)}[/dim italic]" - ) + if paths: + _print_paths(console, curies, xref_list, labels) + else: + query_set = set(curies) + depth_map = build_depth_map(list(curies), xref_list) if recurse else None + for xref in xref_list: + if depth_map is not None: + subj_str = hl_curie_at_depth(xref.subj, depth_map.get(xref.subj)) + obj_str = hl_curie_at_depth(xref.obj, depth_map.get(xref.obj)) + else: + subj_str = hl_curie(xref.subj, xref.subj in query_set) + obj_str = hl_curie(xref.obj, xref.obj in query_set) + if isinstance(xref, LabeledCrossReference): + if xref.subj_label: + subj_str += f" ({escape(xref.subj_label)})" + if xref.obj_label: + obj_str += f" ({escape(xref.obj_label)})" + console.print( + f"{subj_str} [dim]{escape(xref.pred)}[/dim] " + f"{obj_str} [dim italic]{escape(xref.filename)}[/dim italic]" + ) else: - write_records(xrefs, fmt=fmt, indent=json_indent) + write_records(xref_list, fmt=fmt, indent=json_indent) @cli.command("ids") diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 0f7a038..b200010 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -7,6 +7,7 @@ import dataclasses import logging +from collections import deque import duckdb from babel_explorer.core.downloader import BabelDownloader @@ -75,6 +76,56 @@ def __str__(self): return f"IdentifierRecord({', '.join(parts)})" +def build_depth_map(query_curies: list[str], xrefs: list) -> dict[str, int]: + """BFS from query_curies over xref edges; returns {curie: depth_from_nearest_query}.""" + adj: dict[str, list[str]] = {} + for xref in xrefs: + adj.setdefault(xref.subj, []).append(xref.obj) + adj.setdefault(xref.obj, []).append(xref.subj) + + depths: dict[str, int] = {c: 0 for c in query_curies} + frontier = list(query_curies) + while frontier: + next_frontier = [] + for node in frontier: + for neighbor in adj.get(node, []): + if neighbor not in depths: + depths[neighbor] = depths[node] + 1 + next_frontier.append(neighbor) + frontier = next_frontier + return depths + + +def find_shortest_path( + from_curie: str, to_curie: str, xrefs: list +) -> list | None: + """Return the shortest list of CrossReference edges from from_curie to to_curie. + + Returns ``[]`` if from_curie == to_curie, or ``None`` if no path exists. + The returned edges may be stored in either direction; callers should check + ``edge.subj`` / ``edge.obj`` against the expected traversal direction. + """ + if from_curie == to_curie: + return [] + + adj: dict[str, list] = {} + for xref in xrefs: + adj.setdefault(xref.subj, []).append((xref.obj, xref)) + adj.setdefault(xref.obj, []).append((xref.subj, xref)) + + visited = {from_curie} + queue: deque = deque([(from_curie, [])]) + while queue: + current, path = queue.popleft() + for neighbor, xref in adj.get(current, []): + if neighbor == to_curie: + return path + [xref] + if neighbor not in visited: + visited.add(neighbor) + queue.append((neighbor, path + [xref])) + return None + + class BabelXRefs: """Query engine for Babel cross-reference and identifier Parquet files. diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index 84a44a2..5ff69ae 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -52,6 +52,29 @@ def hl_curie(curie: str, highlight: bool) -> str: return f"[bold cyan]{escaped}[/bold cyan]" if highlight else escaped +# Styles indexed by BFS depth from the nearest query CURIE. +# Depth 0 = the query term itself; higher = further away. +_DEPTH_STYLES = [ + "bold cyan", # 0: query CURIE + "bold yellow", # 1: one hop away + "yellow", # 2: two hops + "green", # 3: three hops + "dim", # 4+: further +] + + +def hl_curie_at_depth(curie: str, depth: int | None) -> str: + """Return rich markup for a CURIE colored by its BFS depth from the nearest query CURIE. + + Pass ``depth=None`` for CURIEs whose depth is unknown (rendered unstyled). + """ + escaped = escape(curie) + if depth is None: + return escaped + style = _DEPTH_STYLES[min(depth, len(_DEPTH_STYLES) - 1)] + return f"[{style}]{escaped}[/{style}]" + + def write_records(records, fmt: str, indent: int = 2, file=None): """Write an iterable of dataclass records (or dicts) in the requested format. From ef3cc2fd2d58599f9e76f354e049b181be3b2156 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sat, 30 May 2026 19:21:18 -0400 Subject: [PATCH 081/123] Prefix path edge lines with "- " for GitHub markdown list paste Co-Authored-By: Claude Sonnet 4.6 --- src/babel_explorer/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index c5a526f..47e62ee 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -185,7 +185,7 @@ def _print_paths(console, curies, xrefs_list, labels: bool) -> None: obj_str = _curie_str(obj_node, obj_node in query_set, i + 1, obj_label) console.print( - f" {subj_str} [dim]{escape(edge.pred)}[/dim] " + f" - {subj_str} [dim]{escape(edge.pred)}[/dim] " f"{obj_str} [dim italic]{escape(edge.filename)}[/dim italic]" ) From 87eca6e1d5085efcc5a13fa3dd9ed05af01fad62 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sat, 30 May 2026 19:34:05 -0400 Subject: [PATCH 082/123] Use double-quoted labels in all console output (CURIE "label") Replaces the previous parenthesis format so labels are easy to parse by downstream tools. Embedded backslashes and double quotes are escaped (\\ and \"). Documents the convention in CLAUDE.md and README.md. Co-Authored-By: Claude Sonnet 4.6 --- CLAUDE.md | 16 ++++++++++++++++ README.md | 2 ++ src/babel_explorer/cli.py | 13 +++++++++---- tests/test_cli.py | 2 +- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cc0a558..2fe0b8d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,6 +67,22 @@ uv run ruff check uv run ruff format ``` +## Console Output Format Conventions + +### Label display + +When a human-readable label is shown alongside a CURIE in console output, it always appears **immediately after the CURIE, in double quotes**: + +``` +MONDO:0004979 "asthma" skos:exactMatch EFO:0000270 "asthma" +``` + +This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels`, and `test-concord`. + +**Escaping:** embedded backslashes are escaped as `\\` and embedded double quotes as `\"`. Downstream tools can parse labels with the regex `"([^"\\]|\\.)*"`. + +**Do not** use parentheses `(label)` or any other delimiter — double quotes are the sole convention. + ## Architecture ### Core Components diff --git a/README.md b/README.md index 6d2d934..96f7a38 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ uv run babel-explorer xrefs MONDO:0004979 --recurse # Get cross-references with labels from NodeNorm uv run babel-explorer xrefs MONDO:0004979 --labels +# Labels appear in double quotes immediately after the CURIE: +# MONDO:0004979 "asthma" skos:exactMatch EFO:0000270 "asthma" # Get ID records for CURIEs uv run babel-explorer ids MONDO:0004979 diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 47e62ee..e4acfa7 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -103,6 +103,11 @@ def parse_duration(value: str) -> int | float: return result +def _fmt_label(label: str) -> str: + """Escape a label for double-quoted display: backslashes first, then double quotes.""" + return escape(label.replace("\\", "\\\\").replace('"', '\\"')) + + def _curie_str(curie: str, is_query: bool, depth: int | None, label: str | None) -> str: """Build a Rich-marked-up CURIE string with optional label.""" if is_query: @@ -110,7 +115,7 @@ def _curie_str(curie: str, is_query: bool, depth: int | None, label: str | None) else: s = hl_curie_at_depth(curie, depth) if label: - s += f" ({escape(label)})" + s += f' "{_fmt_label(label)}"' return s @@ -265,9 +270,9 @@ def xrefs( obj_str = hl_curie(xref.obj, xref.obj in query_set) if isinstance(xref, LabeledCrossReference): if xref.subj_label: - subj_str += f" ({escape(xref.subj_label)})" + subj_str += f' "{_fmt_label(xref.subj_label)}"' if xref.obj_label: - obj_str += f" ({escape(xref.obj_label)})" + obj_str += f' "{_fmt_label(xref.obj_label)}"' console.print( f"{subj_str} [dim]{escape(xref.pred)}[/dim] " f"{obj_str} [dim italic]{escape(xref.filename)}[/dim italic]" @@ -340,7 +345,7 @@ def test_concord(curies, nodenorm_url, fmt, json_indent): console.print( f"{hl_curie(curie, True)} " f"{hl_curie(ident.curie, ident.curie in query_set)} " - f"{escape(ident.label or '-')} " + f'"{_fmt_label(ident.label or "")}" ' f"[dim]{escape(biolink)}[/dim]" ) else: diff --git a/tests/test_cli.py b/tests/test_cli.py index c4b9e95..c1a8965 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -286,7 +286,7 @@ def test_test_concord_console_no_label_shows_dash(self): ) assert result.exit_code == 0 - assert "-" in result.output + assert '""' in result.output # -- json format -- From ac04249ac1bdf59652495e0b68b6b1e3b7b7f84e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Sun, 31 May 2026 19:34:36 -0400 Subject: [PATCH 083/123] Omit label entirely when absent instead of showing empty placeholder Consistent with xrefs --labels behaviour: a missing label produces no output, not "" or "-". Documents the rule in CLAUDE.md. Co-Authored-By: Claude Sonnet 4.6 --- CLAUDE.md | 2 ++ src/babel_explorer/cli.py | 4 ++-- tests/test_cli.py | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2fe0b8d..159ea56 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,6 +79,8 @@ MONDO:0004979 "asthma" skos:exactMatch EFO:0000270 "asthma" This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels`, and `test-concord`. +**When a label is absent, omit it entirely** — do not substitute a placeholder like `-` or `""`. A CURIE with no label renders as just the bare CURIE. + **Escaping:** embedded backslashes are escaped as `\\` and embedded double quotes as `\"`. Downstream tools can parse labels with the regex `"([^"\\]|\\.)*"`. **Do not** use parentheses `(label)` or any other delimiter — double quotes are the sole convention. diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index e4acfa7..8c6b7e4 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -342,10 +342,10 @@ def test_concord(curies, nodenorm_url, fmt, json_indent): for curie in curies: for ident in nodenorm.get_clique_identifiers(curie): biolink = ", ".join(ident.biolink_type) + label_str = f' "{_fmt_label(ident.label)}"' if ident.label else "" console.print( f"{hl_curie(curie, True)} " - f"{hl_curie(ident.curie, ident.curie in query_set)} " - f'"{_fmt_label(ident.label or "")}" ' + f"{hl_curie(ident.curie, ident.curie in query_set)}{label_str} " f"[dim]{escape(biolink)}[/dim]" ) else: diff --git a/tests/test_cli.py b/tests/test_cli.py index c1a8965..9221020 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -271,8 +271,8 @@ def test_test_concord_console_format(self): assert "asthma" in result.output assert "biolink:Disease" in result.output - def test_test_concord_console_no_label_shows_dash(self): - """Identifiers with no label display '-' in console format.""" + def test_test_concord_console_no_label_omits_label(self): + """Identifiers with no label omit the label field entirely in console format.""" runner = CliRunner() mock_ident = MagicMock() mock_ident.curie = "MONDO:0004979" @@ -286,7 +286,7 @@ def test_test_concord_console_no_label_shows_dash(self): ) assert result.exit_code == 0 - assert '""' in result.output + assert '"' not in result.output # -- json format -- From 21770b851eeb90b32fcf903fb6a8790088ef11ef Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:19:29 -0400 Subject: [PATCH 084/123] Apply ruff formatting to babel_xrefs and formatting CI runs `ruff format --check src/ tests/`, but these two files had drifted out of ruff-formatted shape. No behaviour change. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/core/babel_xrefs.py | 4 +--- src/babel_explorer/formatting.py | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index b200010..9f6b4d0 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -96,9 +96,7 @@ def build_depth_map(query_curies: list[str], xrefs: list) -> dict[str, int]: return depths -def find_shortest_path( - from_curie: str, to_curie: str, xrefs: list -) -> list | None: +def find_shortest_path(from_curie: str, to_curie: str, xrefs: list) -> list | None: """Return the shortest list of CrossReference edges from from_curie to to_curie. Returns ``[]`` if from_curie == to_curie, or ``None`` if no path exists. diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index 5ff69ae..1f7716c 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -55,11 +55,11 @@ def hl_curie(curie: str, highlight: bool) -> str: # Styles indexed by BFS depth from the nearest query CURIE. # Depth 0 = the query term itself; higher = further away. _DEPTH_STYLES = [ - "bold cyan", # 0: query CURIE + "bold cyan", # 0: query CURIE "bold yellow", # 1: one hop away - "yellow", # 2: two hops - "green", # 3: three hops - "dim", # 4+: further + "yellow", # 2: two hops + "green", # 3: three hops + "dim", # 4+: further ] From 1b48c7a0761ab414d74537c9b8aaf7bf5cd0fcb4 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:19:38 -0400 Subject: [PATCH 085/123] Detect the Babel release behind a server URL and pin the cache to it Resolve the Babel version from VERSION.txt, which every full release publishes as `Babel `, falling back to the final URL path segment for older trees that predate it (such as the 2025nov19 development directory). `latest/` therefore resolves to whichever release it currently points at rather than being treated as a version in its own right. The local cache holds one Babel release at a time, recorded in a .babel-version marker. When the release changes, sync_cache_version() deletes the .meta sidecars under /duckdb/ so the existing ETag path re-checks every cached file immediately, re-downloading only what actually changed. Deleting sidecars rather than the Parquet files themselves means nothing large is destroyed if the version cannot be trusted, an interrupted refresh self-heals (a .meta is only written after a successful download), and a directory the user pointed us at is never cleared wholesale. The hazard this closes is not cache invalidation, which ETag already covers, but cross-release mixing: Concord.parquet and Identifiers.parquet refresh independently, so without a pin a query can read two files from different Babel releases. Public Babel releases do not publish the DuckDB Parquet files this tool queries, so a 404 under duckdb/ now raises MissingBabelFileError naming the release and pointing at BABEL_URL, instead of being retried ten times with backoff before failing opaquely. Also add NodeNorm.get_babel_version(), reading `babel_version` from the status endpoint, so callers can tell which Babel a NodeNorm was built from. It stays silent in offline mode, where every lookup is short-circuited already. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/core/downloader.py | 96 ++++++++++++++++ src/babel_explorer/core/nodenorm.py | 25 +++++ tests/test_downloader.py | 152 +++++++++++++++++++++++++- 3 files changed, 272 insertions(+), 1 deletion(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 6382ad8..7f15f76 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -1,8 +1,10 @@ """HTTP downloader for Babel Parquet files with ETag-based freshness checking.""" import functools +import glob import json import os +import re import tempfile import time import requests @@ -10,6 +12,36 @@ from tqdm import tqdm import logging +#: Name of the file recording which Babel release the local cache holds. +VERSION_MARKER = ".babel-version" + + +class MissingBabelFileError(RuntimeError): + """Raised when a Babel release does not publish a file this tool needs.""" + + +def resolve_babel_version(url_base: str, timeout: int = 30) -> str | None: + """ + Resolve the Babel version behind a Babel base URL. + + Reads ``VERSION.txt`` (present on all full Babel releases, e.g. ``Babel 2026jul22``), + falling back to the final path segment for older trees that predate it. + + :return: The version string, or ``None`` if it cannot be determined. + """ + try: + response = requests.get(url_base + "VERSION.txt", timeout=timeout) + response.raise_for_status() + match = re.search(r"Babel\s+(\S+)", response.text) + if match: + return match.group(1) + except requests.RequestException: + pass + + # Legacy trees (e.g. the 2025nov19 development directory) have no VERSION.txt. + segment = url_base.rstrip("/").rsplit("/", 1)[-1] + return None if segment == "latest" else segment + class BabelDownloader: """ @@ -42,6 +74,8 @@ def __init__( self.freshness_seconds = freshness_seconds self.timeout = timeout self.logger = logging.getLogger(BabelDownloader.__name__) + self._babel_version: str | None = None + self._babel_version_resolved = False if local_path is None: local_path = tempfile.gettempdir() @@ -56,6 +90,59 @@ def __init__( f"Invalid local_path (must be an existing directory): '{local_path}'" ) + @property + def babel_version(self) -> str | None: + """The Babel release behind ``url_base``, resolved once and cached.""" + if not self._babel_version_resolved: + self._babel_version = resolve_babel_version(self.url_base, self.timeout) + self._babel_version_resolved = True + return self._babel_version + + def sync_cache_version(self): + """ + Point the local cache at the Babel release behind ``url_base``. + + The cache holds one release at a time. When the release changes, the ``.meta`` + sidecars are removed so the existing ETag path re-checks every cached file on + next use — whatever actually changed is re-downloaded, and files that are + unchanged cost one HEAD instead of a fresh multi-gigabyte download. + + Deleting the sidecars rather than the Parquet files also means nothing large is + destroyed if the version cannot be trusted, and an interrupted refresh self-heals: + a ``.meta`` file is only written after a successful download. + """ + version = self.babel_version + if version is None: + self.logger.warning( + f"Could not determine the Babel version at {self.url_base}; " + f"using cached files in {self.local_path} as-is" + ) + return + + marker_path = os.path.join(self.local_path, VERSION_MARKER) + try: + with open(marker_path) as f: + cached_version = f.read().strip() + except OSError: + cached_version = None + + if cached_version and cached_version != version: + self.logger.warning( + f"Babel version changed: {cached_version} → {version}; " + f"refreshing cached files in {self.local_path}" + ) + # Only ever touch sidecars this downloader wrote. Not recursive: local_path + # may be a directory the user pointed us at (or one holding other Babel + # releases in sibling subdirectories), and must not be cleared wholesale. + for meta_path in glob.glob( + os.path.join(self.local_path, "duckdb", "*.meta") + ): + os.remove(meta_path) + + if cached_version != version: + with open(marker_path, "w") as f: + f.write(version + "\n") + @functools.lru_cache(maxsize=None) def get_output_file(self, filename): """Return (and create) the local filesystem path for a given relative filename.""" @@ -256,6 +343,15 @@ def _download_with_retry(self, url, local_path, chunk_size): resume_byte_pos = 0 if os.path.exists(local_path): os.remove(local_path) + elif response.status_code == 404: + # Not worth retrying, and worth explaining: public Babel releases + # do not currently publish the DuckDB Parquet files. + raise MissingBabelFileError( + f"This Babel release ({self.babel_version or self.url_base}) does not " + f"publish {url[len(self.url_base) :]}. Translator team members should " + f"contact the Babel developers for the Translator-specific URL and set " + f"BABEL_URL in .env." + ) else: response.raise_for_status() diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index bfb0c27..bc4f404 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -55,6 +55,31 @@ def __init__(self, nodenorm_url: str = "", timeout: int = 30): self._normalize_cache: dict[str, dict | None] = {} self._identifier_cache: dict[str, Identifier] = {} self._clique_cache: dict[str, list[Identifier]] = {} + self._babel_version: str | None = None + self._babel_version_resolved = False + + def get_babel_version(self) -> str | None: + """Return the Babel release this NodeNorm instance was built from. + + :return: The version reported by the ``status`` endpoint, or ``None`` in offline + mode or if the endpoint cannot be reached or does not report one. + + The result is cached per instance. + """ + if self._babel_version_resolved: + return self._babel_version + + self._babel_version_resolved = True + if self.nodenorm_url: + try: + response = requests.get( + f"{self.nodenorm_url}status", timeout=self.timeout + ) + response.raise_for_status() + self._babel_version = response.json().get("babel_version") + except (requests.RequestException, ValueError) as e: + logging.warning(f"Could not read the Babel version from NodeNorm: {e}") + return self._babel_version def get_identifier(self, curie: str) -> "Identifier": """Return the ``Identifier`` for *curie* by looking it up in its NodeNorm clique. diff --git a/tests/test_downloader.py b/tests/test_downloader.py index db2e44b..c48a257 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -14,11 +14,161 @@ import requests from unittest.mock import Mock, patch, MagicMock -from babel_explorer.core.downloader import BabelDownloader +from babel_explorer.core.downloader import ( + BabelDownloader, + MissingBabelFileError, + VERSION_MARKER, + resolve_babel_version, +) from tests.constants import CONCORD_FILE +def _version_response(text): + """A mock requests response serving *text* as the body of VERSION.txt.""" + response = Mock() + response.text = text + response.raise_for_status = Mock() + return response + + +class TestResolveBabelVersion: + """Unit tests for resolve_babel_version().""" + + def test_reads_version_txt(self): + with patch( + "babel_explorer.core.downloader.requests.get", + return_value=_version_response( + "Babel 2026jul22\nhttps://github.com/NCATSTranslator/Babel\n" + ), + ) as mock_get: + assert ( + resolve_babel_version("https://example.com/babel/latest/") + == "2026jul22" + ) + assert ( + mock_get.call_args[0][0] == "https://example.com/babel/latest/VERSION.txt" + ) + + def test_version_txt_wins_over_path_segment(self): + """VERSION.txt is authoritative even when the URL names a version.""" + with patch( + "babel_explorer.core.downloader.requests.get", + return_value=_version_response("Babel 2026jul22\n"), + ): + assert ( + resolve_babel_version("https://example.com/babel/2025nov19/") + == "2026jul22" + ) + + def test_falls_back_to_path_segment(self): + """Trees predating VERSION.txt fall back to the final path segment.""" + with patch( + "babel_explorer.core.downloader.requests.get", + side_effect=requests.HTTPError("404"), + ): + assert ( + resolve_babel_version("https://example.com/babel/2025nov19/") + == "2025nov19" + ) + + def test_unresolvable_latest_returns_none(self): + """'latest' is not a version, so an unreachable VERSION.txt means unknown.""" + with patch( + "babel_explorer.core.downloader.requests.get", + side_effect=requests.ConnectionError("boom"), + ): + assert resolve_babel_version("https://example.com/babel/latest/") is None + + def test_unparseable_version_txt_falls_back(self): + with patch( + "babel_explorer.core.downloader.requests.get", + return_value=_version_response("something else entirely"), + ): + assert ( + resolve_babel_version("https://example.com/babel/2025nov19/") + == "2025nov19" + ) + + +class TestSyncCacheVersion: + """Unit tests for BabelDownloader.sync_cache_version().""" + + @staticmethod + def _downloader(tmp_path, version): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + dl._babel_version = version + dl._babel_version_resolved = True + return dl + + @staticmethod + def _seed_cache(tmp_path): + """Create a cached parquet file with its .meta sidecar.""" + duckdb_dir = tmp_path / "duckdb" + duckdb_dir.mkdir() + parquet = duckdb_dir / "Concord.parquet" + parquet.write_text("data") + meta = duckdb_dir / "Concord.parquet.meta" + meta.write_text("{}") + return parquet, meta + + def test_writes_marker_when_absent(self, tmp_path): + self._downloader(tmp_path, "2026jul22").sync_cache_version() + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2026jul22" + + def test_matching_version_keeps_meta(self, tmp_path): + _, meta = self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2026jul22\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert meta.exists() + + def test_changed_version_removes_meta_but_keeps_parquet(self, tmp_path): + parquet, meta = self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert not meta.exists(), "stale .meta sidecar should be removed" + assert parquet.exists(), "the Parquet file itself must never be deleted" + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2026jul22" + + def test_unknown_version_leaves_cache_untouched(self, tmp_path): + """An unresolvable version must not trigger a multi-gigabyte re-download.""" + _, meta = self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, None).sync_cache_version() + + assert meta.exists() + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2025nov19" + + +class TestMissingBabelFile: + """A 404 should explain itself and not be retried.""" + + def test_404_raises_immediately(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=10 + ) + dl._babel_version = "2025dec11" + dl._babel_version_resolved = True + + response = MagicMock() + response.status_code = 404 + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + + with patch( + "babel_explorer.core.downloader.requests.get", return_value=response + ) as mock_get: + with pytest.raises(MissingBabelFileError, match="2025dec11"): + dl.get_downloaded_file(CONCORD_FILE) + + assert mock_get.call_count == 1, "a 404 must not be retried" + + # ========================================================================== # Unit Tests — no network required # ========================================================================== From ec43b98e16c70f14f0174a12d2bbe0410113dde8 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:19:59 -0400 Subject: [PATCH 086/123] Read Babel and NodeNorm URLs from .env instead of hardcoding them The default Babel URL was a Translator-internal server that should not ship in a public repository. BABEL_URL, BABEL_LOCAL_DIR, BABEL_CHECK_DOWNLOAD, NODENORM_URL and BABEL_ALLOW_VERSION_MISMATCH are now read from .env via python-dotenv, wired through Click's envvar= so precedence runs flag > environment > .env > built-in default. The committed .env.example carries the public URL only, with a note telling Translator team members to ask the Babel developers for the internal one. The public default does not work end to end yet, because public releases do not publish the DuckDB Parquet files; that now surfaces as a plain error naming BABEL_URL rather than an opaque failure part-way through a multi-gigabyte download. Enriching cross-references with a NodeNorm built from a different Babel release yields labels and cliques that do not match the cross-references, so `xrefs` now fails on that mismatch, overridable with --allow-version-mismatch. The check runs only where NodeNorm is actually consulted: plain `xrefs` constructs one but never queries it, `ids` has none, and `test-concord` takes no --babel-url, since comparing NodeNorm against a rebuild is the whole point of that command. Integration tests now run against whatever BABEL_URL points at and skip when that release does not publish the Parquet files, so the suite stays usable for both Translator developers and public contributors. Also fold the duplicated --nodenorm-url declaration into a shared decorator, show defaults for --babel-url and --local-dir in --help, and drop a dead local and a placeholder-free f-string that were failing `ruff check`. Co-Authored-By: Claude Opus 5 --- .env.example | 17 +++++ pyproject.toml | 1 + src/babel_explorer/cli.py | 128 +++++++++++++++++++++++++++++--------- tests/conftest.py | 10 ++- tests/constants.py | 14 ++++- tests/test_cli.py | 112 ++++++++++++++++++++++++++++++++- uv.lock | 11 ++++ 7 files changed, 259 insertions(+), 34 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..bbd24c1 --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# Copy this file to .env and edit as needed. Every value here can also be set as an +# environment variable, or overridden per-run by the matching command-line option. + +# Public Babel release. "latest" is resolved through VERSION.txt; when the release +# behind it changes, babel-explorer refreshes the cached files automatically. +BABEL_URL=https://stars.renci.org/var/babel/latest/ + +# Where downloaded Babel files are cached. This holds one Babel release at a time. +# Point it somewhere per-release if you need to keep several around. +BABEL_LOCAL_DIR=data + +NODENORM_URL=https://nodenormalization-sri.renci.org/ + +# NOTE: public Babel releases do not currently publish the DuckDB Parquet files +# (duckdb/Concord.parquet, duckdb/Identifiers.parquet) that babel-explorer needs. +# Translator team members should contact the Babel developers for the +# Translator-specific URL and set BABEL_URL to it here. diff --git a/pyproject.toml b/pyproject.toml index 34a0be3..9e3ba51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.11" dependencies = [ "click>=8.3.1", "duckdb>=1.4.2", + "python-dotenv>=1.0", "requests>=2.32.5", "rich>=13", "tqdm>=4.67.0", diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 8c6b7e4..98558af 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -4,8 +4,14 @@ import logging from itertools import combinations -from babel_explorer.core.downloader import BabelDownloader -from babel_explorer.core.babel_xrefs import BabelXRefs, build_depth_map, find_shortest_path +from dotenv import load_dotenv + +from babel_explorer.core.downloader import BabelDownloader, MissingBabelFileError +from babel_explorer.core.babel_xrefs import ( + BabelXRefs, + build_depth_map, + find_shortest_path, +) from babel_explorer.core.nodenorm import NodeNorm from babel_explorer.core.babel_xrefs import LabeledCrossReference from babel_explorer.formatting import ( @@ -20,29 +26,89 @@ def babel_options(f): """Decorator adding --local-dir, --babel-url, and --check-download options to a command.""" + f = click.option( + "--allow-version-mismatch", + is_flag=True, + envvar="BABEL_ALLOW_VERSION_MISMATCH", + help="Proceed even if NodeNorm was built from a different Babel release than --babel-url", + )(f) f = click.option( "--check-download", type=str, default="3h", show_default=True, + envvar="BABEL_CHECK_DOWNLOAD", help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", )(f) f = click.option( "--babel-url", type=str, - default="https://stars.renci.org/var/babel/2025nov19/", + default="https://stars.renci.org/var/babel/latest/", + show_default=True, + envvar="BABEL_URL", help="Base URL of the Babel server", )(f) f = click.option( "--local-dir", type=str, - default="data/2025nov19", - help="Local location to save Babel download files to", + default="data", + show_default=True, + envvar="BABEL_LOCAL_DIR", + help="Local location to save Babel download files to. Holds one Babel release at " + "a time; cached files are refreshed automatically when --babel-url points at a new one.", )(f) return f +def nodenorm_options(f): + """Decorator adding --nodenorm-url to a command.""" + return click.option( + "--nodenorm-url", + type=str, + default="https://nodenormalization-sri.renci.org/", + show_default=True, + envvar="NODENORM_URL", + help="NodeNorm base URL used for node normalization and label enrichment", + )(f) + + +def make_downloader(babel_url: str, local_dir: str, check_download: str): + """Build a BabelDownloader and point its cache at the Babel release behind *babel_url*.""" + downloader = BabelDownloader( + babel_url, + local_path=local_dir, + freshness_seconds=parse_duration(check_download), + ) + downloader.sync_cache_version() + return downloader + + +def check_babel_versions( + downloader: BabelDownloader, nodenorm: NodeNorm, allow_version_mismatch: bool +): + """Fail if NodeNorm was built from a different Babel release than the one being queried. + + Cross-release results are silently wrong rather than obviously wrong: labels and + cliques would come from one Babel while the cross-references come from another. + Skipped when either version is unavailable. + """ + babel_version = downloader.babel_version + nodenorm_version = nodenorm.get_babel_version() + if ( + babel_version + and nodenorm_version + and babel_version != nodenorm_version + and not allow_version_mismatch + ): + raise click.ClickException( + f"NodeNorm at {nodenorm.nodenorm_url} was built from Babel {nodenorm_version}, " + f"but {downloader.url_base} is Babel {babel_version}. Labels and cliques would " + f"not match the cross-references. Point --nodenorm-url at a matching NodeNorm, " + f"or pass --allow-version-mismatch to proceed anyway." + ) + + def format_option(f): """Decorator adding --format and --json-indent options to a command.""" f = click.option( @@ -167,13 +233,12 @@ def _print_paths(console, curies, xrefs_list, labels: bool) -> None: step_word = "step" if n_steps == 1 else "steps" console.print( f"[bold]Path ({n_steps} {step_word}):[/bold] " - + f" [dim]→[/dim] ".join(node_strs) + + " [dim]→[/dim] ".join(node_strs) ) # Edge details, indented, oriented in traversal direction. for i, edge in enumerate(path): from_node = nodes[i] - to_node = nodes[i + 1] subj_node = edge.subj if edge.subj == from_node else edge.obj obj_node = edge.obj if edge.subj == from_node else edge.subj @@ -197,21 +262,28 @@ def _print_paths(console, curies, xrefs_list, labels: bool) -> None: console.print() -@click.group() +class BabelExplorerGroup(click.Group): + """Group that reports missing Babel files as a plain error rather than a traceback.""" + + def invoke(self, ctx): + try: + return super().invoke(ctx) + except MissingBabelFileError as e: + raise click.ClickException(str(e)) from e + + +@click.group(cls=BabelExplorerGroup) def cli(): """babel-explorer: query and explore Babel intermediate files.""" logging.basicConfig(level=logging.INFO) + # Runs before subcommand parameters are parsed, so .env feeds the envvar= defaults. + load_dotenv() @cli.command("xrefs") @click.argument("curies", type=str, required=True, nargs=-1) @babel_options -@click.option( - "--nodenorm-url", - type=str, - default="https://nodenormalization-sri.renci.org/", - help="NodeNorm base URL used for node normalization and label enrichment", -) +@nodenorm_options @click.option("--recurse", is_flag=True, help="Recursively query returned xrefs") @click.option("--labels", is_flag=True, help="Include labels for CURIEs") @click.option( @@ -229,6 +301,7 @@ def xrefs( labels: bool, paths: bool, check_download: str, + allow_version_mismatch: bool, fmt: str, json_indent: int, ): @@ -245,13 +318,16 @@ def xrefs( :return: None """ - freshness = parse_duration(check_download) - bxref = BabelXRefs( - BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness), - NodeNorm(nodenorm_url), - ) if paths: recurse = True + + downloader = make_downloader(babel_url, local_dir, check_download) + nodenorm = NodeNorm(nodenorm_url) + # NodeNorm is only consulted when labels or the recursive expansion need it. + if labels or recurse: + check_babel_versions(downloader, nodenorm, allow_version_mismatch) + + bxref = BabelXRefs(downloader, nodenorm) xref_list = bxref.get_curie_xrefs(curies, recurse, label_curies=labels) if fmt == "console": @@ -290,6 +366,7 @@ def ids( babel_url: str, local_dir: str, check_download: str, + allow_version_mismatch: bool, fmt: str, json_indent: int, ): @@ -306,10 +383,8 @@ def ids( :return: None """ - freshness = parse_duration(check_download) - bxref = BabelXRefs( - BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness) - ) + # No NodeNorm here, so there is no version to check against. + bxref = BabelXRefs(make_downloader(babel_url, local_dir, check_download)) xrefs = bxref.get_curie_ids(curies) if fmt == "console": @@ -322,12 +397,7 @@ def ids( @cli.command("test-concord") @click.argument("curies", type=str, required=True, nargs=-1) -@click.option( - "--nodenorm-url", - type=str, - default="https://nodenormalization-sri.renci.org/", - help="NodeNorm URL to check for concord changes", -) +@nodenorm_options @format_option def test_concord(curies, nodenorm_url, fmt, json_indent): """For each CURIE, print the current NodeNorm clique (all equivalent identifiers, labels, and Biolink types). diff --git a/tests/conftest.py b/tests/conftest.py index 92ecb06..9de874c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,7 @@ import shutil import pytest +import requests from filelock import FileLock from babel_explorer.core.downloader import BabelDownloader @@ -66,7 +67,14 @@ def test_data_dir(request): @pytest.fixture(scope="session") def shared_downloader(test_data_dir) -> BabelDownloader: - """A BabelDownloader pointed at the test data directory.""" + """A BabelDownloader pointed at the test data directory. + + Skips the whole session when BABEL_URL points at a Babel release that does not + publish the DuckDB Parquet files (as the public releases currently do not). + """ + response = requests.head(BABEL_URL + CONCORD_FILE, timeout=30) + if response.status_code == 404: + pytest.skip(f"{BABEL_URL} does not publish {CONCORD_FILE}") return BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) diff --git a/tests/constants.py b/tests/constants.py index 8366840..cfe7999 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,9 +1,19 @@ """Shared constants for babel-explorer tests.""" +import os import pathlib -BABEL_URL = "https://stars.renci.org/var/babel/2025nov19/" -NODENORM_URL = "https://nodenormalization-sri.renci.org/" +from dotenv import load_dotenv + +# Integration tests run against whatever BABEL_URL points at, so a Translator developer +# with a .env exercises them while public contributors and CI fall back to the public +# release (which does not yet publish the DuckDB Parquet files, so those tests skip). +load_dotenv() + +BABEL_URL = os.environ.get("BABEL_URL", "https://stars.renci.org/var/babel/latest/") +NODENORM_URL = os.environ.get( + "NODENORM_URL", "https://nodenormalization-sri.renci.org/" +) TEST_DATA_DIR = "data/test" # Parquet file paths (relative to the Babel server / local data dir) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9221020..1c0d841 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -94,10 +94,12 @@ def test_xrefs_recurse_and_labels_flags(self): mock_xref.filename = "test.parquet" with ( - patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelDownloader") as mock_dl, patch("babel_explorer.cli.BabelXRefs") as mock_bx, - patch("babel_explorer.cli.NodeNorm"), + patch("babel_explorer.cli.NodeNorm") as mock_nn, ): + mock_dl.return_value.babel_version = "2026jul22" + mock_nn.return_value.get_babel_version.return_value = "2026jul22" mock_bx.return_value.get_curie_xrefs.return_value = [mock_xref] result = runner.invoke( cli, ["xrefs", "MONDO:0004979", "--recurse", "--labels"] @@ -427,3 +429,109 @@ def test_text_format_rejected_by_click(self): result = runner.invoke(cli, ["xrefs", "A:1", "--format", "text"]) assert result.exit_code != 0 + + +class TestVersionChecking: + """The Babel release behind --babel-url must match the one NodeNorm was built from.""" + + @staticmethod + def _run(args, babel_version="2026jul22", nodenorm_version="2026jul22", env=None): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = babel_version + mock_nn.return_value.get_babel_version.return_value = nodenorm_version + mock_bx.return_value.get_curie_xrefs.return_value = [] + mock_bx.return_value.get_curie_ids.return_value = [] + result = runner.invoke(cli, args, env=env) + return result, mock_dl, mock_nn + + def test_mismatch_fails(self): + result, _, _ = self._run( + ["xrefs", "A:1", "--labels"], nodenorm_version="2025sep1" + ) + assert result.exit_code != 0 + assert "2025sep1" in result.output and "2026jul22" in result.output + + def test_mismatch_allowed_with_flag(self): + result, _, _ = self._run( + ["xrefs", "A:1", "--labels", "--allow-version-mismatch"], + nodenorm_version="2025sep1", + ) + assert result.exit_code == 0 + + def test_mismatch_allowed_via_env(self): + result, _, _ = self._run( + ["xrefs", "A:1", "--labels"], + nodenorm_version="2025sep1", + env={"BABEL_ALLOW_VERSION_MISMATCH": "1"}, + ) + assert result.exit_code == 0 + + def test_unknown_version_skips_check(self): + """Nothing to compare means nothing to complain about.""" + result, _, _ = self._run(["xrefs", "A:1", "--labels"], babel_version=None) + assert result.exit_code == 0 + + def test_plain_xrefs_skips_check(self): + """Plain xrefs builds a NodeNorm but never queries it, so skew is irrelevant.""" + result, _, mock_nn = self._run(["xrefs", "A:1"], nodenorm_version="2025sep1") + assert result.exit_code == 0 + mock_nn.return_value.get_babel_version.assert_not_called() + + def test_ids_skips_check(self): + """ids uses no NodeNorm at all.""" + result, _, mock_nn = self._run(["ids", "A:1"], nodenorm_version="2025sep1") + assert result.exit_code == 0 + mock_nn.return_value.get_babel_version.assert_not_called() + + def test_cache_is_synced_to_the_babel_release(self): + _, mock_dl, _ = self._run(["ids", "A:1"]) + mock_dl.return_value.sync_cache_version.assert_called_once() + + +class TestUrlConfiguration: + """URLs come from the environment (and hence .env), overridable per-run.""" + + @staticmethod + def _invoke(args, env): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke(cli, args, env=env) + assert result.exit_code == 0, result.output + return mock_dl, mock_nn + + def test_defaults_are_public(self): + mock_dl, mock_nn = self._invoke( + ["xrefs", "A:1"], env={"BABEL_URL": None, "NODENORM_URL": None} + ) + assert mock_dl.call_args[0][0] == "https://stars.renci.org/var/babel/latest/" + assert mock_nn.call_args[0][0] == "https://nodenormalization-sri.renci.org/" + + def test_env_overrides_defaults(self): + mock_dl, mock_nn = self._invoke( + ["xrefs", "A:1"], + env={ + "BABEL_URL": "https://example.com/babel/", + "BABEL_LOCAL_DIR": "/tmp/babel-cache", + "NODENORM_URL": "https://example.com/nn/", + }, + ) + assert mock_dl.call_args[0][0] == "https://example.com/babel/" + assert mock_dl.call_args.kwargs["local_path"] == "/tmp/babel-cache" + assert mock_nn.call_args[0][0] == "https://example.com/nn/" + + def test_flag_beats_env(self): + mock_dl, _ = self._invoke( + ["xrefs", "A:1", "--babel-url", "https://flag.example.com/"], + env={"BABEL_URL": "https://env.example.com/"}, + ) + assert mock_dl.call_args[0][0] == "https://flag.example.com/" diff --git a/uv.lock b/uv.lock index 7b201e4..f69e0e0 100644 --- a/uv.lock +++ b/uv.lock @@ -9,6 +9,7 @@ source = { editable = "." } dependencies = [ { name = "click" }, { name = "duckdb" }, + { name = "python-dotenv" }, { name = "requests" }, { name = "rich" }, { name = "tqdm" }, @@ -26,6 +27,7 @@ dev = [ requires-dist = [ { name = "click", specifier = ">=8.3.1" }, { name = "duckdb", specifier = ">=1.4.2" }, + { name = "python-dotenv", specifier = ">=1.0" }, { name = "requests", specifier = ">=2.32.5" }, { name = "rich", specifier = ">=13" }, { name = "tqdm", specifier = ">=4.67.0" }, @@ -324,6 +326,15 @@ psutil = [ { name = "psutil" }, ] +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + [[package]] name = "requests" version = "2.32.5" From 233630c6db191b9eccb6d48285ea0cbdcb5cddd4 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:20:02 -0400 Subject: [PATCH 087/123] Document .env configuration and Babel version handling Replace the Translator-internal URL and pinned 2025nov19 version throughout the docs with the public URL and the .env workflow, and describe how the single-release cache and the NodeNorm version check behave. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 52 +++++++++++++++++++++++++++++++++++++++++++++------- README.md | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 159ea56..124797d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,10 +17,43 @@ uv sync # Install with dev dependencies uv sync --group dev +# Configure the Babel and NodeNorm endpoints +cp .env.example .env + # Run the CLI uv run babel-explorer --help ``` +## Configuration + +`BABEL_URL`, `BABEL_LOCAL_DIR`, `BABEL_CHECK_DOWNLOAD`, `NODENORM_URL`, and +`BABEL_ALLOW_VERSION_MISMATCH` are read from `.env` (via `python-dotenv`, loaded in the `cli()` +group) or the environment. Each is also a command-line option, and precedence runs +**flag > environment variable > `.env` > built-in default**. + +`.env.example` ships with the **public** Babel URL only. Public Babel releases do not currently +publish the DuckDB Parquet files this tool needs, so Translator team members must contact the +Babel developers for the Translator-specific URL and set `BABEL_URL` to it. Never commit that URL +to this repository. + +## Babel versions + +The Babel version behind `--babel-url` is resolved by `resolve_babel_version()` +(`core/downloader.py`), which reads `VERSION.txt` (`Babel 2026jul22`) and falls back to the final +path segment for older trees that predate it. `latest/` resolves to whatever release it currently +points at. + +`BABEL_LOCAL_DIR` holds **one Babel release at a time**, recorded in a `.babel-version` marker. +When the release changes, `BabelDownloader.sync_cache_version()` deletes the `.meta` sidecars in +`/duckdb/` — never the Parquet files — so the existing ETag path re-checks each cached +file and re-downloads only what actually changed. This keeps `Concord.parquet` and +`Identifiers.parquet` from being read together across two different Babel releases. + +`xrefs` fails when NodeNorm's `status` endpoint reports a different `babel_version` than the Babel +being queried, since labels and cliques would not match the cross-references. Pass +`--allow-version-mismatch` to proceed anyway. The check is skipped when NodeNorm is not consulted +(plain `xrefs`, `ids`) or when either version is unavailable. + ## Commands ### Running the Application @@ -41,8 +74,8 @@ uv run babel-explorer ids MONDO:0004979 # Test concordance changes with NodeNorm uv run babel-explorer test-concord MONDO:0004979 HP:0000001 -# Use custom Babel server or local directory -uv run babel-explorer xrefs MONDO:0004979 --local-dir data/2025nov19 --babel-url https://stars.renci.org:443/var/babel/2025nov19/ +# Use a custom Babel server or local directory (overrides .env) +uv run babel-explorer xrefs MONDO:0004979 --local-dir data --babel-url https://stars.renci.org/var/babel/latest/ ``` ### Development Commands @@ -91,8 +124,10 @@ This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels 1. **BabelDownloader** (`src/babel_explorer/core/downloader.py`): - Downloads Babel intermediate files from a remote HTTP(S) server using Python's `requests` library (streaming downloads) - - Caches files locally in configurable directory (default: `data/2025nov19/`) + - Caches files locally in a configurable directory (default: `data/`), one Babel release at a time - Uses `@functools.lru_cache` to avoid re-downloading + - Resolves the Babel version (`resolve_babel_version`) and refreshes the cache when it changes (`sync_cache_version`) + - Raises `MissingBabelFileError` on a 404 for a `duckdb/` file, since public releases do not publish them - **Important**: Requires network access but no external tools like `wget` 2. **BabelXRefs** (`src/babel_explorer/core/babel_xrefs.py`): @@ -105,6 +140,7 @@ This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels - Integration with NodeNormalization API (https://nodenormalization-sri.renci.org/) - Fetches labels, biolink types, and equivalent identifiers for CURIEs - Uses `@functools.lru_cache` for performance + - `get_babel_version()` reads the `status` endpoint to report which Babel release it was built from - Optional component for label enrichment 4. **CLI** (`src/babel_explorer/cli.py`): @@ -113,8 +149,8 @@ This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels ### Data Flow -1. User provides CURIEs via CLI -2. BabelDownloader ensures required Parquet files are downloaded +1. User provides CURIEs via CLI; `BABEL_URL` / `NODENORM_URL` come from `.env` or the environment +2. BabelDownloader resolves the Babel version, refreshes the cache if it changed, and ensures required Parquet files are downloaded 3. BabelXRefs queries files using DuckDB 4. If `--labels` or `--recurse` flags are set, NodeNorm is queried for additional metadata 5. Results are printed to stdout @@ -158,12 +194,14 @@ Tests live in `tests/` and are split into fast **unit tests** (mocked, no networ ## Important Notes - **Data directory**: The `data/` directory is gitignored and contains downloaded Parquet files and generated DuckDB databases -- **Babel versions**: The default Babel version is `2025nov19`, but this can be customized via `--local-dir` and `--babel-url` +- **Babel versions**: The Babel release comes from whatever `--babel-url` / `BABEL_URL` points at; see [Babel versions](#babel-versions) above +- **`.env`**: gitignored. Only `.env.example` is committed, and it must never contain the Translator-specific Babel URL ## File Locations - Source code: `src/babel_explorer/` - Tests: `tests/` - Test CURIEs: `tests/data/valid_curies.txt` -- Downloaded Babel files: `data//duckdb/*.parquet` +- Downloaded Babel files: `/duckdb/*.parquet` (default `data/duckdb/`) +- Endpoint configuration: `.env` (gitignored), template in `.env.example` - Entry point: `src/babel_explorer/cli.py` diff --git a/README.md b/README.md index 96f7a38..dbc8312 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,38 @@ This project uses [uv](https://docs.astral.sh/uv/) for package management: ```bash uv sync --group dev +cp .env.example .env ``` +## Configuration + +`.env` holds the endpoints babel-explorer talks to: + +| Variable | Default | Purpose | +|---|---|---| +| `BABEL_URL` | `https://stars.renci.org/var/babel/latest/` | Babel release to query | +| `BABEL_LOCAL_DIR` | `data` | Where downloaded Babel files are cached | +| `BABEL_CHECK_DOWNLOAD` | `3h` | How often to re-check downloads | +| `NODENORM_URL` | `https://nodenormalization-sri.renci.org/` | NodeNorm instance for labels and cliques | + +Each has a matching command-line option, and precedence runs **flag > environment variable > +`.env` > default**. + +> **Translator team members:** public Babel releases do not currently publish the DuckDB Parquet +> files (`duckdb/Concord.parquet`, `duckdb/Identifiers.parquet`) that babel-explorer needs, so the +> default `BABEL_URL` will report that the files are missing. Contact the Babel developers for the +> Translator-specific URL and set `BABEL_URL` to it in your `.env`. + +### Babel versions + +`BABEL_LOCAL_DIR` holds one Babel release at a time. When `BABEL_URL` starts pointing at a +different release, babel-explorer notices and re-downloads the files that changed — you do not +need to clear the cache by hand. + +`xrefs` refuses to run when NodeNorm was built from a different Babel release than the one being +queried, since the labels and cliques would not match the cross-references. Pass +`--allow-version-mismatch` to override. + ## Usage ```bash @@ -36,6 +66,9 @@ uv run babel-explorer test-concord MONDO:0004979 HP:0000001 Tests are split into fast **unit tests** (mocked, no network) and slower **integration tests** (real file downloads and API calls), controlled by pytest markers. +Integration tests run against whatever `BABEL_URL` points at, and skip when that release does not +publish the DuckDB Parquet files. + ```bash # Unit tests only — fast, no network required uv run pytest -v -m "not integration" From 6e97ebc9a536b1025bf7ade29f96b83a4638adf0 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:27:56 -0400 Subject: [PATCH 088/123] Configure ruff lint rules and apply the resulting fixes The repository had no [tool.ruff] section, so ruff ran with its default rule set (E4, E7, E9, F) and never checked import ordering. Select E, F, I and UP, matching NCATSTranslator/Babel, with E501 left to the formatter since it owns wrapping. Line length stays at ruff's default of 88 rather than Babel's 120: adopting 120 would reflow 12 of 15 files for no correctness gain. Exclude *.md, because ruff 0.16 began formatting Python inside Markdown code blocks and this repository's snippets are illustrative fragments rather than runnable modules. ruff is currently 0.15.2 here, but the dependency is declared as >=0.11.0, so a lock refresh would hit this. The 30 resulting violations are all mechanical and auto-fixed: unsorted imports, datetime.timezone.utc to datetime.UTC, IOError to OSError, a redundant open() mode, and lru_cache(maxsize=None) to functools.cache. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 17 ++++++++++++++++ src/babel_explorer/cli.py | 12 ++++++------ src/babel_explorer/core/babel_xrefs.py | 1 + src/babel_explorer/core/downloader.py | 17 ++++++++-------- src/babel_explorer/core/nodenorm.py | 3 ++- tests/conftest.py | 10 ++++------ tests/test_babel_xrefs.py | 6 +++--- tests/test_cli.py | 5 ++--- tests/test_downloader.py | 27 +++++++++++++------------- tests/test_formatting.py | 7 +++---- tests/test_nodenorm.py | 5 ++--- 11 files changed, 62 insertions(+), 48 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9e3ba51..fe6e3a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,23 @@ dev = [ [project.scripts] babel-explorer = "babel_explorer.cli:cli" +[tool.ruff] +# ruff 0.16 began formatting Python inside Markdown code blocks. Our snippets are +# illustrative fragments rather than runnable modules, so keep ruff out of them. +# Line length is left at ruff's default of 88. +extend-exclude = ["*.md"] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort (import sorting) + "UP", # pyupgrade +] +ignore = [ + "E501", # let the formatter handle wrapping consistently +] + [tool.pytest.ini_options] addopts = "-n auto" markers = [ diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 98558af..e8e4ac7 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -1,27 +1,27 @@ """Command-line interface for babel-explorer.""" -import click import logging from itertools import combinations +import click from dotenv import load_dotenv +from rich.markup import escape -from babel_explorer.core.downloader import BabelDownloader, MissingBabelFileError from babel_explorer.core.babel_xrefs import ( BabelXRefs, + LabeledCrossReference, build_depth_map, find_shortest_path, ) +from babel_explorer.core.downloader import BabelDownloader, MissingBabelFileError from babel_explorer.core.nodenorm import NodeNorm -from babel_explorer.core.babel_xrefs import LabeledCrossReference from babel_explorer.formatting import ( - write_records, _record_to_dict, - make_console, hl_curie, hl_curie_at_depth, + make_console, + write_records, ) -from rich.markup import escape def babel_options(f): diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 9f6b4d0..00320f6 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -8,6 +8,7 @@ import dataclasses import logging from collections import deque + import duckdb from babel_explorer.core.downloader import BabelDownloader diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 7f15f76..61d6988 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -3,14 +3,15 @@ import functools import glob import json +import logging import os import re import tempfile import time +from datetime import UTC, datetime + import requests -from datetime import datetime, timezone from tqdm import tqdm -import logging #: Name of the file recording which Babel release the local cache holds. VERSION_MARKER = ".babel-version" @@ -143,7 +144,7 @@ def sync_cache_version(self): with open(marker_path, "w") as f: f.write(version + "\n") - @functools.lru_cache(maxsize=None) + @functools.cache def get_output_file(self, filename): """Return (and create) the local filesystem path for a given relative filename.""" filepath = os.path.join(self.local_path, filename) @@ -160,7 +161,7 @@ def _load_meta(self, local_path): if not os.path.exists(meta_path): return None try: - with open(meta_path, "r") as f: + with open(meta_path) as f: return json.load(f) except (json.JSONDecodeError, OSError): return None @@ -182,7 +183,7 @@ def _save_meta(self, local_path, headers, update_last_checked=True): if "Content-Length" in headers: meta["content_length"] = int(headers["Content-Length"]) if update_last_checked: - meta["last_checked"] = datetime.now(timezone.utc).isoformat() + meta["last_checked"] = datetime.now(UTC).isoformat() meta_path = self._get_meta_path(local_path) with open(meta_path, "w") as f: @@ -206,7 +207,7 @@ def _is_within_freshness(self, meta, freshness_seconds): return False try: last_checked = datetime.fromisoformat(last_checked_str) - age = (datetime.now(timezone.utc) - last_checked).total_seconds() + age = (datetime.now(UTC) - last_checked).total_seconds() return age < freshness_seconds except (ValueError, TypeError): return False @@ -360,7 +361,7 @@ def _download_with_retry(self, url, local_path, chunk_size): ) return response.headers - except (requests.RequestException, IOError) as e: + except (OSError, requests.RequestException) as e: self.logger.warning( f"Download attempt {attempt}/{self.retries} failed: {e}" ) @@ -408,7 +409,7 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): # Tier 2: stale but maybe unchanged — HEAD request if self._etag_matches(url_to_download, meta): # Update last_checked timestamp - meta["last_checked"] = datetime.now(timezone.utc).isoformat() + meta["last_checked"] = datetime.now(UTC).isoformat() meta_path = self._get_meta_path(local_path_to_download_to) with open(meta_path, "w") as f: json.dump(meta, f, indent=2) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index bc4f404..866b207 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -1,9 +1,10 @@ """NodeNorm API client for identifier normalisation and label enrichment.""" import dataclasses -import requests import logging +import requests + @dataclasses.dataclass(frozen=True) class Identifier: diff --git a/tests/conftest.py b/tests/conftest.py index 9de874c..51b9d08 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,21 +12,19 @@ import requests from filelock import FileLock -from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.babel_xrefs import BabelXRefs +from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.nodenorm import NodeNorm - from tests.constants import ( BABEL_URL, - NODENORM_URL, - TEST_DATA_DIR, CONCORD_FILE, - METADATA_FILE, IDENTIFIERS_FILE, + METADATA_FILE, + NODENORM_URL, + TEST_DATA_DIR, load_curies, ) - # --------------------------------------------------------------------------- # Session-scoped fixtures # --------------------------------------------------------------------------- diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index f4304f7..3e6ba39 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -4,18 +4,18 @@ Unit tests use mocks; integration tests query real Parquet files via DuckDB. """ +from unittest.mock import MagicMock, patch + import pytest -from unittest.mock import patch, MagicMock from babel_explorer.core.babel_xrefs import ( BabelXRefs, CrossReference, - LabeledCrossReference, IdentifierRecord, + LabeledCrossReference, ) from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.nodenorm import NodeNorm - from tests.constants import load_curies VALID_CURIES = load_curies() diff --git a/tests/test_cli.py b/tests/test_cli.py index 1c0d841..8392bd8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,17 +5,16 @@ """ import json +from unittest.mock import MagicMock, patch import click import pytest from click.testing import CliRunner -from unittest.mock import patch, MagicMock -from babel_explorer.cli import parse_duration, cli +from babel_explorer.cli import cli, parse_duration from babel_explorer.core.babel_xrefs import CrossReference, IdentifierRecord from babel_explorer.core.nodenorm import Identifier - # ========================================================================== # Unit Tests — no network required # ========================================================================== diff --git a/tests/test_downloader.py b/tests/test_downloader.py index c48a257..3a808c6 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -8,19 +8,18 @@ import json import os import tempfile -from datetime import datetime, timezone, timedelta +from datetime import UTC, datetime, timedelta +from unittest.mock import MagicMock, Mock, patch import pytest import requests -from unittest.mock import Mock, patch, MagicMock from babel_explorer.core.downloader import ( + VERSION_MARKER, BabelDownloader, MissingBabelFileError, - VERSION_MARKER, resolve_babel_version, ) - from tests.constants import CONCORD_FILE @@ -297,7 +296,7 @@ def test_last_checked_is_recent_utc(self, tmp_path): meta = json.load(f) last_checked = datetime.fromisoformat(meta["last_checked"]) - age = (datetime.now(timezone.utc) - last_checked).total_seconds() + age = (datetime.now(UTC) - last_checked).total_seconds() assert age < 5 # written less than 5 seconds ago def test_missing_headers_not_written(self, tmp_path): @@ -360,13 +359,13 @@ def _make_dl(self, tmp_path): def test_returns_true_when_recent(self, tmp_path): dl = self._make_dl(tmp_path) - recent = datetime.now(timezone.utc).isoformat() + recent = datetime.now(UTC).isoformat() meta = {"last_checked": recent} assert dl._is_within_freshness(meta, 3600) is True def test_returns_false_when_stale(self, tmp_path): dl = self._make_dl(tmp_path) - old = (datetime.now(timezone.utc) - timedelta(hours=5)).isoformat() + old = (datetime.now(UTC) - timedelta(hours=5)).isoformat() meta = {"last_checked": old} assert dl._is_within_freshness(meta, 3600) is False @@ -376,13 +375,13 @@ def test_returns_false_when_missing_last_checked(self, tmp_path): def test_returns_true_when_freshness_is_inf(self, tmp_path): dl = self._make_dl(tmp_path) - old = (datetime.now(timezone.utc) - timedelta(days=365)).isoformat() + old = (datetime.now(UTC) - timedelta(days=365)).isoformat() meta = {"last_checked": old} assert dl._is_within_freshness(meta, float("inf")) is True def test_returns_false_when_freshness_is_zero(self, tmp_path): dl = self._make_dl(tmp_path) - just_now = datetime.now(timezone.utc).isoformat() + just_now = datetime.now(UTC).isoformat() meta = {"last_checked": just_now} # Even with freshness=0, age >= 0 so it's not < 0 assert dl._is_within_freshness(meta, 0) is False @@ -461,7 +460,7 @@ def test_tier1_returns_immediately_no_http(self, tmp_path): local.parent.mkdir(parents=True) local.write_bytes(b"data") - meta = {"etag": '"abc"', "last_checked": datetime.now(timezone.utc).isoformat()} + meta = {"etag": '"abc"', "last_checked": datetime.now(UTC).isoformat()} with open(str(local) + ".meta", "w") as f: json.dump(meta, f) @@ -482,7 +481,7 @@ def test_tier2_head_check_no_redownload(self, tmp_path): local.parent.mkdir(parents=True) local.write_bytes(b"data") - old_ts = (datetime.now(timezone.utc) - timedelta(hours=5)).isoformat() + old_ts = (datetime.now(UTC) - timedelta(hours=5)).isoformat() meta = {"etag": '"abc"', "last_checked": old_ts} with open(str(local) + ".meta", "w") as f: json.dump(meta, f) @@ -507,7 +506,7 @@ def test_tier2_updates_last_checked_after_head(self, tmp_path): local.parent.mkdir(parents=True) local.write_bytes(b"data") - old_ts = (datetime.now(timezone.utc) - timedelta(hours=5)).isoformat() + old_ts = (datetime.now(UTC) - timedelta(hours=5)).isoformat() meta = {"etag": '"abc"', "last_checked": old_ts} with open(str(local) + ".meta", "w") as f: json.dump(meta, f) @@ -524,7 +523,7 @@ def test_tier2_updates_last_checked_after_head(self, tmp_path): with open(str(local) + ".meta") as f: updated_meta = json.load(f) updated_ts = datetime.fromisoformat(updated_meta["last_checked"]) - assert (datetime.now(timezone.utc) - updated_ts).total_seconds() < 5 + assert (datetime.now(UTC) - updated_ts).total_seconds() < 5 # --- Tier 3: ETag changed, re-download --- @@ -536,7 +535,7 @@ def test_tier3_redownloads_when_etag_changed(self, tmp_path): local.parent.mkdir(parents=True) local.write_bytes(b"old data") - old_ts = (datetime.now(timezone.utc) - timedelta(hours=5)).isoformat() + old_ts = (datetime.now(UTC) - timedelta(hours=5)).isoformat() meta = {"etag": '"old"', "last_checked": old_ts} with open(str(local) + ".meta", "w") as f: json.dump(meta, f) diff --git a/tests/test_formatting.py b/tests/test_formatting.py index 643544c..6a42855 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -10,18 +10,17 @@ from babel_explorer.core.babel_xrefs import ( CrossReference, - LabeledCrossReference, IdentifierRecord, + LabeledCrossReference, ) from babel_explorer.core.nodenorm import Identifier from babel_explorer.formatting import ( _record_to_dict, - write_records, - make_console, hl_curie, + make_console, + write_records, ) - # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index b0bc921..a301ecb 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -4,13 +4,12 @@ Unit tests use mocks; integration tests call the real NodeNorm API. """ -import pytest from unittest.mock import Mock, patch +import pytest import requests -from babel_explorer.core.nodenorm import NodeNorm, Identifier - +from babel_explorer.core.nodenorm import Identifier, NodeNorm from tests.constants import load_curies VALID_CURIES = load_curies() From ba84b9dc688827c9e40031bf2e44dcdf7c4b5de6 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:28:03 -0400 Subject: [PATCH 089/123] Annotate ruff failures on the PR diff and document the pre-commit step Nothing told contributors or coding agents to run ruff before pushing, so formatting drift reached main and only surfaced as a red PR later. Add an explicit "run before committing or pushing" instruction to CLAUDE.md and README.md, including what to do when ruff reports files you did not touch: commit that reformatting separately rather than reverting it. In CI, pass --output-format github so failures appear as inline annotations on the diff instead of buried in the log, and drop the hardcoded `src/ tests/` paths now that [tool.ruff] defines the scope. CI keeps using `uv run ruff` rather than astral-sh/ruff-action: uv already resolves the ruff pinned in uv.lock, so CI and local runs share a version without the action's version-file plumbing. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 7 +++++-- CLAUDE.md | 23 +++++++++++++++++++---- README.md | 9 +++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3ef4c8..c50f042 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,11 @@ jobs: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 - run: uv sync --group dev - - run: uv run ruff check src/ tests/ - - run: uv run ruff format --check src/ tests/ + # `uv run` uses the ruff pinned in uv.lock, so CI lints with the same version + # developers have locally. --output-format github annotates the PR diff inline. + # Paths come from [tool.ruff] in pyproject.toml rather than being repeated here. + - run: uv run ruff check --output-format github + - run: uv run ruff format --check test: runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index 124797d..1ba785a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,14 +92,29 @@ uv run pytest -v -m "integration and not slow" # Run a single test file uv run pytest -v tests/test_nodenorm.py +``` + +### Linting -# Run linter -uv run ruff check +**Run both of these before committing or pushing.** CI checks them on every PR, and a push +that skips them turns the PR red for reasons unrelated to the change under review. -# Format code -uv run ruff format +```bash +uv run ruff check # Python lint +uv run ruff check --fix # Python auto-fix +uv run ruff format --check # Python format check +uv run ruff format # Python auto-format ``` +Run them over the whole repository, not just the files you touched — `[tool.ruff]` in +`pyproject.toml` sets the scope. If `ruff format` reports files you did not edit, the repository +had drifted; commit that reformatting separately from your change so review stays readable, and +do not silently revert it. + +Rules are `E`, `F`, `I` (import sorting) and `UP` (pyupgrade), with `E501` left to the formatter. +Line length is ruff's default of 88. `*.md` is excluded, because ruff 0.16+ reformats Python +inside Markdown code blocks and this repository's snippets are illustrative fragments. + ## Console Output Format Conventions ### Label display diff --git a/README.md b/README.md index dbc8312..6ecdb23 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,15 @@ uv run pytest -v -m "integration and not slow" uv run pytest -v ``` +## Linting + +Run both checks before committing; CI enforces them on every pull request: + +```bash +uv run ruff check --fix # lint, with auto-fix +uv run ruff format # format +``` + ### Adding Test CURIEs Integration tests are parametrized over the CURIEs listed in `tests/data/valid_curies.txt`. Add a new CURIE on its own line to automatically expand test coverage: From ac9a0e6585ad9d1d23771f3151326714c3e6608a Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:30:56 -0400 Subject: [PATCH 090/123] Delete the unused BabelDownloader.get_output_file Nothing in src/ called it: BabelXRefs builds DuckDB paths itself and now queries Parquet through inline read_parquet(), so the helper only existed to be tested. Its @functools.cache decorator also kept a strong reference to self for the lifetime of the process, the same leak that motivated replacing lru_cache with instance dicts in NodeNorm. Drops its three tests, and a vestigial patch.object() in test_get_curie_xref_calls_downloader that stubbed the method without ever asserting on it. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/core/downloader.py | 8 -------- tests/test_babel_xrefs.py | 21 +++++++++------------ tests/test_downloader.py | 20 -------------------- 3 files changed, 9 insertions(+), 40 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 61d6988..7a4f62b 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -1,6 +1,5 @@ """HTTP downloader for Babel Parquet files with ETag-based freshness checking.""" -import functools import glob import json import logging @@ -144,13 +143,6 @@ def sync_cache_version(self): with open(marker_path, "w") as f: f.write(version + "\n") - @functools.cache - def get_output_file(self, filename): - """Return (and create) the local filesystem path for a given relative filename.""" - filepath = os.path.join(self.local_path, filename) - os.makedirs(os.path.dirname(filepath), exist_ok=True) - return filepath - def _get_meta_path(self, local_path): """Return the sidecar metadata file path for a given local file.""" return local_path + ".meta" diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 3e6ba39..10a8486 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -223,19 +223,16 @@ def test_get_curie_xref_calls_downloader(self, tmp_path): with patch.object( bx.downloader, "get_downloaded_file", return_value="/fake/path" ) as mock_dl: - with patch.object( - bx.downloader, "get_output_file", return_value="/fake/db" + with patch( + "babel_explorer.core.babel_xrefs.duckdb.connect", + return_value=mock_db, ): - with patch( - "babel_explorer.core.babel_xrefs.duckdb.connect", - return_value=mock_db, - ): - result = bx.get_curie_xref("A:1") - # Downloader should be called for Concord only (Metadata unused here) - assert mock_dl.call_count == 1 - result_list = list(result) - assert len(result_list) == 1 - assert isinstance(result_list[0], CrossReference) + result = bx.get_curie_xref("A:1") + # Downloader should be called for Concord only (Metadata unused here) + assert mock_dl.call_count == 1 + result_list = list(result) + assert len(result_list) == 1 + assert isinstance(result_list[0], CrossReference) def test_get_curie_xrefs_no_expand(self, tmp_path): bx = self._make_bx(tmp_path) diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 3a808c6..d26f797 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -234,26 +234,6 @@ def test_invalid_path_raises_value_error(self): os.unlink(f.name) -class TestGetOutputFile: - """Tests for get_output_file.""" - - def test_returns_correct_path(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - result = dl.get_output_file("output/duckdbs/test.duckdb") - assert result == os.path.join(str(tmp_path), "output/duckdbs/test.duckdb") - - def test_creates_parent_directories(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - result = dl.get_output_file("deep/nested/dir/file.txt") - assert os.path.isdir(os.path.dirname(result)) - - def test_lru_caching(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - result1 = dl.get_output_file("some/file.txt") - result2 = dl.get_output_file("some/file.txt") - assert result1 is result2 # identity check — same cached object - - class TestSaveMeta: """Tests for _save_meta.""" From 189b29e649bfddec83ac0a91e8444a87685acd76 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:38:52 -0400 Subject: [PATCH 091/123] Cover NodeNorm version lookup, missing-file reporting and refresh scoping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three behaviours added this session had no tests: NodeNorm.get_babel_version() — reads the status endpoint, stays silent in offline mode, returns None rather than raising when NodeNorm is unreachable or reports no version, and caches both outcomes so a failed lookup is not retried on every call. The group-level conversion of MissingBabelFileError into a Click error, so a Babel release that does not publish the Parquet files reads as a message rather than a traceback, for every command rather than just xrefs. That a cache refresh clears only /duckdb/*.meta. The glob is deliberately not recursive: local_path may hold other Babel releases in nested directories, and sweeping those up would force needless re-checks of gigabyte files. Verified the test fails when the glob is made recursive again. Also drop the per-file test-count table from CLAUDE.md. It had drifted badly (test_downloader listed 41 unit tests against an actual 49, test_formatting was missing entirely), so replace it with the collect-only commands, and note that integration tests skipping in bulk is the expected result without a Translator BABEL_URL. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 19 +++++++----- tests/test_cli.py | 41 +++++++++++++++++++++++++ tests/test_downloader.py | 15 ++++++++++ tests/test_nodenorm.py | 65 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1ba785a..81fd62d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -186,16 +186,21 @@ Tests live in `tests/` and are split into fast **unit tests** (mocked, no networ - **`@pytest.mark.integration`** — requires network access (downloads Parquet files or calls NodeNorm API) - **`@pytest.mark.slow`** — downloads very large files (2 GB+) -| File | Unit | Integration | Slow | Total | -|------|------|-------------|------|-------| -| `tests/test_downloader.py` | 41 | 4 | 1 | 46 | -| `tests/test_babel_xrefs.py` | 23 | 20 | 3 | 46 | -| `tests/test_nodenorm.py` | 20 | 13 | 0 | 33 | -| `tests/test_cli.py` | 24 | 0 | 0 | 24 | +Do not record per-file test counts here — they drift silently and then mislead. Get them on demand: + +```bash +uv run pytest --collect-only -q -m "not integration" # unit test count +uv run pytest --collect-only -q # full count +``` + +**Integration tests skip when `BABEL_URL` points at a Babel release that does not publish +`duckdb/Concord.parquet`**, which is the case for every public release right now. A run reporting +a couple of dozen skips is the expected result without a Translator `BABEL_URL` in `.env`, not a +broken test environment. ### Test Infrastructure -- **`tests/conftest.py`** — Session-scoped fixtures that download Parquet files once and share them across all integration tests. Teardown removes the `data/test/` directory so the next run starts fresh. +- **`tests/conftest.py`** — Session-scoped fixtures that download Parquet files once and share them across all integration tests. The `shared_downloader` fixture HEADs `duckdb/Concord.parquet` first and skips the session on 404. Teardown removes the `data/test/` directory so the next run starts fresh. - **`tests/constants.py`** — Shared constants (URLs, file paths) and `load_curies()` helper. - **`tests/data/valid_curies.txt`** — One CURIE per line (`#` comments allowed). Integration tests are parametrized over this list — adding a new line automatically expands test coverage. diff --git a/tests/test_cli.py b/tests/test_cli.py index 8392bd8..88adeb7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,6 +13,7 @@ from babel_explorer.cli import cli, parse_duration from babel_explorer.core.babel_xrefs import CrossReference, IdentifierRecord +from babel_explorer.core.downloader import MissingBabelFileError from babel_explorer.core.nodenorm import Identifier # ========================================================================== @@ -534,3 +535,43 @@ def test_flag_beats_env(self): env={"BABEL_URL": "https://env.example.com/"}, ) assert mock_dl.call_args[0][0] == "https://flag.example.com/" + + +class TestMissingBabelFileReporting: + """A missing Parquet file should read as an error, not a traceback.""" + + def test_reported_without_traceback(self): + runner = CliRunner() + message = ( + "This Babel release (2025dec11) does not publish duckdb/Concord.parquet." + ) + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.side_effect = MissingBabelFileError( + message + ) + result = runner.invoke(cli, ["xrefs", "A:1"]) + + assert result.exit_code == 1 + assert message in result.output + assert "Traceback" not in result.output + assert isinstance(result.exception, SystemExit) + + def test_also_wrapped_for_ids(self): + """The conversion lives on the group, so every command inherits it.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + ): + mock_bx.return_value.get_curie_ids.side_effect = MissingBabelFileError( + "nope" + ) + result = runner.invoke(cli, ["ids", "A:1"]) + + assert result.exit_code == 1 + assert "nope" in result.output + assert "Traceback" not in result.output diff --git a/tests/test_downloader.py b/tests/test_downloader.py index d26f797..b75a9a2 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -133,6 +133,21 @@ def test_changed_version_removes_meta_but_keeps_parquet(self, tmp_path): assert parquet.exists(), "the Parquet file itself must never be deleted" assert (tmp_path / VERSION_MARKER).read_text().strip() == "2026jul22" + def test_refresh_does_not_reach_into_sibling_directories(self, tmp_path): + """local_path may hold other Babel releases; only our own duckdb/ is cleared.""" + self._seed_cache(tmp_path) + sibling = tmp_path / "2025nov19" / "duckdb" + sibling.mkdir(parents=True) + sibling_meta = sibling / "Concord.parquet.meta" + sibling_meta.write_text("{}") + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert sibling_meta.exists(), ( + "a nested release directory must not be swept up in the refresh" + ) + def test_unknown_version_leaves_cache_untouched(self, tmp_path): """An unresolvable version must not trigger a multi-gigabyte re-download.""" _, meta = self._seed_cache(tmp_path) diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index a301ecb..dbdc30d 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -292,6 +292,71 @@ def test_caching(self): mock_norm.assert_called_once() +class TestGetBabelVersionMocked: + """Tests for get_babel_version().""" + + @staticmethod + def _status_response(payload): + response = Mock() + response.json = Mock(return_value=payload) + response.raise_for_status = Mock() + return response + + def test_reads_babel_version_from_status(self): + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._status_response({"babel_version": "2026jul22"}), + ) as mock_get: + assert nn.get_babel_version() == "2026jul22" + assert mock_get.call_args[0][0] == "https://example.com/nn/status" + + def test_offline_mode_makes_no_request(self): + """An empty URL short-circuits every lookup, including this one.""" + nn = NodeNorm(nodenorm_url="") + with patch("babel_explorer.core.nodenorm.requests.get") as mock_get: + assert nn.get_babel_version() is None + mock_get.assert_not_called() + + def test_unreachable_status_returns_none(self): + """A version check must never take down the command that called it.""" + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + side_effect=requests.ConnectionError("boom"), + ): + assert nn.get_babel_version() is None + + def test_status_without_babel_version_returns_none(self): + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._status_response({"biolink_model": {"tag": "v4.2.6"}}), + ): + assert nn.get_babel_version() is None + + def test_result_is_cached(self): + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._status_response({"babel_version": "2026jul22"}), + ) as mock_get: + nn.get_babel_version() + nn.get_babel_version() + mock_get.assert_called_once() + + def test_failure_is_cached_too(self): + """A failed lookup must not be retried on every subsequent call.""" + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + side_effect=requests.ConnectionError("boom"), + ) as mock_get: + nn.get_babel_version() + nn.get_babel_version() + mock_get.assert_called_once() + + # ========================================================================== # Integration Tests — require real NodeNorm API # ========================================================================== From 666126632a31afe04e911f9f8a72c704c3d7855e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:46:14 -0400 Subject: [PATCH 092/123] Ignore .idea/ The GitHub Python template ships this line commented out. JetBrains project files are local editor state, so uncomment it rather than have .idea/ show up as untracked in every git status. Co-Authored-By: Claude Opus 5 --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 55c6b2a..f1f9988 100644 --- a/.gitignore +++ b/.gitignore @@ -177,7 +177,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Abstra # Abstra is an AI-powered process automation framework. From 711d2f8b8e788dc5cbc7a9c751719bc6dedd4f53 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:46:14 -0400 Subject: [PATCH 093/123] Remove the test data directory once all xdist workers have finished `addopts = "-n auto"` means every run is parallel, and the session fixture's teardown was guarded on being the "master" worker -- which never happens under xdist. data/test/ therefore survived every run, contrary to the comment saying it was removed so the next run starts fresh. Move the cleanup to pytest_sessionfinish, which the xdist controller runs after all workers exit. That removes the race the guard existed to avoid (gw0 deleting Concord.parquet while gw5 still reads it) without disabling cleanup, and still fires on a non-parallel run, where there is no worker either. Co-Authored-By: Claude Opus 5 --- tests/conftest.py | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 51b9d08..82f7de0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,29 +38,34 @@ def valid_curies() -> list[str]: return curies -@pytest.fixture(scope="session") -def test_data_dir(request): - """ - Provide a test data directory for the entire session. - - Creates the directory before tests, removes it after all tests complete. - When running under pytest-xdist, cleanup is skipped: worker sessions end at - unpredictable times and deleting the shared directory from one worker while - others are still reading the same files causes flaky IO errors. The files - are re-used (or re-validated) on the next run via the freshness-window logic - in BabelDownloader.get_downloaded_file. +def pytest_sessionfinish(session, exitstatus): + """Remove the shared test data directory once every worker has finished. + + This runs in the xdist controller, which finishes only after all workers do. + Workers are identified by having a ``workerinput`` attribute; a plain + non-parallel run has none either, so cleanup happens there too. + + Cleaning up from a session fixture's teardown instead does not work: with + ``-n auto`` in ``addopts`` every run is parallel, so each worker would tear + down at an unpredictable time and gw0 could delete Concord.parquet while gw5 + is still reading it. Guarding that teardown on the worker id, as this used to, + meant the directory was simply never removed. """ - worker_id = getattr(request.config, "workerinput", {}).get("workerid", "master") - os.makedirs(TEST_DATA_DIR, exist_ok=True) + if hasattr(session.config, "workerinput"): + return + if os.path.exists(TEST_DATA_DIR): + shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) - yield TEST_DATA_DIR - # Only clean up when running without xdist (sequential run). In a parallel - # run each worker session may finish at a different time; gw0 cleaning up - # while gw5 is still reading Concord.parquet causes spurious failures. - if worker_id == "master": - if os.path.exists(TEST_DATA_DIR): - shutil.rmtree(TEST_DATA_DIR) +@pytest.fixture(scope="session") +def test_data_dir(): + """Provide a test data directory for the entire session. + + Removed by ``pytest_sessionfinish`` once all workers are done, so the next + run starts fresh. + """ + os.makedirs(TEST_DATA_DIR, exist_ok=True) + return TEST_DATA_DIR @pytest.fixture(scope="session") From f137b92e25b9296c7fe7b5cac5c2b021a8e2593d Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:46:19 -0400 Subject: [PATCH 094/123] Reject --paths with non-console output formats --paths has a renderer only for the console format. With --format json, tsv or csv the flag was silently ignored and the full recursive cross-reference list was emitted instead, which looks like a successful --paths run but is not one. Fail with a usage error naming the alternative, checked before anything is downloaded so the mistake costs nothing. Emitting paths as structured records would be a feature rather than a fix; nothing asks for it yet. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 10 ++++- src/babel_explorer/cli.py | 26 +++++++++++-- tests/test_cli.py | 81 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 110 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 81fd62d..329c481 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,9 @@ uv run babel-explorer xrefs MONDO:0004979 --labels # Get ID records for CURIEs uv run babel-explorer ids MONDO:0004979 +# Get ID records with labels from NodeNorm +uv run babel-explorer ids MONDO:0004979 --labels + # Test concordance changes with NodeNorm uv run babel-explorer test-concord MONDO:0004979 HP:0000001 @@ -125,7 +128,10 @@ When a human-readable label is shown alongside a CURIE in console output, it alw MONDO:0004979 "asthma" skos:exactMatch EFO:0000270 "asthma" ``` -This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels`, and `test-concord`. +This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels`, `ids --labels`, and `test-concord`. + +`--paths` is console-only; combining it with `--format json`/`tsv`/`csv` is rejected up front rather +than silently emitting the full cross-reference list. **When a label is absent, omit it entirely** — do not substitute a placeholder like `-` or `""`. A CURIE with no label renders as just the bare CURIE. @@ -167,7 +173,7 @@ This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels 1. User provides CURIEs via CLI; `BABEL_URL` / `NODENORM_URL` come from `.env` or the environment 2. BabelDownloader resolves the Babel version, refreshes the cache if it changed, and ensures required Parquet files are downloaded 3. BabelXRefs queries files using DuckDB -4. If `--labels` or `--recurse` flags are set, NodeNorm is queried for additional metadata +4. If `--labels` or `--recurse` flags are set, NodeNorm is queried for additional metadata (`ids` consults NodeNorm only for `--labels`) 5. Results are printed to stdout ### Key Design Patterns diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index e8e4ac7..ea6da9d 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -319,6 +319,14 @@ def xrefs( :return: None """ if paths: + # Checked before anything is downloaded. Only the console renderer knows how to + # lay out paths; the other formats would silently emit the full recursive xref + # list instead, which looks like a successful --paths run but is not one. + if fmt != "console": + raise click.UsageError( + f"--paths is only supported with --format console, not --format {fmt}. " + f"Drop --paths to emit the full cross-reference list as {fmt}." + ) recurse = True downloader = make_downloader(babel_url, local_dir, check_download) @@ -360,11 +368,15 @@ def xrefs( @cli.command("ids") @click.argument("curies", type=str, required=True, nargs=-1) @babel_options +@nodenorm_options +@click.option("--labels", is_flag=True, help="Include labels for CURIEs") @format_option def ids( curies: list[str], babel_url: str, + nodenorm_url: str, local_dir: str, + labels: bool, check_download: str, allow_version_mismatch: bool, fmt: str, @@ -383,14 +395,20 @@ def ids( :return: None """ - # No NodeNorm here, so there is no version to check against. - bxref = BabelXRefs(make_downloader(babel_url, local_dir, check_download)) - xrefs = bxref.get_curie_ids(curies) + downloader = make_downloader(babel_url, local_dir, check_download) + nodenorm = NodeNorm(nodenorm_url) + # NodeNorm is only consulted for labels, so only then can its Babel release differ. + if labels: + check_babel_versions(downloader, nodenorm, allow_version_mismatch) + + bxref = BabelXRefs(downloader, nodenorm) + xrefs = bxref.get_curie_ids(curies, label_curies=labels) if fmt == "console": console = make_console() for record in xrefs: - console.print(str(record)) + # Parquet values are arbitrary text; escape so they are not read as markup. + console.print(escape(str(record))) else: write_records(xrefs, fmt=fmt, indent=json_indent) diff --git a/tests/test_cli.py b/tests/test_cli.py index 88adeb7..e38e9c4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -140,7 +140,9 @@ def test_ids_happy_path(self): result = runner.invoke(cli, ["ids", "MONDO:0004979"]) assert result.exit_code == 0 - mock_bx.return_value.get_curie_ids.assert_called_once_with(("MONDO:0004979",)) + mock_bx.return_value.get_curie_ids.assert_called_once_with( + ("MONDO:0004979",), label_curies=False + ) def test_test_concord_happy_path(self): runner = CliRunner() @@ -575,3 +577,80 @@ def test_also_wrapped_for_ids(self): assert result.exit_code == 1 assert "nope" in result.output assert "Traceback" not in result.output + + +class TestPathsFormatGuard: + """--paths only has a renderer for the console format.""" + + @pytest.mark.parametrize("fmt", ["json", "tsv", "csv"]) + def test_rejected_for_non_console_formats(self, fmt): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs"), + patch("babel_explorer.cli.NodeNorm"), + ): + result = runner.invoke( + cli, ["xrefs", "A:1", "B:2", "--paths", "--format", fmt] + ) + + assert result.exit_code != 0 + assert "--paths is only supported with --format console" in result.output + # Rejected before anything is downloaded. + mock_dl.assert_not_called() + + def test_allowed_for_console(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = "2026jul22" + mock_nn.return_value.get_babel_version.return_value = "2026jul22" + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke(cli, ["xrefs", "A:1", "B:2", "--paths"]) + + assert result.exit_code == 0 + + +class TestIdsLabels: + """`ids --labels` enriches records via NodeNorm.""" + + @staticmethod + def _run(args, babel_version="2026jul22", nodenorm_version="2026jul22"): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = babel_version + mock_nn.return_value.get_babel_version.return_value = nodenorm_version + mock_bx.return_value.get_curie_ids.return_value = [ + IdentifierRecord(curie="MONDO:0004979", label="asthma") + ] + result = runner.invoke(cli, args) + return result, mock_bx, mock_nn + + def test_labels_flag_is_passed_through(self): + result, mock_bx, _ = self._run(["ids", "MONDO:0004979", "--labels"]) + assert result.exit_code == 0 + mock_bx.return_value.get_curie_ids.assert_called_once_with( + ("MONDO:0004979",), label_curies=True + ) + + def test_label_rendered_in_double_quotes(self): + result, _, _ = self._run(["ids", "MONDO:0004979", "--labels"]) + assert '"asthma"' in result.output + + def test_version_checked_only_with_labels(self): + _, _, mock_nn = self._run(["ids", "MONDO:0004979"], nodenorm_version="2025sep1") + mock_nn.return_value.get_babel_version.assert_not_called() + + def test_version_mismatch_fails_with_labels(self): + result, _, _ = self._run( + ["ids", "MONDO:0004979", "--labels"], nodenorm_version="2025sep1" + ) + assert result.exit_code != 0 + assert "2025sep1" in result.output From 2f133b768505f43c8f4cc2ffab14a151a5f7a265 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 18:46:27 -0400 Subject: [PATCH 095/123] Add --labels to the ids command `ids` had no NodeNorm integration at all, so IdentifierRecord output carried only the raw Identifiers.parquet columns and there was no way to see what a CURIE actually refers to without a second xrefs or test-concord call. IdentifierRecord grows a label field, populated from NodeNorm when --labels is passed, and rendered in double quotes immediately after the CURIE per the console output convention. As with xrefs, the Babel version check runs only when labels are requested, since that is the only time NodeNorm is consulted. An absent label is omitted from serialized output rather than emitted as an empty string, matching the console convention and keeping TSV/CSV columns stable for runs that did not ask for labels. Also escape ids console output as Rich markup: Parquet values are arbitrary text and a stray bracket would otherwise be swallowed as a style tag. Co-Authored-By: Claude Opus 5 --- README.md | 3 +++ src/babel_explorer/core/babel_xrefs.py | 37 ++++++++++++++++++++------ src/babel_explorer/formatting.py | 4 +++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6ecdb23..713be82 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ uv run babel-explorer xrefs MONDO:0004979 --labels # Get ID records for CURIEs uv run babel-explorer ids MONDO:0004979 +# Get ID records with labels from NodeNorm +uv run babel-explorer ids MONDO:0004979 --labels + # Test concordance changes with NodeNorm uv run babel-explorer test-concord MONDO:0004979 HP:0000001 ``` diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 00320f6..80d2910 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -59,19 +59,27 @@ class IdentifierRecord: curie: str extra_fields: tuple = () + label: str = "" @staticmethod - def from_row(row: tuple, column_names: list[str]): + def from_row(row: tuple, column_names: list[str], label: str = ""): """Create an IdentifierRecord from a DuckDB result row and its column names.""" curie_idx = column_names.index("curie") extra = tuple( (col, row[i]) for i, col in enumerate(column_names) if i != curie_idx ) - return IdentifierRecord(curie=row[curie_idx], extra_fields=extra) + return IdentifierRecord(curie=row[curie_idx], extra_fields=extra, label=label) def __str__(self): - """Return a ``key=value`` string of the CURIE and all extra fields.""" + """Return a ``key=value`` string of the CURIE, its label and all extra fields. + + The label sits immediately after the CURIE in double quotes, per the console + output convention, and is omitted entirely when absent. + """ parts = [f"curie={self.curie!r}"] + if self.label: + escaped = self.label.replace("\\", "\\\\").replace('"', '\\"') + parts.append(f'label="{escaped}"') for name, value in self.extra_fields: parts.append(f"{name}={value!r}") return f"IdentifierRecord({', '.join(parts)})" @@ -150,13 +158,20 @@ def _require_nodenorm(self): "label_curies=True requires a configured NodeNorm instance (nodenorm was None)." ) - def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: + def get_curie_ids( + self, curies: list[str], label_curies: bool = False + ) -> list[IdentifierRecord]: """ Search for all identifiers in the /ids/ files for a particular CURIE. :param curies: A list of CURIEs to search for. + :param label_curies: If ``True``, annotate each record with its NodeNorm label. + Requires a NodeNorm instance to have been passed to ``__init__``. + :raises ValueError: If ``label_curies=True`` but no NodeNorm instance is available. :return: A list of IdentifierRecords containing those CURIEs. """ + if label_curies: + self._require_nodenorm() identifier_parquet = self.downloader.get_downloaded_file( "duckdb/Identifiers.parquet" @@ -169,10 +184,16 @@ def get_curie_ids(self, curies: list[str]) -> list[IdentifierRecord]: [identifier_parquet, list(curies)], ) column_names = [desc[0] for desc in result.description] - return [ - IdentifierRecord.from_row(row, column_names) - for row in result.fetchall() - ] + rows = result.fetchall() + + records = [] + for row in rows: + record = IdentifierRecord.from_row(row, column_names) + if label_curies: + label = self.nodenorm.get_identifier(record.curie).label + record = dataclasses.replace(record, label=label) + records.append(record) + return records def get_curie_xref(self, curie: str, label_curies: bool = False): """Return all cross-references in Concord.parquet where *curie* is the subject or object. diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index 1f7716c..6600b7c 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -27,6 +27,10 @@ def _record_to_dict(record) -> dict[str, Any]: if "extra_fields" in d: for col, val in d.pop("extra_fields"): d[col] = val + # An absent label is omitted rather than emitted as "", matching the console + # convention and keeping TSV/CSV columns stable when labels were not requested. + if not d.get("label", True): + del d["label"] return d From 1e81660b7f3969cd8f7baea92f1746aa56bde8f1 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 19:17:32 -0400 Subject: [PATCH 096/123] Render CURIE labels through a single formatting helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `CURIE "label"` console convention had four independent implementations: _fmt_label and _curie_str in cli.py, an inline copy in each of the xrefs and test-concord console loops, and a hand-rolled escape in IdentifierRecord.__str__ that had already drifted (it escaped quotes but never rich markup). formatting.py now owns it via escape_label(), curie_with_label() and format_identifier_record(), so the convention and its escaping rules are defined once. IdentifierRecord loses the console __str__ it should never have carried in core/, and hl_curie/hl_curie_at_depth collapse into one depth-based function — the boolean variant was just depth 0 or None, and every call site was branching between the two. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/cli.py | 116 ++++++++++++------------- src/babel_explorer/core/babel_xrefs.py | 18 +--- src/babel_explorer/formatting.py | 54 +++++++++--- tests/test_formatting.py | 67 +++++++++++--- 4 files changed, 157 insertions(+), 98 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index ea6da9d..b43b772 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -16,10 +16,11 @@ from babel_explorer.core.downloader import BabelDownloader, MissingBabelFileError from babel_explorer.core.nodenorm import NodeNorm from babel_explorer.formatting import ( - _record_to_dict, + curie_with_label, + format_identifier_record, hl_curie, - hl_curie_at_depth, make_console, + record_to_dict, write_records, ) @@ -169,20 +170,9 @@ def parse_duration(value: str) -> int | float: return result -def _fmt_label(label: str) -> str: - """Escape a label for double-quoted display: backslashes first, then double quotes.""" - return escape(label.replace("\\", "\\\\").replace('"', '\\"')) - - -def _curie_str(curie: str, is_query: bool, depth: int | None, label: str | None) -> str: - """Build a Rich-marked-up CURIE string with optional label.""" - if is_query: - s = hl_curie(curie, True) - else: - s = hl_curie_at_depth(curie, depth) - if label: - s += f' "{_fmt_label(label)}"' - return s +def _depth_of(curie: str, query_set: set, depth: int | None) -> int | None: + """Depth to render a CURIE at: query CURIEs are always depth 0.""" + return 0 if curie in query_set else depth def _print_paths(console, curies, xrefs_list, labels: bool) -> None: @@ -196,8 +186,8 @@ def _print_paths(console, curies, xrefs_list, labels: bool) -> None: for from_c, to_c in combinations(curie_list, 2): path = find_shortest_path(from_c, to_c, xrefs_list) - header_from = hl_curie(from_c, True) - header_to = hl_curie(to_c, True) + header_from = hl_curie(from_c, 0) + header_to = hl_curie(to_c, 0) if path is None: console.print( @@ -221,14 +211,12 @@ def _print_paths(console, curies, xrefs_list, labels: bool) -> None: prev = nodes[-1] nodes.append(edge.obj if edge.subj == prev else edge.subj) - # Header: node1 → node2 → … → nodeN - node_strs = [] - for i, node in enumerate(nodes): - if node in query_set: - node_strs.append(hl_curie(node, True)) - else: - depth = i # position along path == depth from from_c - node_strs.append(hl_curie_at_depth(node, depth)) + # Header: node1 → node2 → … → nodeN. Position along the path is the depth + # from from_c, except for query CURIEs, which always render as depth 0. + node_strs = [ + hl_curie(node, _depth_of(node, query_set, i)) + for i, node in enumerate(nodes) + ] n_steps = len(path) step_word = "step" if n_steps == 1 else "steps" console.print( @@ -251,8 +239,12 @@ def _print_paths(console, curies, xrefs_list, labels: bool) -> None: subj_label = edge.obj_label obj_label = edge.subj_label - subj_str = _curie_str(subj_node, subj_node in query_set, i, subj_label) - obj_str = _curie_str(obj_node, obj_node in query_set, i + 1, obj_label) + subj_str = curie_with_label( + subj_node, _depth_of(subj_node, query_set, i), subj_label + ) + obj_str = curie_with_label( + obj_node, _depth_of(obj_node, query_set, i + 1), obj_label + ) console.print( f" - {subj_str} [dim]{escape(edge.pred)}[/dim] " @@ -344,19 +336,21 @@ def xrefs( _print_paths(console, curies, xref_list, labels) else: query_set = set(curies) - depth_map = build_depth_map(list(curies), xref_list) if recurse else None + # Without --recurse every result is one hop from a query CURIE, so there + # is no depth to show: only the query CURIEs themselves are highlighted. + depth_map = build_depth_map(list(curies), xref_list) if recurse else {} for xref in xref_list: - if depth_map is not None: - subj_str = hl_curie_at_depth(xref.subj, depth_map.get(xref.subj)) - obj_str = hl_curie_at_depth(xref.obj, depth_map.get(xref.obj)) - else: - subj_str = hl_curie(xref.subj, xref.subj in query_set) - obj_str = hl_curie(xref.obj, xref.obj in query_set) - if isinstance(xref, LabeledCrossReference): - if xref.subj_label: - subj_str += f' "{_fmt_label(xref.subj_label)}"' - if xref.obj_label: - obj_str += f' "{_fmt_label(xref.obj_label)}"' + labeled = isinstance(xref, LabeledCrossReference) + subj_str = curie_with_label( + xref.subj, + _depth_of(xref.subj, query_set, depth_map.get(xref.subj)), + xref.subj_label if labeled else None, + ) + obj_str = curie_with_label( + xref.obj, + _depth_of(xref.obj, query_set, depth_map.get(xref.obj)), + xref.obj_label if labeled else None, + ) console.print( f"{subj_str} [dim]{escape(xref.pred)}[/dim] " f"{obj_str} [dim italic]{escape(xref.filename)}[/dim italic]" @@ -407,8 +401,7 @@ def ids( if fmt == "console": console = make_console() for record in xrefs: - # Parquet values are arbitrary text; escape so they are not read as markup. - console.print(escape(str(record))) + console.print(format_identifier_record(record)) else: write_records(xrefs, fmt=fmt, indent=json_indent) @@ -424,25 +417,32 @@ def test_concord(curies, nodenorm_url, fmt, json_indent): run before and after a Babel rebuild to see how cliques would shift. """ nodenorm = NodeNorm(nodenorm_url) + + # Resolved once, before the format branch, so console and JSON report the same rows. + query_set = set(curies) + cliques = [ + (curie, ident) + for curie in curies + for ident in nodenorm.get_clique_identifiers(curie) + ] + if fmt == "console": console = make_console() - query_set = set(curies) - for curie in curies: - for ident in nodenorm.get_clique_identifiers(curie): - biolink = ", ".join(ident.biolink_type) - label_str = f' "{_fmt_label(ident.label)}"' if ident.label else "" - console.print( - f"{hl_curie(curie, True)} " - f"{hl_curie(ident.curie, ident.curie in query_set)}{label_str} " - f"[dim]{escape(biolink)}[/dim]" - ) + for curie, ident in cliques: + member = curie_with_label( + ident.curie, _depth_of(ident.curie, query_set, None), ident.label + ) + biolink = escape(", ".join(ident.biolink_type)) + console.print(f"{hl_curie(curie, 0)} {member} [dim]{biolink}[/dim]") else: - rows = [ - {"query_curie": curie, **_record_to_dict(ident)} - for curie in curies - for ident in nodenorm.get_clique_identifiers(curie) - ] - write_records(rows, fmt=fmt, indent=json_indent) + write_records( + [ + {"query_curie": curie, **record_to_dict(ident)} + for curie, ident in cliques + ], + fmt=fmt, + indent=json_indent, + ) if __name__ == "__main__": diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 80d2910..835e898 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -62,27 +62,13 @@ class IdentifierRecord: label: str = "" @staticmethod - def from_row(row: tuple, column_names: list[str], label: str = ""): + def from_row(row: tuple, column_names: list[str]): """Create an IdentifierRecord from a DuckDB result row and its column names.""" curie_idx = column_names.index("curie") extra = tuple( (col, row[i]) for i, col in enumerate(column_names) if i != curie_idx ) - return IdentifierRecord(curie=row[curie_idx], extra_fields=extra, label=label) - - def __str__(self): - """Return a ``key=value`` string of the CURIE, its label and all extra fields. - - The label sits immediately after the CURIE in double quotes, per the console - output convention, and is omitted entirely when absent. - """ - parts = [f"curie={self.curie!r}"] - if self.label: - escaped = self.label.replace("\\", "\\\\").replace('"', '\\"') - parts.append(f'label="{escaped}"') - for name, value in self.extra_fields: - parts.append(f"{name}={value!r}") - return f"IdentifierRecord({', '.join(parts)})" + return IdentifierRecord(curie=row[curie_idx], extra_fields=extra) def build_depth_map(query_curies: list[str], xrefs: list) -> dict[str, int]: diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index 6600b7c..e5f7a2d 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -2,7 +2,7 @@ Provides: - write_records() for machine-readable output (json, tsv, csv) -- make_console() and hl_curie() for rich console output +- make_console(), hl_curie() and curie_with_label() for rich console output """ import csv @@ -15,7 +15,7 @@ from rich.markup import escape -def _record_to_dict(record) -> dict[str, Any]: +def record_to_dict(record) -> dict[str, Any]: """Convert a dataclass (or plain dict) to a flat dict. Handles IdentifierRecord's extra_fields, which asdict() returns as a @@ -50,12 +50,6 @@ def make_console(file=None) -> Console: return Console(file=file, highlight=False) -def hl_curie(curie: str, highlight: bool) -> str: - """Return rich markup for a CURIE — bold cyan if it is a query CURIE.""" - escaped = escape(curie) - return f"[bold cyan]{escaped}[/bold cyan]" if highlight else escaped - - # Styles indexed by BFS depth from the nearest query CURIE. # Depth 0 = the query term itself; higher = further away. _DEPTH_STYLES = [ @@ -67,10 +61,11 @@ def hl_curie(curie: str, highlight: bool) -> str: ] -def hl_curie_at_depth(curie: str, depth: int | None) -> str: +def hl_curie(curie: str, depth: int | None) -> str: """Return rich markup for a CURIE colored by its BFS depth from the nearest query CURIE. - Pass ``depth=None`` for CURIEs whose depth is unknown (rendered unstyled). + Depth 0 is a query CURIE itself. Pass ``depth=None`` for CURIEs whose depth is + unknown or irrelevant (rendered unstyled). """ escaped = escape(curie) if depth is None: @@ -79,6 +74,41 @@ def hl_curie_at_depth(curie: str, depth: int | None) -> str: return f"[{style}]{escaped}[/{style}]" +def escape_label(label: str) -> str: + """Escape a label for display inside double quotes: backslashes first, then quotes. + + Downstream tools can parse the result with the regex ``"([^"\\\\]|\\\\.)*"``. + """ + return label.replace("\\", "\\\\").replace('"', '\\"') + + +def curie_with_label(curie: str, depth: int | None, label: str | None = None) -> str: + """Render a CURIE as rich markup, followed by its label in double quotes. + + The sole implementation of the console label convention: the label sits + immediately after the CURIE in double quotes, and is omitted entirely when + absent rather than rendered as a placeholder. + """ + markup = hl_curie(curie, depth) + if label: + markup += f' "{escape(escape_label(label))}"' + return markup + + +def format_identifier_record(record) -> str: + """Render an IdentifierRecord as a ``key=value`` line of rich markup. + + The label sits immediately after the CURIE in double quotes and is omitted + entirely when absent, per the console convention. + """ + parts = [f"curie={record.curie!r}"] + if record.label: + parts.append(f'label="{escape_label(record.label)}"') + parts.extend(f"{name}={value!r}" for name, value in record.extra_fields) + # Parquet values are arbitrary text; escape so they are not read as markup. + return escape(f"IdentifierRecord({', '.join(parts)})") + + def write_records(records, fmt: str, indent: int = 2, file=None): """Write an iterable of dataclass records (or dicts) in the requested format. @@ -94,14 +124,14 @@ def write_records(records, fmt: str, indent: int = 2, file=None): records = list(records) if fmt == "json": - rows = [_record_to_dict(r) for r in records] + rows = [record_to_dict(r) for r in records] json.dump(rows, file, indent=indent, default=str) print(file=file) # trailing newline elif fmt in ("tsv", "csv"): if not records: return - rows = [_flatten_for_tabular(_record_to_dict(r)) for r in records] + rows = [_flatten_for_tabular(record_to_dict(r)) for r in records] delimiter = "\t" if fmt == "tsv" else "," writer = csv.DictWriter( file, diff --git a/tests/test_formatting.py b/tests/test_formatting.py index 6a42855..c660fc5 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -15,9 +15,12 @@ ) from babel_explorer.core.nodenorm import Identifier from babel_explorer.formatting import ( - _record_to_dict, + curie_with_label, + escape_label, + format_identifier_record, hl_curie, make_console, + record_to_dict, write_records, ) @@ -84,12 +87,12 @@ def test_make_console_accepts_file(self): assert "hello" in out.getvalue() def test_hl_curie_highlighted_contains_markup(self): - result = hl_curie("HGNC:1100", highlight=True) + result = hl_curie("HGNC:1100", 0) assert "bold cyan" in result assert "HGNC:1100" in result def test_hl_curie_not_highlighted_is_plain(self): - result = hl_curie("HGNC:1100", highlight=False) + result = hl_curie("HGNC:1100", None) assert result == "HGNC:1100" assert "[" not in result @@ -97,27 +100,27 @@ def test_hl_curie_highlighted_renders_correctly(self): """Markup renders to plain text on a non-TTY console.""" out = io.StringIO() console = Console(file=out, highlight=False, no_color=True) - console.print(hl_curie("HGNC:1100", highlight=True)) + console.print(hl_curie("HGNC:1100", 0)) assert "HGNC:1100" in out.getvalue() def test_hl_curie_highlighted_renders_with_color(self): """On a forced-TTY console, ANSI codes are emitted.""" out = io.StringIO() console = Console(file=out, highlight=False, force_terminal=True) - console.print(hl_curie("HGNC:1100", highlight=True)) + console.print(hl_curie("HGNC:1100", 0)) output = out.getvalue() assert "HGNC:1100" in output assert "\x1b[" in output # ANSI escape present # --------------------------------------------------------------------------- -# Tests for _record_to_dict +# Tests for record_to_dict # --------------------------------------------------------------------------- class TestRecordToDict: def test_cross_reference(self, xref): - d = _record_to_dict(xref) + d = record_to_dict(xref) assert d == { "filename": "Concord.parquet", "subj": "A:1", @@ -126,7 +129,7 @@ def test_cross_reference(self, xref): } def test_labeled_cross_reference_has_all_eight_fields(self, labeled_xref): - d = _record_to_dict(labeled_xref) + d = record_to_dict(labeled_xref) assert set(d.keys()) == { "filename", "subj", @@ -142,7 +145,7 @@ def test_labeled_cross_reference_has_all_eight_fields(self, labeled_xref): assert d["obj_biolink_type"] == ("biolink:Gene", "biolink:NamedThing") def test_identifier_record_extra_fields_expanded(self, id_record): - d = _record_to_dict(id_record) + d = record_to_dict(id_record) assert "extra_fields" not in d assert d["curie"] == "A:1" assert d["type"] == "gene" @@ -150,15 +153,15 @@ def test_identifier_record_extra_fields_expanded(self, id_record): def test_identifier_record_no_extra_fields(self): rec = IdentifierRecord(curie="X:1") - d = _record_to_dict(rec) + d = record_to_dict(rec) assert d == {"curie": "X:1"} def test_plain_dict_passthrough(self): data = {"a": 1, "b": "hello"} - assert _record_to_dict(data) is data + assert record_to_dict(data) is data def test_identifier_dataclass(self, identifier): - d = _record_to_dict(identifier) + d = record_to_dict(identifier) assert d["curie"] == "MONDO:0004979" assert d["label"] == "asthma" # dataclasses.asdict() preserves tuple types @@ -289,3 +292,43 @@ def test_unknown_format_raises_value_error(self, xref): out = io.StringIO() with pytest.raises(ValueError, match="Unknown format"): write_records([xref], "xml", file=out) + + +# --------------------------------------------------------------------------- +# Tests for the label convention (escape_label / curie_with_label) +# --------------------------------------------------------------------------- + + +class TestLabelConvention: + """CLAUDE.md: a label follows its CURIE in double quotes, or is omitted.""" + + def test_label_follows_curie_in_double_quotes(self): + assert curie_with_label("MONDO:1", None, "asthma") == 'MONDO:1 "asthma"' + + @pytest.mark.parametrize("label", [None, ""]) + def test_absent_label_is_omitted_entirely(self, label): + """No placeholder — a CURIE with no label renders as the bare CURIE.""" + assert curie_with_label("MONDO:1", None, label) == "MONDO:1" + + def test_backslashes_escape_before_quotes(self): + assert escape_label(r'a\b"c') == r"a\\b\"c" + + def test_escaped_label_matches_the_documented_regex(self): + import re + + rendered = curie_with_label("MONDO:1", None, r'say "hi" \ bye') + assert re.search(r'"([^"\\]|\\.)*"', rendered).group(0) == ( + r'"say \"hi\" \\ bye"' + ) + + def test_query_curie_is_highlighted_at_depth_zero(self): + assert curie_with_label("MONDO:1", 0, "asthma").startswith("[bold cyan]") + + def test_identifier_record_uses_the_same_convention(self): + rec = IdentifierRecord(curie="A:1", extra_fields=(("n", 1),), label='a"b') + rendered = format_identifier_record(rec) + assert r'label="a\"b"' in rendered + + def test_identifier_record_omits_absent_label(self): + rec = IdentifierRecord(curie="A:1", extra_fields=(("n", 1),)) + assert "label=" not in format_identifier_record(rec) From d608957ef36b40ce6a676a5d57ec3b54f52e3e32 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 19:17:46 -0400 Subject: [PATCH 097/123] Keep tabular output valid when records omit an absent label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_records took its CSV/TSV field names from the first row alone, so any run where one record carried a label and another did not raised a ValueError inside DictWriter on the first record with an extra key. This was already reachable via `ids --labels` whenever NodeNorm knew some CURIEs but not others. Field names are now the union of keys across all rows, with restval="" filling the gaps. The omit-an-absent-label rule also moves off the literal field name "label" and onto any field ending in it, so LabeledCrossReference's subj_label/obj_label follow the same convention that ids already did — they were being emitted as "" in JSON and TSV. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/formatting.py | 12 ++++++--- tests/test_formatting.py | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index e5f7a2d..5b0d744 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -29,8 +29,10 @@ def record_to_dict(record) -> dict[str, Any]: d[col] = val # An absent label is omitted rather than emitted as "", matching the console # convention and keeping TSV/CSV columns stable when labels were not requested. - if not d.get("label", True): - del d["label"] + # Keyed on the concept, not one field name: LabeledCrossReference spells it + # subj_label/obj_label, and those must follow the same rule. + for key in [k for k, v in d.items() if k.endswith("label") and not v]: + del d[key] return d @@ -132,10 +134,14 @@ def write_records(records, fmt: str, indent: int = 2, file=None): if not records: return rows = [_flatten_for_tabular(record_to_dict(r)) for r in records] + # Union of keys, in first-seen order: records that omit an absent label have + # fewer keys than their neighbours, and DictWriter rejects any key not declared. + fieldnames = list(dict.fromkeys(k for row in rows for k in row)) delimiter = "\t" if fmt == "tsv" else "," writer = csv.DictWriter( file, - fieldnames=list(rows[0].keys()), + fieldnames=fieldnames, + restval="", delimiter=delimiter, lineterminator="\n", ) diff --git a/tests/test_formatting.py b/tests/test_formatting.py index c660fc5..f544338 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -332,3 +332,48 @@ def test_identifier_record_uses_the_same_convention(self): def test_identifier_record_omits_absent_label(self): rec = IdentifierRecord(curie="A:1", extra_fields=(("n", 1),)) assert "label=" not in format_identifier_record(rec) + + +class TestLabelOmissionInRecords: + """The omit-when-absent rule is keyed on the concept, not one field name.""" + + def test_empty_subj_and_obj_labels_are_dropped(self): + xref = LabeledCrossReference( + filename="f", + subj="A:1", + pred="p", + obj="B:2", + subj_label="", + subj_biolink_type=(), + obj_label="", + obj_biolink_type=(), + ) + d = record_to_dict(xref) + assert "subj_label" not in d and "obj_label" not in d + + def test_present_labels_are_kept(self): + xref = LabeledCrossReference( + filename="f", + subj="A:1", + pred="p", + obj="B:2", + subj_label="asthma", + subj_biolink_type=(), + obj_label="", + obj_biolink_type=(), + ) + d = record_to_dict(xref) + assert d["subj_label"] == "asthma" and "obj_label" not in d + + def test_tabular_output_survives_rows_with_differing_keys(self): + """A labelled row after an unlabelled one must not blow up DictWriter.""" + rows = [ + IdentifierRecord(curie="A:1", extra_fields=()), + IdentifierRecord(curie="B:2", extra_fields=(), label="asthma"), + ] + out = io.StringIO() + write_records(rows, "csv", file=out) + lines = out.getvalue().splitlines() + assert lines[0] == "curie,label" + assert lines[1] == "A:1," + assert lines[2] == "B:2,asthma" From 27979b4195e7cc343b6b6c5a852e9c4432c48ece Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 19:17:59 -0400 Subject: [PATCH 098/123] Batch NodeNorm lookups and Parquet scans Three N+1 patterns dominated runtime on anything larger than a toy query: - Every labelled CURIE cost its own get_normalized_nodes round-trip, so `xrefs --labels --recurse` over a 500-CURIE clique issued ~500 sequential HTTPS requests. NodeNorm.normalize_curies() now prefetches a whole batch (100 CURIEs per request) and the per-CURIE accessors serve from cache. - Multi-CURIE `xrefs` ran one full scan of the multi-gigabyte Concord.parquet per CURIE. One scan now matches every CURIE, with results bucketed back into the per-CURIE cache; a CURIE with no cross-references caches an empty list so it is not rescanned. - _print_paths rebuilt the undirected neighbour map for each of the C(n,2) pairs, and build_depth_map built the same structure a third time. All three share one build_adjacency(). Co-Authored-By: Claude Opus 5 --- src/babel_explorer/cli.py | 7 +- src/babel_explorer/core/babel_xrefs.py | 146 +++++++++++++++++-------- src/babel_explorer/core/nodenorm.py | 71 ++++++++---- tests/test_babel_xrefs.py | 87 ++++++++++++++- tests/test_nodenorm.py | 83 +++++++++++++- 5 files changed, 323 insertions(+), 71 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index b43b772..48c2ca4 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -10,6 +10,7 @@ from babel_explorer.core.babel_xrefs import ( BabelXRefs, LabeledCrossReference, + build_adjacency, build_depth_map, find_shortest_path, ) @@ -183,9 +184,12 @@ def _print_paths(console, curies, xrefs_list, labels: bool) -> None: return query_set = set(curie_list) + # One neighbour map for every pair: rebuilding it per pair re-walks the whole + # recursive xref list C(n,2) times. + adj = build_adjacency(xrefs_list) for from_c, to_c in combinations(curie_list, 2): - path = find_shortest_path(from_c, to_c, xrefs_list) + path = find_shortest_path(from_c, to_c, xrefs_list, adj) header_from = hl_curie(from_c, 0) header_to = hl_curie(to_c, 0) @@ -417,6 +421,7 @@ def test_concord(curies, nodenorm_url, fmt, json_indent): run before and after a Babel rebuild to see how cliques would shift. """ nodenorm = NodeNorm(nodenorm_url) + nodenorm.normalize_curies(curies) # Resolved once, before the format branch, so console and JSON report the same rows. query_set = set(curies) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 835e898..38d7210 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -71,19 +71,36 @@ def from_row(row: tuple, column_names: list[str]): return IdentifierRecord(curie=row[curie_idx], extra_fields=extra) -def build_depth_map(query_curies: list[str], xrefs: list) -> dict[str, int]: - """BFS from query_curies over xref edges; returns {curie: depth_from_nearest_query}.""" - adj: dict[str, list[str]] = {} +def build_adjacency(xrefs: list) -> dict[str, list]: + """Build the undirected neighbour map ``{curie: [(neighbor, xref), ...]}``. + + Cross-references are stored in one direction but traversed in both, so each + edge contributes an entry to both of its endpoints. Build this once and share + it: it is O(len(xrefs)) and the recursive expansion can return 10^5+ edges. + """ + adj: dict[str, list] = {} for xref in xrefs: - adj.setdefault(xref.subj, []).append(xref.obj) - adj.setdefault(xref.obj, []).append(xref.subj) + adj.setdefault(xref.subj, []).append((xref.obj, xref)) + adj.setdefault(xref.obj, []).append((xref.subj, xref)) + return adj + + +def build_depth_map( + query_curies: list[str], xrefs: list, adj: dict[str, list] | None = None +) -> dict[str, int]: + """BFS from query_curies over xref edges; returns {curie: depth_from_nearest_query}. + + Pass *adj* from ``build_adjacency`` to reuse an already-built neighbour map. + """ + if adj is None: + adj = build_adjacency(xrefs) depths: dict[str, int] = {c: 0 for c in query_curies} frontier = list(query_curies) while frontier: next_frontier = [] for node in frontier: - for neighbor in adj.get(node, []): + for neighbor, _ in adj.get(node, []): if neighbor not in depths: depths[neighbor] = depths[node] + 1 next_frontier.append(neighbor) @@ -91,20 +108,23 @@ def build_depth_map(query_curies: list[str], xrefs: list) -> dict[str, int]: return depths -def find_shortest_path(from_curie: str, to_curie: str, xrefs: list) -> list | None: +def find_shortest_path( + from_curie: str, to_curie: str, xrefs: list, adj: dict[str, list] | None = None +) -> list | None: """Return the shortest list of CrossReference edges from from_curie to to_curie. Returns ``[]`` if from_curie == to_curie, or ``None`` if no path exists. The returned edges may be stored in either direction; callers should check ``edge.subj`` / ``edge.obj`` against the expected traversal direction. + + Pass *adj* from ``build_adjacency`` to reuse an already-built neighbour map; + callers resolving many pairs over one graph should always do so. """ if from_curie == to_curie: return [] - adj: dict[str, list] = {} - for xref in xrefs: - adj.setdefault(xref.subj, []).append((xref.obj, xref)) - adj.setdefault(xref.obj, []).append((xref.subj, xref)) + if adj is None: + adj = build_adjacency(xrefs) visited = {from_curie} queue: deque = deque([(from_curie, [])]) @@ -172,13 +192,15 @@ def get_curie_ids( column_names = [desc[0] for desc in result.description] rows = result.fetchall() - records = [] - for row in rows: - record = IdentifierRecord.from_row(row, column_names) - if label_curies: - label = self.nodenorm.get_identifier(record.curie).label - record = dataclasses.replace(record, label=label) - records.append(record) + records = [IdentifierRecord.from_row(row, column_names) for row in rows] + if label_curies: + self.nodenorm.normalize_curies({r.curie for r in records}) + records = [ + dataclasses.replace( + r, label=self.nodenorm.get_identifier(r.curie).label + ) + for r in records + ] return records def get_curie_xref(self, curie: str, label_curies: bool = False): @@ -193,40 +215,74 @@ def get_curie_xref(self, curie: str, label_curies: bool = False): :return: A list of ``CrossReference`` (or ``LabeledCrossReference``) objects. """ cache_key = (curie, label_curies) - if cache_key in self._xref_cache: - return self._xref_cache[cache_key] + if cache_key not in self._xref_cache: + self._query_xrefs([curie], label_curies) + return self._xref_cache[cache_key] + def _query_xrefs(self, curies: list[str], label_curies: bool = False) -> list: + """Fetch the direct cross-references for *curies* in a single Parquet scan. + + Concord.parquet is multi-gigabyte, so one scan matching every CURIE at once + costs a fraction of one scan per CURIE. Results are bucketed back into the + per-CURIE cache that ``get_curie_xref`` reads. + """ if label_curies: self._require_nodenorm() + if not curies: + return [] concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") with duckdb.connect() as db: xref_tuples = db.execute( - "SELECT filename, subj, pred, obj FROM read_parquet($1) WHERE subj=$2 OR obj=$2", - [concord_parquet, curie], + """ + SELECT filename, subj, pred, obj FROM read_parquet($1) + WHERE subj IN (SELECT unnest($2::VARCHAR[])) + OR obj IN (SELECT unnest($2::VARCHAR[])) + """, + [concord_parquet, list(curies)], ).fetchall() xrefs = [CrossReference.from_tuple(rec) for rec in xref_tuples] if label_curies: - xrefs = [self._to_labeled_xref(xref) for xref in xrefs] - self._xref_cache[cache_key] = xrefs + xrefs = self._to_labeled_xrefs(xrefs) + + # Bucket per query CURIE so get_curie_xref's cache stays exact: a CURIE with + # no cross-references must cache an empty list, not stay absent. + wanted = set(curies) + buckets: dict[str, list] = {curie: [] for curie in wanted} + for xref in xrefs: + for curie in xref.curies & wanted: + buckets[curie].append(xref) + for curie, bucket in buckets.items(): + self._xref_cache[(curie, label_curies)] = bucket return xrefs - def _to_labeled_xref(self, xref: CrossReference) -> LabeledCrossReference: - """Convert a CrossReference to a LabeledCrossReference using NodeNorm.""" - subj_ident = self.nodenorm.get_identifier(xref.subj) - obj_ident = self.nodenorm.get_identifier(xref.obj) - return LabeledCrossReference( - subj=xref.subj, - obj=xref.obj, - filename=xref.filename, - pred=xref.pred, - subj_label=subj_ident.label, - subj_biolink_type=subj_ident.biolink_type, - obj_label=obj_ident.label, - obj_biolink_type=obj_ident.biolink_type, - ) + def _to_labeled_xrefs(self, xrefs: list) -> list[LabeledCrossReference]: + """Annotate cross-references with NodeNorm labels and Biolink types. + + Every CURIE in the batch is normalised up front in a handful of requests; + the per-edge ``get_identifier`` calls below are then served from cache. + """ + self.nodenorm.normalize_curies({c for xref in xrefs for c in xref.curies}) + + labeled = [] + for xref in xrefs: + subj = self.nodenorm.get_identifier(xref.subj) + obj = self.nodenorm.get_identifier(xref.obj) + labeled.append( + LabeledCrossReference( + subj=xref.subj, + obj=xref.obj, + filename=xref.filename, + pred=xref.pred, + subj_label=subj.label, + subj_biolink_type=subj.biolink_type, + obj_label=obj.label, + obj_biolink_type=obj.biolink_type, + ) + ) + return labeled def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = False): """Traverse the cross-reference graph in one DuckDB WITH RECURSIVE query.""" @@ -268,7 +324,7 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal xrefs = [CrossReference.from_tuple(row) for row in rows] if label_curies: - xrefs = [self._to_labeled_xref(xref) for xref in xrefs] + xrefs = self._to_labeled_xrefs(xrefs) return xrefs @@ -287,9 +343,13 @@ def get_curie_xrefs( if recurse: return self._get_curie_xrefs_recursive(curies, label_curies) - xrefs = set() - for curie in curies: - logging.info(f"Searching for cross-references for {curie}") - xrefs.update(self.get_curie_xref(curie, label_curies)) + logging.info(f"Searching for cross-references for {', '.join(curies)}") + + # One Parquet scan covers every CURIE not already cached; the scan returns the + # union of their cross-references, so only cached CURIEs need adding separately. + uncached = [c for c in curies if (c, label_curies) not in self._xref_cache] + xrefs = set(self._query_xrefs(uncached, label_curies)) + for curie in set(curies) - set(uncached): + xrefs.update(self._xref_cache[(curie, label_curies)]) return sorted(xrefs) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index 866b207..4f1e536 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -5,6 +5,11 @@ import requests +#: Maximum CURIEs per get_normalized_nodes request. Keeps the query string well +#: inside the usual 8 KB server limit while collapsing hundreds of lookups into +#: a handful of round-trips. +NORMALIZE_BATCH_SIZE = 100 + @dataclasses.dataclass(frozen=True) class Identifier: @@ -113,26 +118,16 @@ def get_identifier(self, curie: str) -> "Identifier": self._identifier_cache[curie] = ident return ident - def normalize_curie(self, curie: str): - """Call ``get_normalized_nodes`` and return the per-CURIE result dict. + def _fetch_normalized(self, curies: list[str]) -> None: + """Fetch *curies* from ``get_normalized_nodes`` and populate the cache. - :return: The normalisation dict for *curie* (contains ``id``, ``equivalent_identifiers``, - ``type``, etc.), or ``None`` if the CURIE is not recognised by NodeNorm. :raises requests.HTTPError: If the API returns a non-2xx status code. - - Results are cached per instance. HTTP errors are not cached. + Nothing is cached in that case, so the lookup is retried next time. """ - if curie in self._normalize_cache: - return self._normalize_cache[curie] - - if not self.nodenorm_url: - self._normalize_cache[curie] = None - return None - response = requests.get( f"{self.nodenorm_url}get_normalized_nodes", params={ - "curie": curie, + "curie": curies, "conflate": True, "drug_chemical_conflate": True, "description": True, @@ -144,16 +139,46 @@ def normalize_curie(self, curie: str): response.raise_for_status() result = response.json() - try: - value = result[curie] - except KeyError: - logging.debug( - f"NodeNorm response did not contain CURIE {curie!r}; returning None" - ) - value = None + for curie in curies: + if curie not in result: + logging.debug( + f"NodeNorm response did not contain CURIE {curie!r}; caching None" + ) + # NodeNorm reports an unrecognised CURIE as a null value, not a missing key. + self._normalize_cache[curie] = result.get(curie) - self._normalize_cache[curie] = value - return value + def normalize_curies(self, curies) -> None: + """Populate the normalisation cache for *curies* in as few requests as possible. + + Callers that are about to look up many CURIEs should call this first: the + per-CURIE accessors then hit the cache instead of issuing one HTTPS + round-trip each, which dominates runtime on a large clique. + + :raises requests.HTTPError: If the API returns a non-2xx status code. + """ + missing = sorted({c for c in curies if c not in self._normalize_cache}) + if not missing: + return + + if not self.nodenorm_url: + self._normalize_cache.update(dict.fromkeys(missing)) + return + + for i in range(0, len(missing), NORMALIZE_BATCH_SIZE): + self._fetch_normalized(missing[i : i + NORMALIZE_BATCH_SIZE]) + + def normalize_curie(self, curie: str): + """Call ``get_normalized_nodes`` and return the per-CURIE result dict. + + :return: The normalisation dict for *curie* (contains ``id``, ``equivalent_identifiers``, + ``type``, etc.), or ``None`` if the CURIE is not recognised by NodeNorm. + :raises requests.HTTPError: If the API returns a non-2xx status code. + + Results are cached per instance. HTTP errors are not cached. + """ + if curie not in self._normalize_cache: + self.normalize_curies([curie]) + return self._normalize_cache[curie] def get_clique_identifiers(self, curie: str) -> list[Identifier]: """Return all ``Identifier`` objects in the NodeNorm clique for *curie*. diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 10a8486..29d4cb3 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -13,6 +13,9 @@ CrossReference, IdentifierRecord, LabeledCrossReference, + build_adjacency, + build_depth_map, + find_shortest_path, ) from babel_explorer.core.downloader import BabelDownloader from babel_explorer.core.nodenorm import NodeNorm @@ -237,7 +240,7 @@ def test_get_curie_xref_calls_downloader(self, tmp_path): def test_get_curie_xrefs_no_expand(self, tmp_path): bx = self._make_bx(tmp_path) xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") - with patch.object(bx, "get_curie_xref", return_value=[xr]): + with patch.object(bx, "_query_xrefs", return_value=[xr]): result = bx.get_curie_xrefs(["A:1"], recurse=False) assert len(result) == 1 assert result[0] == xr @@ -300,11 +303,91 @@ def test_results_are_sorted(self, tmp_path): xr_b = CrossReference(filename="b", subj="B:1", pred="p", obj="C:1") xr_a = CrossReference(filename="a", subj="A:1", pred="p", obj="B:1") - with patch.object(bx, "get_curie_xref", return_value=[xr_b, xr_a]): + with patch.object(bx, "_query_xrefs", return_value=[xr_b, xr_a]): result = bx.get_curie_xrefs(["X:1"], recurse=False) assert result == [xr_a, xr_b] +class TestQueryXrefsBatching: + """Multi-CURIE lookups must cost one Parquet scan, not one per CURIE.""" + + @staticmethod + def _mock_db(rows): + result = MagicMock() + result.fetchall.return_value = rows + db = MagicMock() + db.__enter__.return_value = db + db.execute.return_value = result + return db + + def _run(self, tmp_path, curies, rows): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + bx = BabelXRefs(dl) + db = self._mock_db(rows) + with patch.object(bx.downloader, "get_downloaded_file", return_value="/fake"): + with patch( + "babel_explorer.core.babel_xrefs.duckdb.connect", return_value=db + ): + return bx, bx.get_curie_xrefs(curies), db + + def test_one_scan_for_many_curies(self, tmp_path): + rows = [ + ("f", "A:1", "p", "B:2"), + ("f", "C:3", "p", "D:4"), + ] + _, result, db = self._run(tmp_path, ["A:1", "C:3"], rows) + assert db.execute.call_count == 1 + assert len(result) == 2 + + def test_results_are_bucketed_per_curie(self, tmp_path): + rows = [ + ("f", "A:1", "p", "B:2"), + ("f", "C:3", "p", "D:4"), + ] + bx, _, _ = self._run(tmp_path, ["A:1", "C:3"], rows) + assert [x.obj for x in bx._xref_cache[("A:1", False)]] == ["B:2"] + assert [x.obj for x in bx._xref_cache[("C:3", False)]] == ["D:4"] + + def test_curie_with_no_xrefs_caches_an_empty_list(self, tmp_path): + """An absent bucket would silently re-scan the whole Parquet file.""" + bx, result, _ = self._run( + tmp_path, ["A:1", "LONELY:9"], [("f", "A:1", "p", "B:2")] + ) + assert bx._xref_cache[("LONELY:9", False)] == [] + assert len(result) == 1 + + def test_cached_curies_are_not_rescanned(self, tmp_path): + bx, _, db = self._run(tmp_path, ["A:1"], [("f", "A:1", "p", "B:2")]) + with patch.object(bx.downloader, "get_downloaded_file", return_value="/fake"): + with patch( + "babel_explorer.core.babel_xrefs.duckdb.connect", return_value=db + ) as mock_connect: + again = bx.get_curie_xrefs(["A:1"]) + mock_connect.assert_not_called() + assert [x.obj for x in again] == ["B:2"] + + +class TestBuildAdjacency: + """The neighbour map is shared by the depth BFS and the path search.""" + + def test_edges_are_traversable_in_both_directions(self): + xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + adj = build_adjacency([xr]) + assert adj["A:1"] == [("B:2", xr)] + assert adj["B:2"] == [("A:1", xr)] + + def test_shared_adjacency_matches_a_freshly_built_one(self): + xrefs = [ + CrossReference(filename="f", subj="A:1", pred="p", obj="B:2"), + CrossReference(filename="f", subj="B:2", pred="p", obj="C:3"), + ] + adj = build_adjacency(xrefs) + assert find_shortest_path("A:1", "C:3", xrefs, adj) == find_shortest_path( + "A:1", "C:3", xrefs + ) + assert build_depth_map(["A:1"], xrefs, adj) == build_depth_map(["A:1"], xrefs) + + # ========================================================================== # Integration Tests — require downloaded Parquet files # ========================================================================== diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py index dbdc30d..47c58fc 100644 --- a/tests/test_nodenorm.py +++ b/tests/test_nodenorm.py @@ -9,7 +9,7 @@ import pytest import requests -from babel_explorer.core.nodenorm import Identifier, NodeNorm +from babel_explorer.core.nodenorm import NORMALIZE_BATCH_SIZE, Identifier, NodeNorm from tests.constants import load_curies VALID_CURIES = load_curies() @@ -168,7 +168,8 @@ def test_correct_api_endpoint_and_params(self): mock_get.assert_called_once() args, kwargs = mock_get.call_args assert args[0] == "https://example.com/get_normalized_nodes" - assert kwargs["params"]["curie"] == "X:1" + # CURIEs are always sent as a batch, even when there is only one. + assert kwargs["params"]["curie"] == ["X:1"] def test_returns_result_for_curie(self): nn = self._make_nn() @@ -406,3 +407,81 @@ def test_normalize_curie_nonexistent(nodenorm): """A made-up CURIE returns None.""" result = nodenorm.normalize_curie("FAKENS:9999999999") assert result is None + + +class TestNormalizeCuriesBatching: + """Many CURIEs must cost a handful of requests, not one each.""" + + @staticmethod + def _resp(payload): + r = Mock() + r.json.return_value = payload + r.raise_for_status = Mock() + return r + + def test_one_request_for_many_curies(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + curies = [f"X:{i}" for i in range(50)] + payload = {c: {"id": {"identifier": c}} for c in curies} + + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._resp(payload), + ) as mock_get: + nn.normalize_curies(curies) + assert mock_get.call_count == 1 + assert sorted(mock_get.call_args.kwargs["params"]["curie"]) == sorted( + curies + ) + + def test_chunks_above_the_batch_size(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + curies = [f"X:{i:04d}" for i in range(NORMALIZE_BATCH_SIZE * 2 + 1)] + + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._resp({}), + ) as mock_get: + nn.normalize_curies(curies) + assert mock_get.call_count == 3 + sizes = [len(c.kwargs["params"]["curie"]) for c in mock_get.call_args_list] + assert sizes == [NORMALIZE_BATCH_SIZE, NORMALIZE_BATCH_SIZE, 1] + + def test_prefetched_curies_are_not_refetched(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + payload = {"X:1": {"id": {"identifier": "X:1"}}} + + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._resp(payload), + ) as mock_get: + nn.normalize_curies(["X:1"]) + nn.normalize_curie("X:1") + nn.get_identifier("X:1") + assert mock_get.call_count == 1 + + def test_unrecognised_curie_caches_none(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + with patch( + "babel_explorer.core.nodenorm.requests.get", return_value=self._resp({}) + ) as mock_get: + nn.normalize_curies(["MISSING:1"]) + assert nn.normalize_curie("MISSING:1") is None + assert mock_get.call_count == 1 + + def test_offline_mode_makes_no_requests(self): + nn = NodeNorm(nodenorm_url="") + with patch("babel_explorer.core.nodenorm.requests.get") as mock_get: + nn.normalize_curies(["X:1", "X:2"]) + assert nn.normalize_curie("X:1") is None + mock_get.assert_not_called() + + def test_http_error_is_not_cached(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + bad = Mock() + bad.raise_for_status.side_effect = requests.HTTPError("500 Server Error") + + with patch("babel_explorer.core.nodenorm.requests.get", return_value=bad): + with pytest.raises(requests.HTTPError): + nn.normalize_curies(["X:1"]) + assert "X:1" not in nn._normalize_cache From 8effe1f11d240907e89a3ec660d8c3ee0f54a94d Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 19:18:19 -0400 Subject: [PATCH 099/123] Simplify duration parsing and version caching parse_duration spent 39 lines and four separately-worded error messages on what one regex rejects in a single branch: empty, negative, and non-integer values now share one message, and the bare-seconds path stops duplicating the unit-suffix path. BabelDownloader and NodeNorm each hand-rolled the same lazy-once cache as a value field plus a _resolved flag, the flag existing only because the resolved value may legitimately be None. functools.cached_property caches None too, so both collapse to a single property. Also extracts _write_meta(), which the tier-2 ETag refresh had been inlining alongside _save_meta, and drops two parameters no caller ever passed. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/cli.py | 47 ++++++++------------------- src/babel_explorer/core/downloader.py | 39 ++++++++++------------ src/babel_explorer/core/nodenorm.py | 31 +++++++++--------- tests/test_downloader.py | 6 ++-- 4 files changed, 49 insertions(+), 74 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 48c2ca4..2b6c93b 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -1,6 +1,7 @@ """Command-line interface for babel-explorer.""" import logging +import re from itertools import combinations import click @@ -27,7 +28,8 @@ def babel_options(f): - """Decorator adding --local-dir, --babel-url, and --check-download options to a command.""" + """Decorator adding the Babel source options: --local-dir, --babel-url, + --check-download and --allow-version-mismatch.""" f = click.option( "--allow-version-mismatch", is_flag=True, @@ -130,45 +132,24 @@ def format_option(f): return f +#: Duration suffixes accepted by --check-download; no suffix means seconds. +_DURATION_UNITS = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} + + def parse_duration(value: str) -> int | float: """Parse a duration string like '3h', '30m', '1d', '7200', or 'never' → seconds.""" - units = {"s": 1, "m": 60, "h": 3600, "d": 86400} lower = (value or "").strip().lower() - if not lower: - raise click.BadParameter( - "Invalid duration: value cannot be empty. " - "Use an integer number of seconds, optionally followed by 's', 'm', 'h', or 'd', " - "or 'never'." - ) if lower == "never": return float("inf") - # Value with unit suffix (e.g. '3h', '30m') - if lower[-1] in units: - try: - amount = int(lower[:-1]) - except ValueError: - raise click.BadParameter( - f"Invalid duration {value!r}: expected an integer followed by an optional unit " - "('s', 'm', 'h', or 'd'), or 'never'." - ) - if amount < 0: - raise click.BadParameter( - f"Invalid duration {value!r}: duration must be non-negative." - ) - return amount * units[lower[-1]] - # Bare integer seconds - try: - result = int(lower) - except ValueError: - raise click.BadParameter( - f"Invalid duration {value!r}: expected an integer number of seconds, optionally " - "followed by 's', 'm', 'h', or 'd', or 'never'." - ) - if result < 0: + # The pattern rejects empty, negative and non-integer values in one go, so there + # is a single wording of the error rather than one per rejected shape. + match = re.fullmatch(r"(\d+)([smhd]?)", lower) + if not match: raise click.BadParameter( - f"Invalid duration {value!r}: duration must be non-negative." + f"Invalid duration {value!r}: expected a non-negative integer number of " + "seconds, optionally followed by 's', 'm', 'h', or 'd', or 'never'." ) - return result + return int(match[1]) * _DURATION_UNITS[match[2]] def _depth_of(curie: str, query_set: set, depth: int | None) -> int | None: diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 7a4f62b..417d6cb 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -1,5 +1,6 @@ """HTTP downloader for Babel Parquet files with ETag-based freshness checking.""" +import functools import glob import json import logging @@ -74,8 +75,6 @@ def __init__( self.freshness_seconds = freshness_seconds self.timeout = timeout self.logger = logging.getLogger(BabelDownloader.__name__) - self._babel_version: str | None = None - self._babel_version_resolved = False if local_path is None: local_path = tempfile.gettempdir() @@ -90,13 +89,14 @@ def __init__( f"Invalid local_path (must be an existing directory): '{local_path}'" ) - @property + @functools.cached_property def babel_version(self) -> str | None: - """The Babel release behind ``url_base``, resolved once and cached.""" - if not self._babel_version_resolved: - self._babel_version = resolve_babel_version(self.url_base, self.timeout) - self._babel_version_resolved = True - return self._babel_version + """The Babel release behind ``url_base``, resolved once and cached. + + ``cached_property`` caches a ``None`` result too, so a tree without a + readable version is not re-fetched on every access. + """ + return resolve_babel_version(self.url_base, self.timeout) def sync_cache_version(self): """ @@ -158,14 +158,19 @@ def _load_meta(self, local_path): except (json.JSONDecodeError, OSError): return None - def _save_meta(self, local_path, headers, update_last_checked=True): + def _write_meta(self, local_path, meta): + """Write the sidecar .meta JSON file for local_path, stamping last_checked as now.""" + meta = meta | {"last_checked": datetime.now(UTC).isoformat()} + with open(self._get_meta_path(local_path), "w") as f: + json.dump(meta, f, indent=2) + + def _save_meta(self, local_path, headers): """ - Write a sidecar .meta JSON file next to local_path. + Write a sidecar .meta JSON file next to local_path from response headers. Args: local_path: Path to the downloaded file headers: Response headers dict (or requests.structures.CaseInsensitiveDict) - update_last_checked: If True, set last_checked to now """ meta = {} if "ETag" in headers: @@ -174,12 +179,8 @@ def _save_meta(self, local_path, headers, update_last_checked=True): meta["last_modified"] = headers["Last-Modified"] if "Content-Length" in headers: meta["content_length"] = int(headers["Content-Length"]) - if update_last_checked: - meta["last_checked"] = datetime.now(UTC).isoformat() - meta_path = self._get_meta_path(local_path) - with open(meta_path, "w") as f: - json.dump(meta, f, indent=2) + self._write_meta(local_path, meta) def _is_within_freshness(self, meta, freshness_seconds): """ @@ -400,11 +401,7 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): # Tier 2: stale but maybe unchanged — HEAD request if self._etag_matches(url_to_download, meta): - # Update last_checked timestamp - meta["last_checked"] = datetime.now(UTC).isoformat() - meta_path = self._get_meta_path(local_path_to_download_to) - with open(meta_path, "w") as f: - json.dump(meta, f, indent=2) + self._write_meta(local_path_to_download_to, meta) self.logger.info( f"ETag matches, using existing file: {local_path_to_download_to}" ) diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py index 4f1e536..31592b5 100644 --- a/src/babel_explorer/core/nodenorm.py +++ b/src/babel_explorer/core/nodenorm.py @@ -1,6 +1,7 @@ """NodeNorm API client for identifier normalisation and label enrichment.""" import dataclasses +import functools import logging import requests @@ -61,8 +62,6 @@ def __init__(self, nodenorm_url: str = "", timeout: int = 30): self._normalize_cache: dict[str, dict | None] = {} self._identifier_cache: dict[str, Identifier] = {} self._clique_cache: dict[str, list[Identifier]] = {} - self._babel_version: str | None = None - self._babel_version_resolved = False def get_babel_version(self) -> str | None: """Return the Babel release this NodeNorm instance was built from. @@ -70,23 +69,23 @@ def get_babel_version(self) -> str | None: :return: The version reported by the ``status`` endpoint, or ``None`` in offline mode or if the endpoint cannot be reached or does not report one. - The result is cached per instance. + The result is cached per instance — including ``None``, so an unreachable + status endpoint is not re-queried on every lookup. """ - if self._babel_version_resolved: - return self._babel_version - - self._babel_version_resolved = True - if self.nodenorm_url: - try: - response = requests.get( - f"{self.nodenorm_url}status", timeout=self.timeout - ) - response.raise_for_status() - self._babel_version = response.json().get("babel_version") - except (requests.RequestException, ValueError) as e: - logging.warning(f"Could not read the Babel version from NodeNorm: {e}") return self._babel_version + @functools.cached_property + def _babel_version(self) -> str | None: + if not self.nodenorm_url: + return None + try: + response = requests.get(f"{self.nodenorm_url}status", timeout=self.timeout) + response.raise_for_status() + return response.json().get("babel_version") + except (requests.RequestException, ValueError) as e: + logging.warning(f"Could not read the Babel version from NodeNorm: {e}") + return None + def get_identifier(self, curie: str) -> "Identifier": """Return the ``Identifier`` for *curie* by looking it up in its NodeNorm clique. diff --git a/tests/test_downloader.py b/tests/test_downloader.py index b75a9a2..157fcfc 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -96,8 +96,7 @@ class TestSyncCacheVersion: @staticmethod def _downloader(tmp_path, version): dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) - dl._babel_version = version - dl._babel_version_resolved = True + dl.babel_version = version return dl @staticmethod @@ -166,8 +165,7 @@ def test_404_raises_immediately(self, tmp_path): dl = BabelDownloader( url_base="https://example.com/", local_path=str(tmp_path), retries=10 ) - dl._babel_version = "2025dec11" - dl._babel_version_resolved = True + dl.babel_version = "2025dec11" response = MagicMock() response.status_code = 404 From e18e8c6f861c799977cc3b55c5a7d72e15f3f6bf Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 14 Aug 2026 19:21:00 -0400 Subject: [PATCH 100/123] Ignore node_modules/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scoped to node_modules/ rather than /web so that frontend source added under web/ is still tracked — .gitignore already anticipates web/src/lib/. Co-Authored-By: Claude Opus 5 --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index f1f9988..f3689c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ # Ignore data files. /data +# Node dependencies, wherever they are installed (e.g. web/node_modules). +# Deliberately not /web, so frontend source under it is still tracked. +node_modules/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[codz] From 5401fae61edbed9ee2b48811106229f9433e2fdf Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 18 Aug 2026 14:29:07 -0400 Subject: [PATCH 101/123] Skip the NodeNorm version check for --recurse Recursive expansion moved into a single WITH RECURSIVE DuckDB query, so --recurse no longer consults NodeNorm at all; only --labels does. Keeping `labels or recurse` made plain `xrefs ... --recurse` fail outright against the public NodeNorm, which is still built from an older Babel release. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/cli.py | 5 +++-- tests/test_cli.py | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 2b6c93b..d4a7fbd 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -308,8 +308,9 @@ def xrefs( downloader = make_downloader(babel_url, local_dir, check_download) nodenorm = NodeNorm(nodenorm_url) - # NodeNorm is only consulted when labels or the recursive expansion need it. - if labels or recurse: + # NodeNorm is only consulted for labels; --recurse is served entirely by the + # recursive DuckDB query, so its results cannot disagree with NodeNorm's release. + if labels: check_babel_versions(downloader, nodenorm, allow_version_mismatch) bxref = BabelXRefs(downloader, nodenorm) diff --git a/tests/test_cli.py b/tests/test_cli.py index e38e9c4..fdbd909 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -110,6 +110,22 @@ def test_xrefs_recurse_and_labels_flags(self): ("MONDO:0004979",), True, label_curies=True ) + def test_recurse_alone_does_not_consult_nodenorm_for_its_version(self): + """--recurse is served entirely by DuckDB, so a NodeNorm skew is irrelevant.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = "2026jul22" + mock_nn.return_value.get_babel_version.return_value = "2025sep1" + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke(cli, ["xrefs", "MONDO:0004979", "--recurse"]) + + assert result.exit_code == 0, result.output + mock_nn.return_value.get_babel_version.assert_not_called() + def test_xrefs_check_download_option(self): runner = CliRunner() @@ -628,7 +644,7 @@ def _run(args, babel_version="2026jul22", nodenorm_version="2026jul22"): mock_dl.return_value.babel_version = babel_version mock_nn.return_value.get_babel_version.return_value = nodenorm_version mock_bx.return_value.get_curie_ids.return_value = [ - IdentifierRecord(curie="MONDO:0004979", label="asthma") + IdentifierRecord(curie="MONDO:0004979", nodenorm_label="asthma") ] result = runner.invoke(cli, args) return result, mock_bx, mock_nn From 78c6e35e8cda5337c9df92d5e57dde43c6db38d0 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 18 Aug 2026 14:29:15 -0400 Subject: [PATCH 102/123] Stop the Babel label column from eating the NodeNorm one Identifiers.parquet has its own `label` column, which from_row() puts into extra_fields. record_to_dict() applies extra_fields after the dataclass fields, so `ids --labels --format json` emitted the Babel label under `label` and dropped the NodeNorm label entirely; console output printed `label=` twice. Rename the dataclass field to `nodenorm_label` so both survive. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/core/babel_xrefs.py | 7 +++++-- src/babel_explorer/formatting.py | 4 ++-- tests/test_formatting.py | 21 +++++++++++++++++---- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 38d7210..407639a 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -59,7 +59,10 @@ class IdentifierRecord: curie: str extra_fields: tuple = () - label: str = "" + # Not "label": Identifiers.parquet has its own label column, which lands in + # extra_fields and would otherwise collide with (and silently win over) this one + # once record_to_dict() flattens the record for json/tsv/csv output. + nodenorm_label: str = "" @staticmethod def from_row(row: tuple, column_names: list[str]): @@ -197,7 +200,7 @@ def get_curie_ids( self.nodenorm.normalize_curies({r.curie for r in records}) records = [ dataclasses.replace( - r, label=self.nodenorm.get_identifier(r.curie).label + r, nodenorm_label=self.nodenorm.get_identifier(r.curie).label ) for r in records ] diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index 5b0d744..1a2c629 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -104,8 +104,8 @@ def format_identifier_record(record) -> str: entirely when absent, per the console convention. """ parts = [f"curie={record.curie!r}"] - if record.label: - parts.append(f'label="{escape_label(record.label)}"') + if record.nodenorm_label: + parts.append(f'nodenorm_label="{escape_label(record.nodenorm_label)}"') parts.extend(f"{name}={value!r}" for name, value in record.extra_fields) # Parquet values are arbitrary text; escape so they are not read as markup. return escape(f"IdentifierRecord({', '.join(parts)})") diff --git a/tests/test_formatting.py b/tests/test_formatting.py index f544338..593df38 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -325,14 +325,27 @@ def test_query_curie_is_highlighted_at_depth_zero(self): assert curie_with_label("MONDO:1", 0, "asthma").startswith("[bold cyan]") def test_identifier_record_uses_the_same_convention(self): - rec = IdentifierRecord(curie="A:1", extra_fields=(("n", 1),), label='a"b') + rec = IdentifierRecord( + curie="A:1", extra_fields=(("n", 1),), nodenorm_label='a"b' + ) rendered = format_identifier_record(rec) - assert r'label="a\"b"' in rendered + assert r'nodenorm_label="a\"b"' in rendered def test_identifier_record_omits_absent_label(self): rec = IdentifierRecord(curie="A:1", extra_fields=(("n", 1),)) assert "label=" not in format_identifier_record(rec) + def test_nodenorm_label_does_not_collide_with_the_parquet_label_column(self): + """Identifiers.parquet carries its own label column; both must survive.""" + rec = IdentifierRecord( + curie="MONDO:0004979", + extra_fields=(("label", "asthma (Babel)"),), + nodenorm_label="asthma (NodeNorm)", + ) + d = record_to_dict(rec) + assert d["nodenorm_label"] == "asthma (NodeNorm)" + assert d["label"] == "asthma (Babel)" + class TestLabelOmissionInRecords: """The omit-when-absent rule is keyed on the concept, not one field name.""" @@ -369,11 +382,11 @@ def test_tabular_output_survives_rows_with_differing_keys(self): """A labelled row after an unlabelled one must not blow up DictWriter.""" rows = [ IdentifierRecord(curie="A:1", extra_fields=()), - IdentifierRecord(curie="B:2", extra_fields=(), label="asthma"), + IdentifierRecord(curie="B:2", extra_fields=(), nodenorm_label="asthma"), ] out = io.StringIO() write_records(rows, "csv", file=out) lines = out.getvalue().splitlines() - assert lines[0] == "curie,label" + assert lines[0] == "curie,nodenorm_label" assert lines[1] == "A:1," assert lines[2] == "B:2,asthma" From 3385e89710f317477468b9374bc8852d4f4a7bbe Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 18 Aug 2026 14:29:22 -0400 Subject: [PATCH 103/123] Normalise BABEL_URL before probing for Concord.parquet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BabelDownloader appends a trailing slash to url_base, but the integration skip probe joined BABEL_URL and the file path directly. A BABEL_URL without a trailing slash HEADed ".../latestduckdb/Concord.parquet", got a 404, and silently skipped the whole integration session — indistinguishable from the expected skip on a public release. Also skip rather than error when the Babel server is unreachable. Co-Authored-By: Claude Opus 5 --- tests/conftest.py | 6 +++++- tests/constants.py | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 82f7de0..1b54ee5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -75,7 +75,11 @@ def shared_downloader(test_data_dir) -> BabelDownloader: Skips the whole session when BABEL_URL points at a Babel release that does not publish the DuckDB Parquet files (as the public releases currently do not). """ - response = requests.head(BABEL_URL + CONCORD_FILE, timeout=30) + probe_url = BABEL_URL + CONCORD_FILE + try: + response = requests.head(probe_url, timeout=30) + except requests.RequestException as e: + pytest.skip(f"Babel server unreachable at {probe_url}: {e}") if response.status_code == 404: pytest.skip(f"{BABEL_URL} does not publish {CONCORD_FILE}") return BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) diff --git a/tests/constants.py b/tests/constants.py index cfe7999..58750c6 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -10,7 +10,12 @@ # release (which does not yet publish the DuckDB Parquet files, so those tests skip). load_dotenv() -BABEL_URL = os.environ.get("BABEL_URL", "https://stars.renci.org/var/babel/latest/") +# Normalised the same way BabelDownloader does, so tests that join paths onto it +# directly agree with the downloader instead of quietly requesting ".../latestduckdb/". +BABEL_URL = ( + os.environ.get("BABEL_URL", "https://stars.renci.org/var/babel/latest/").rstrip("/") + + "/" +) NODENORM_URL = os.environ.get( "NODENORM_URL", "https://nodenormalization-sri.renci.org/" ) From a58fe6a1a394e7fe1432198b86748442add383a7 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 18 Aug 2026 14:29:31 -0400 Subject: [PATCH 104/123] Make a Babel version change re-check rather than re-download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sync_cache_version() deleted the .meta sidecars, but get_downloaded_file() only enters the ETag branch when a sidecar exists — so every release rollover re-downloaded Concord (~626 MB) and Identifiers (2 GB+) in full even when byte-identical, the opposite of what the docstring claimed. Clear only last_checked and keep the ETag, so unchanged files cost one HEAD. Also delete partial .tmp downloads on a version change: they are resumed by byte offset with no If-Range validation, so a Ctrl-C (a BaseException, which the tmp cleanup in get_downloaded_file does not catch) followed by a release rollover would append the new release's bytes onto a prefix of the old one and land a corrupt Parquet that passes every later freshness check. And on HTTP 416, HEAD for the file's real headers instead of returning the 416 response's own — those describe the error body, and persisting them as the file's metadata poisoned the very cache entry the fast path just confirmed. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/core/downloader.py | 42 ++++++++++++------ tests/test_downloader.py | 61 +++++++++++++++++++++++---- 2 files changed, 82 insertions(+), 21 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 417d6cb..e778a5b 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -102,14 +102,20 @@ def sync_cache_version(self): """ Point the local cache at the Babel release behind ``url_base``. - The cache holds one release at a time. When the release changes, the ``.meta`` - sidecars are removed so the existing ETag path re-checks every cached file on - next use — whatever actually changed is re-downloaded, and files that are - unchanged cost one HEAD instead of a fresh multi-gigabyte download. - - Deleting the sidecars rather than the Parquet files also means nothing large is + The cache holds one release at a time. When the release changes, ``last_checked`` + is cleared from every ``.meta`` sidecar so the existing ETag path re-checks each + cached file on next use — whatever actually changed is re-downloaded, and files + that are unchanged cost one HEAD instead of a fresh multi-gigabyte download. The + ETag itself is deliberately kept: deleting the sidecar outright would skip the + HEAD entirely and force an unconditional re-download of every file. + + Editing the sidecars rather than the Parquet files also means nothing large is destroyed if the version cannot be trusted, and an interrupted refresh self-heals: a ``.meta`` file is only written after a successful download. + + Partial ``.tmp`` downloads are removed, because they are resumed by byte offset + with no ETag validation — appending the new release's bytes onto a prefix of the + old one would produce a corrupt Parquet that then passes every freshness check. """ version = self.babel_version if version is None: @@ -131,13 +137,20 @@ def sync_cache_version(self): f"Babel version changed: {cached_version} → {version}; " f"refreshing cached files in {self.local_path}" ) - # Only ever touch sidecars this downloader wrote. Not recursive: local_path + # Only ever touch files this downloader wrote. Not recursive: local_path # may be a directory the user pointed us at (or one holding other Babel # releases in sibling subdirectories), and must not be cleared wholesale. - for meta_path in glob.glob( - os.path.join(self.local_path, "duckdb", "*.meta") - ): - os.remove(meta_path) + duckdb_dir = os.path.join(self.local_path, "duckdb") + for meta_path in glob.glob(os.path.join(duckdb_dir, "*.meta")): + meta = self._load_meta(meta_path.removesuffix(".meta")) + if meta is None: + os.remove(meta_path) + continue + meta.pop("last_checked", None) + with open(meta_path, "w") as f: + json.dump(meta, f, indent=2) + for tmp_path in glob.glob(os.path.join(duckdb_dir, "*.tmp")): + os.remove(tmp_path) if cached_version != version: with open(marker_path, "w") as f: @@ -326,7 +339,12 @@ def _download_with_retry(self, url, local_path, chunk_size): ) as response: if response.status_code == 416: self.logger.info(f"File already complete: {local_path}") - return response.headers + # The 416 headers describe the error body, not the file; saving + # them as this file's metadata would record a bogus + # content_length and force a full re-download on the next check. + head = requests.head(url, timeout=self.timeout) + head.raise_for_status() + return head.headers elif response.status_code == 206: self.logger.info("Resuming download (HTTP 206)") elif response.status_code == 200: diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 157fcfc..e3fb8ff 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -20,7 +20,12 @@ MissingBabelFileError, resolve_babel_version, ) -from tests.constants import CONCORD_FILE +from tests.constants import BABEL_URL, CONCORD_FILE + + +def test_babel_url_is_normalised_for_direct_path_joins(): + """A slashless BABEL_URL would probe ".../latestduckdb/..." and 404-skip everything.""" + assert BABEL_URL.endswith("/") def _version_response(text): @@ -107,7 +112,9 @@ def _seed_cache(tmp_path): parquet = duckdb_dir / "Concord.parquet" parquet.write_text("data") meta = duckdb_dir / "Concord.parquet.meta" - meta.write_text("{}") + meta.write_text( + json.dumps({"etag": '"abc"', "last_checked": "2026-07-22T00:00:00+00:00"}) + ) return parquet, meta def test_writes_marker_when_absent(self, tmp_path): @@ -121,17 +128,43 @@ def test_matching_version_keeps_meta(self, tmp_path): self._downloader(tmp_path, "2026jul22").sync_cache_version() assert meta.exists() + assert "last_checked" in json.loads(meta.read_text()) - def test_changed_version_removes_meta_but_keeps_parquet(self, tmp_path): + def test_changed_version_expires_meta_but_keeps_etag_and_parquet(self, tmp_path): + """The ETag must survive so the refresh costs a HEAD, not a full re-download.""" parquet, meta = self._seed_cache(tmp_path) (tmp_path / VERSION_MARKER).write_text("2025nov19\n") self._downloader(tmp_path, "2026jul22").sync_cache_version() - assert not meta.exists(), "stale .meta sidecar should be removed" + remaining = json.loads(meta.read_text()) + assert "last_checked" not in remaining, "sidecar should no longer look fresh" + assert remaining["etag"] == '"abc"', ( + "dropping the ETag would force an unconditional multi-gigabyte re-download" + ) assert parquet.exists(), "the Parquet file itself must never be deleted" assert (tmp_path / VERSION_MARKER).read_text().strip() == "2026jul22" + def test_changed_version_removes_partial_downloads(self, tmp_path): + """A .tmp from the previous release must not be resumed against the new one.""" + self._seed_cache(tmp_path) + partial = tmp_path / "duckdb" / "Concord.parquet.tmp" + partial.write_text("half of the previous release") + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert not partial.exists() + + def test_changed_version_drops_unreadable_meta(self, tmp_path): + _, meta = self._seed_cache(tmp_path) + meta.write_text("not json") + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert not meta.exists() + def test_refresh_does_not_reach_into_sibling_directories(self, tmp_path): """local_path may hold other Babel releases; only our own duckdb/ is cleared.""" self._seed_cache(tmp_path) @@ -150,11 +183,12 @@ def test_refresh_does_not_reach_into_sibling_directories(self, tmp_path): def test_unknown_version_leaves_cache_untouched(self, tmp_path): """An unresolvable version must not trigger a multi-gigabyte re-download.""" _, meta = self._seed_cache(tmp_path) + before = meta.read_text() (tmp_path / VERSION_MARKER).write_text("2025nov19\n") self._downloader(tmp_path, None).sync_cache_version() - assert meta.exists() + assert meta.read_text() == before assert (tmp_path / VERSION_MARKER).read_text().strip() == "2025nov19" @@ -689,12 +723,21 @@ def test_http_416_file_already_complete(self, tmp_path): out_path.write_bytes(b"full file") mock_response = self._make_response(416) - with patch( - "babel_explorer.core.downloader.requests.get", return_value=mock_response + head = MagicMock(status_code=200, headers={"Content-Length": "9"}) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + return_value=mock_response, + ), + patch("babel_explorer.core.downloader.requests.head", return_value=head), ): - dl._download_with_retry("https://example.com/file", str(out_path), 1024) - # Should return without error + headers = dl._download_with_retry( + "https://example.com/file", str(out_path), 1024 + ) assert out_path.read_bytes() == b"full file" + # The 416 response describes the error body; saving its length as the file's + # metadata would fail every later freshness check and re-download the file. + assert headers == {"Content-Length": "9"} def test_server_no_resume_restarts_download(self, tmp_path): """When server responds 200 (instead of 206), partial file is removed and download restarts.""" From abd2df13142d8bf98f60a074a90a05566e201665 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 18 Aug 2026 14:29:36 -0400 Subject: [PATCH 105/123] Document the cache refresh, version check and label field Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 329c481..074a14c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,15 +44,20 @@ path segment for older trees that predate it. `latest/` resolves to whatever rel points at. `BABEL_LOCAL_DIR` holds **one Babel release at a time**, recorded in a `.babel-version` marker. -When the release changes, `BabelDownloader.sync_cache_version()` deletes the `.meta` sidecars in -`/duckdb/` — never the Parquet files — so the existing ETag path re-checks each cached -file and re-downloads only what actually changed. This keeps `Concord.parquet` and -`Identifiers.parquet` from being read together across two different Babel releases. +When the release changes, `BabelDownloader.sync_cache_version()` clears `last_checked` from the +`.meta` sidecars in `/duckdb/` — never the Parquet files — so the existing ETag path +re-checks each cached file and re-downloads only what actually changed. The stored ETag is kept +deliberately: deleting the sidecar outright skips the HEAD and forces an unconditional +multi-gigabyte re-download. Partial `.tmp` downloads *are* deleted, because they resume by byte +offset with no ETag validation and would otherwise splice two releases into one corrupt Parquet. +This keeps `Concord.parquet` and `Identifiers.parquet` from being read together across two +different Babel releases. `xrefs` fails when NodeNorm's `status` endpoint reports a different `babel_version` than the Babel being queried, since labels and cliques would not match the cross-references. Pass `--allow-version-mismatch` to proceed anyway. The check is skipped when NodeNorm is not consulted -(plain `xrefs`, `ids`) or when either version is unavailable. +(`xrefs` without `--labels`, including `--recurse`, which is served entirely by DuckDB; and `ids` +without `--labels`) or when either version is unavailable. ## Commands @@ -173,7 +178,7 @@ than silently emitting the full cross-reference list. 1. User provides CURIEs via CLI; `BABEL_URL` / `NODENORM_URL` come from `.env` or the environment 2. BabelDownloader resolves the Babel version, refreshes the cache if it changed, and ensures required Parquet files are downloaded 3. BabelXRefs queries files using DuckDB -4. If `--labels` or `--recurse` flags are set, NodeNorm is queried for additional metadata (`ids` consults NodeNorm only for `--labels`) +4. If `--labels` is set, NodeNorm is queried for additional metadata (`--recurse` alone does not consult NodeNorm — the recursive expansion is a single DuckDB query) 5. Results are printed to stdout ### Key Design Patterns @@ -215,7 +220,7 @@ broken test environment. - **`Identifier`** — Frozen dataclass for a normalized NodeNorm entry (curie, label, biolink_type, taxa, description). Returned by `NodeNorm.get_identifier()` and `get_clique_identifiers()`. - **`CrossReference`** — Frozen dataclass for Concord.parquet rows (filename, subj, pred, obj) - **`LabeledCrossReference`** — Extends CrossReference with labels and biolink types from NodeNorm -- **`IdentifierRecord`** — Frozen dataclass for Identifiers.parquet rows (curie + dynamic extra fields). Returned by `BabelXRefs.get_curie_ids()`. +- **`IdentifierRecord`** — Frozen dataclass for Identifiers.parquet rows (curie + dynamic extra fields, plus `nodenorm_label` under `--labels`). Returned by `BabelXRefs.get_curie_ids()`. The NodeNorm label is *not* called `label`: Identifiers.parquet has its own `label` column, which lands in `extra_fields` and would collide with it once the record is flattened for json/tsv/csv. ## Important Notes From f192ae454c452f4cac0c47a99b9f8139c59c04b8 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 31 Aug 2026 15:55:57 -0400 Subject: [PATCH 106/123] Stop a partial or unverified download from becoming the cached Parquet Five ways the downloader could end up with a corrupt or stale `duckdb/*.parquet` that then passed every later freshness check, because `_save_meta` stamps whatever landed on disk with the *correct* remote ETag. Once that happens the damage is permanent until the Babel release changes. - A `.tmp` left by a killed process was resumed by byte offset. The only way to reach the download at all is that the remote bytes changed, so the server appended the new file's tail to the old file's prefix. Leftover `.tmp` files are now discarded before a download starts, and cleaned up on `BaseException` so a Ctrl-C leaves nothing resumable behind. - Resumes within a run now send `If-Range` with the validator from the response they started writing from, so a file rebuilt mid-download restarts (HTTP 200, already handled) instead of splicing. - HTTP 416 was read as "already complete", but it is also what the server returns when the remote file *shrank* below the resume offset. The local size is now checked against the remote `Content-Length` first. - A stream that ended short of `Content-Length` without raising was promoted as complete. It now raises `IncompleteDownloadError` and is retried. - A failed HEAD returned "unchanged", and the caller then refreshed `last_checked`. One flaky HEAD could therefore pin the previous release's Parquet as freshly validated for the whole freshness window, right after `sync_cache_version` cleared `last_checked` for a new release. `_etag_matches` becomes `_remote_unchanged` and returns `None` for "could not check": the cached file is still used, but nothing is restamped. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/core/downloader.py | 154 ++++++++++++++-- tests/test_downloader.py | 245 +++++++++++++++++++++++++- 2 files changed, 377 insertions(+), 22 deletions(-) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index e778a5b..a039613 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -21,6 +21,16 @@ class MissingBabelFileError(RuntimeError): """Raised when a Babel release does not publish a file this tool needs.""" +class IncompleteDownloadError(RuntimeError): + """Raised when a download ends before the whole advertised file arrived. + + A stream that stops early without raising (a proxy or CDN closing the + connection cleanly, say) would otherwise be promoted to the final path and + stamped with the correct ETag, leaving a truncated Parquet that passes every + later freshness check. Raising instead lets the retry loop resume it. + """ + + def resolve_babel_version(url_base: str, timeout: int = 30) -> str | None: """ Resolve the Babel version behind a Babel base URL. @@ -114,8 +124,10 @@ def sync_cache_version(self): a ``.meta`` file is only written after a successful download. Partial ``.tmp`` downloads are removed, because they are resumed by byte offset - with no ETag validation — appending the new release's bytes onto a prefix of the - old one would produce a corrupt Parquet that then passes every freshness check. + and the bytes already on disk belong to the previous release. ``get_downloaded_file`` + also discards any ``.tmp`` it finds before starting a download, which covers the + cases this sweep cannot see (a content change within one release, or a file that is + never re-downloaded); clearing them here keeps stale gigabytes off disk as well. """ version = self.babel_version if version is None: @@ -177,6 +189,32 @@ def _write_meta(self, local_path, meta): with open(self._get_meta_path(local_path), "w") as f: json.dump(meta, f, indent=2) + @staticmethod + def _full_content_length(headers, local_path): + """ + Return the length of the *whole* remote file, or None if it is not known. + + A partial (HTTP 206) response's ``Content-Length`` is the length of the + returned range, not of the file. Recording that as the file's length would + make the Last-Modified + Content-Length fallback in ``_remote_unchanged`` + compare a partial length against the full remote one forever, re-downloading + a multi-gigabyte file on every freshness expiry. ``Content-Range`` carries + the total (``bytes 100-999/1000``); when it is present but the total is + unknown (``/*``), the file on disk is the better answer. + """ + content_range = headers.get("Content-Range") + if content_range: + total = content_range.rsplit("/", 1)[-1].strip() + if total.isdigit(): + return int(total) + try: + return os.path.getsize(local_path) + except OSError: + return None + if "Content-Length" in headers: + return int(headers["Content-Length"]) + return None + def _save_meta(self, local_path, headers): """ Write a sidecar .meta JSON file next to local_path from response headers. @@ -190,8 +228,9 @@ def _save_meta(self, local_path, headers): meta["etag"] = headers["ETag"] if "Last-Modified" in headers: meta["last_modified"] = headers["Last-Modified"] - if "Content-Length" in headers: - meta["content_length"] = int(headers["Content-Length"]) + content_length = self._full_content_length(headers, local_path) + if content_length is not None: + meta["content_length"] = content_length self._write_meta(local_path, meta) @@ -218,29 +257,38 @@ def _is_within_freshness(self, meta, freshness_seconds): except (ValueError, TypeError): return False - def _etag_matches(self, url, meta): + def _remote_unchanged(self, url, meta): """ Do a HEAD request and check if the ETag (or Last-Modified + Content-Length) matches the stored metadata. Does not write to disk — the caller is responsible for updating last_checked - when this returns True. + when this returns ``True``. Args: url: URL to HEAD meta: dict loaded from .meta file (may have etag, last_modified, content_length) Returns: - bool: True if remote matches local meta (file is still current) + True if the remote file is confirmed to match the local metadata, + False if it is confirmed to have changed, and ``None`` if the check + could not be made (the HEAD request failed). ``None`` is *not* the + same as ``True``: the cached file is still usable, but the caller must + not refresh ``last_checked`` on the strength of a check that never + happened. Doing so would let one flaky HEAD pin the previous release's + Parquet as "freshly validated" for the whole freshness window, right + after ``sync_cache_version`` cleared ``last_checked`` for a new release + — exactly the cross-release mixing the version marker exists to prevent. """ try: response = requests.head(url, timeout=self.timeout) response.raise_for_status() except requests.RequestException as e: self.logger.warning( - f"HEAD request failed for {url}: {e}; assuming file is current" + f"HEAD request failed for {url}: {e}; using the cached file, " + f"but it will be re-checked on next use" ) - return True + return None remote_headers = response.headers @@ -284,6 +332,10 @@ def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): local_path: Local file path to write to resume_byte_pos: Starting byte position (for resume) chunk_size: Size of chunks to read/write + + Raises: + IncompleteDownloadError: If fewer bytes arrived than Content-Length + advertised. """ content_length = response.headers.get("Content-Length") if content_length: @@ -307,6 +359,16 @@ def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): f.write(chunk) progress_bar.update(len(chunk)) + # A stream can end early without raising. Comparing against Content-Length + # is only meaningful for an identity-coded body: with Content-Encoding set, + # iter_content hands back decoded bytes whose count is unrelated to it. + if total_size is not None and not response.headers.get("Content-Encoding"): + written = os.path.getsize(local_path) + if written != total_size: + raise IncompleteDownloadError( + f"{local_path}: expected {total_size} bytes, received {written}" + ) + def _download_with_retry(self, url, local_path, chunk_size): """ Download a file with retry logic and resume capability. @@ -322,6 +384,14 @@ def _download_with_retry(self, url, local_path, chunk_size): Raises: RuntimeError: If all retry attempts fail """ + # Validator (ETag, else Last-Modified) of the response we started writing + # from. Sent back as If-Range on a resume so a file that changed mid-download + # restarts from scratch instead of having the new version's tail appended to + # the old version's prefix — a splice that would pass every later ETag check. + # A leftover .tmp from an earlier run carries no validator and is never + # resumed; get_downloaded_file removes it before we are called. + validator = None + for attempt in range(1, self.retries + 1): try: resume_byte_pos = 0 @@ -331,6 +401,8 @@ def _download_with_retry(self, url, local_path, chunk_size): headers = {} if resume_byte_pos > 0: headers["Range"] = f"bytes={resume_byte_pos}-" + if validator: + headers["If-Range"] = validator self.logger.info(f"Resuming download from byte {resume_byte_pos}") # timeout is per-read (seconds without receiving bytes), not a total time limit. @@ -338,12 +410,29 @@ def _download_with_retry(self, url, local_path, chunk_size): url, headers=headers, stream=True, timeout=self.timeout ) as response: if response.status_code == 416: + # 416 also comes back when the remote file *shrank* below our + # resume offset, so "the range is past the end" does not by + # itself mean the local file is the remote one. Check the size + # before promoting it, or a rebuild that produced a smaller + # Parquet leaves an over-long file with valid-looking metadata. + head = requests.head(url, timeout=self.timeout) + head.raise_for_status() + remote_length = head.headers.get("Content-Length") + if ( + remote_length is not None + and int(remote_length) != resume_byte_pos + ): + self.logger.warning( + f"Local file is {resume_byte_pos} bytes but the remote " + f"file is {remote_length}; discarding it and " + f"downloading afresh" + ) + os.remove(local_path) + continue self.logger.info(f"File already complete: {local_path}") # The 416 headers describe the error body, not the file; saving # them as this file's metadata would record a bogus # content_length and force a full re-download on the next check. - head = requests.head(url, timeout=self.timeout) - head.raise_for_status() return head.headers elif response.status_code == 206: self.logger.info("Resuming download (HTTP 206)") @@ -367,12 +456,15 @@ def _download_with_retry(self, url, local_path, chunk_size): else: response.raise_for_status() + validator = response.headers.get("ETag") or response.headers.get( + "Last-Modified" + ) self._stream_download( response, local_path, resume_byte_pos, chunk_size ) return response.headers - except (OSError, requests.RequestException) as e: + except (OSError, requests.RequestException, IncompleteDownloadError) as e: self.logger.warning( f"Download attempt {attempt}/{self.retries} failed: {e}" ) @@ -386,6 +478,11 @@ def _download_with_retry(self, url, local_path, chunk_size): f"Failed to download {url} after {self.retries} attempts: {e}" ) + # Only reachable if the last attempt was a 416 that restarted the download + # (`continue`) with no attempts left. Falling through would return None and + # leave the caller replacing a .tmp that is no longer there. + raise RuntimeError(f"Failed to download {url} after {self.retries} attempts") + def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): """ Download a file from the Babel server to local storage with ETag-based caching. @@ -393,6 +490,8 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): Three-tier freshness logic: 1. If .meta exists and last_checked is within freshness window → return immediately 2. If .meta exists but stale → HEAD request to compare ETag; return if unchanged + (or if the HEAD failed, in which case last_checked is left alone so the + check is retried on next use) 3. If ETag changed or no .meta → full re-download Args: @@ -418,12 +517,22 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): return local_path_to_download_to # Tier 2: stale but maybe unchanged — HEAD request - if self._etag_matches(url_to_download, meta): + unchanged = self._remote_unchanged(url_to_download, meta) + if unchanged is True: self._write_meta(local_path_to_download_to, meta) self.logger.info( f"ETag matches, using existing file: {local_path_to_download_to}" ) return local_path_to_download_to + if unchanged is None: + # Could not reach the server. Use the cached file, but leave + # last_checked stale so the next run checks again rather than + # treating an unverified file as fresh for hours. + self.logger.warning( + f"Could not check whether {url_to_download} changed; " + f"using the cached file: {local_path_to_download_to}" + ) + return local_path_to_download_to # Tier 3: ETag changed — re-download self.logger.warning( @@ -437,12 +546,29 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): # Download to a sibling .tmp file, then atomically replace the final destination. # This ensures the final file is never partially written. tmp_path = local_path_to_download_to + ".tmp" + + # Discard any .tmp left behind by an earlier run (killed process, Ctrl-C). + # _download_with_retry resumes by byte offset, and we have no way to tell + # which version of the remote file those bytes came from — while the only + # way to reach this point with a cached file present is that the remote + # bytes *changed*. Resuming would splice the new file's tail onto the old + # file's prefix and then stamp the result with the new ETag, making the + # corruption permanent. Restarting costs a re-download; splicing costs + # silent, undetectable data corruption. + if os.path.exists(tmp_path): + self.logger.warning( + f"Discarding partial download from an earlier run: {tmp_path}" + ) + os.remove(tmp_path) + try: response_headers = self._download_with_retry( url_to_download, tmp_path, chunk_size ) os.replace(tmp_path, local_path_to_download_to) - except Exception: + except BaseException: + # BaseException, not Exception: a Ctrl-C mid-download must clean up too, + # since the partial file cannot be safely resumed later. if os.path.exists(tmp_path): os.remove(tmp_path) raise diff --git a/tests/test_downloader.py b/tests/test_downloader.py index e3fb8ff..181c39b 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -17,6 +17,7 @@ from babel_explorer.core.downloader import ( VERSION_MARKER, BabelDownloader, + IncompleteDownloadError, MissingBabelFileError, resolve_babel_version, ) @@ -414,8 +415,8 @@ def test_returns_false_when_freshness_is_zero(self, tmp_path): assert dl._is_within_freshness(meta, 0) is False -class TestEtagMatches: - """Tests for _etag_matches.""" +class TestRemoteUnchanged: + """Tests for _remote_unchanged.""" def _make_dl(self, tmp_path): return BabelDownloader( @@ -431,7 +432,7 @@ def test_returns_true_on_matching_etag(self, tmp_path): with patch( "babel_explorer.core.downloader.requests.head", return_value=mock_resp ): - assert dl._etag_matches("https://example.com/f.parquet", meta) is True + assert dl._remote_unchanged("https://example.com/f.parquet", meta) is True def test_returns_false_on_different_etag(self, tmp_path): dl = self._make_dl(tmp_path) @@ -442,7 +443,7 @@ def test_returns_false_on_different_etag(self, tmp_path): with patch( "babel_explorer.core.downloader.requests.head", return_value=mock_resp ): - assert dl._etag_matches("https://example.com/f.parquet", meta) is False + assert dl._remote_unchanged("https://example.com/f.parquet", meta) is False def test_fallback_last_modified_match(self, tmp_path): dl = self._make_dl(tmp_path) @@ -454,17 +455,18 @@ def test_fallback_last_modified_match(self, tmp_path): with patch( "babel_explorer.core.downloader.requests.head", return_value=mock_resp ): - assert dl._etag_matches("https://example.com/f.parquet", meta) is True + assert dl._remote_unchanged("https://example.com/f.parquet", meta) is True - def test_returns_true_on_request_error(self, tmp_path): - """Network errors are treated as 'assume still fresh' to avoid triggering large re-downloads.""" + def test_returns_none_on_request_error(self, tmp_path): + """A failed HEAD is 'unknown', not 'unchanged' — the caller keeps the cached + file but must not restamp last_checked on the strength of it.""" dl = self._make_dl(tmp_path) meta = {"etag": '"abc"'} with patch( "babel_explorer.core.downloader.requests.head", side_effect=requests.ConnectionError("fail"), ): - assert dl._etag_matches("https://example.com/f.parquet", meta) is True + assert dl._remote_unchanged("https://example.com/f.parquet", meta) is None class TestGetDownloadedFileTiers: @@ -552,6 +554,36 @@ def test_tier2_updates_last_checked_after_head(self, tmp_path): updated_ts = datetime.fromisoformat(updated_meta["last_checked"]) assert (datetime.now(UTC) - updated_ts).total_seconds() < 5 + def test_tier2_failed_head_does_not_refresh_last_checked(self, tmp_path): + """A HEAD that never happened must not mark the file freshly validated. + + sync_cache_version clears last_checked when the Babel release changes so + every cached file is re-checked. If one flaky HEAD restamped it, the old + release's Parquet would look current for the whole freshness window under a + .babel-version marker naming the new release. + """ + dl = self._make_dl(tmp_path, freshness=3600) + test_file = "duckdb/unreachable.parquet" + local = tmp_path / "duckdb" / "unreachable.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data from the previous release") + + # No last_checked: exactly what sync_cache_version leaves behind. + with open(str(local) + ".meta", "w") as f: + json.dump({"etag": '"old"'}, f) + + with patch( + "babel_explorer.core.downloader.requests.head", + side_effect=requests.ConnectionError("network down"), + ): + with patch("babel_explorer.core.downloader.requests.get") as mock_get: + result = dl.get_downloaded_file(test_file) + mock_get.assert_not_called() + + assert result == str(local) + with open(str(local) + ".meta") as f: + assert "last_checked" not in json.load(f) + # --- Tier 3: ETag changed, re-download --- def test_tier3_redownloads_when_etag_changed(self, tmp_path): @@ -640,6 +672,203 @@ def fake_download(url, path, chunk_size): assert open(result, "rb").read() == new_content +class TestPartialDownloadSafety: + """A .tmp must never be resumed across two different versions of a remote file.""" + + def test_leftover_tmp_from_an_earlier_run_is_discarded(self, tmp_path): + """A .tmp of unknown provenance is deleted before the download starts. + + get_downloaded_file only reaches the download block when the remote bytes + changed, so resuming an orphaned .tmp (left by a killed process) would + append the new file's tail to the old file's prefix and then stamp the + splice with the new ETag. + """ + dl = BabelDownloader( + url_base="https://example.com/", + local_path=str(tmp_path), + freshness_seconds=0, + ) + test_file = "duckdb/spliced.parquet" + local = tmp_path / "duckdb" / "spliced.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"old version") + with open(str(local) + ".meta", "w") as f: + json.dump({"etag": '"old"'}, f) + tmp_file = local.parent / "spliced.parquet.tmp" + tmp_file.write_bytes(b"PREFIX-OF-OLD-VERSION") + + seen_sizes = [] + + def fake_download(url, path, chunk_size): + seen_sizes.append(os.path.getsize(path) if os.path.exists(path) else None) + with open(path, "wb") as f: + f.write(b"new version") + return {"ETag": '"new"'} + + head = Mock(headers={"ETag": '"new"'}, raise_for_status=Mock()) + with ( + patch("babel_explorer.core.downloader.requests.head", return_value=head), + patch.object(dl, "_download_with_retry", side_effect=fake_download), + ): + result = dl.get_downloaded_file(test_file) + + assert seen_sizes == [None], "the stale .tmp was still on disk" + assert open(result, "rb").read() == b"new version" + + def test_keyboard_interrupt_removes_the_partial_file(self, tmp_path): + """Ctrl-C is a BaseException; the .tmp must still be cleaned up.""" + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + tmp_file = tmp_path / "interrupted.parquet.tmp" + + def fake_download(url, path, chunk_size): + with open(path, "wb") as f: + f.write(b"half a file") + raise KeyboardInterrupt + + with patch.object(dl, "_download_with_retry", side_effect=fake_download): + with pytest.raises(KeyboardInterrupt): + dl.get_downloaded_file("interrupted.parquet") + + assert not tmp_file.exists() + + def test_resume_sends_if_range_once_a_validator_is_known(self, tmp_path): + """After the first response, a resume is conditional on the file not changing.""" + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = str(tmp_path / "conditional.bin") + + # First attempt streams 4 of 10 bytes and then trips the size check. + first = TestDownloadWithRetry._make_response( + 200, {"Content-Length": "10", "ETag": '"v1"'}, [b"abcd"] + ) + second = TestDownloadWithRetry._make_response( + 206, {"Content-Length": "6", "ETag": '"v1"'}, [b"efghij"] + ) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[first, second], + ) as mock_get, + patch("babel_explorer.core.downloader.time.sleep"), + ): + dl._download_with_retry("https://example.com/file", out_path, 1024) + + assert mock_get.call_args_list[1].kwargs["headers"] == { + "Range": "bytes=4-", + "If-Range": '"v1"', + } + assert open(out_path, "rb").read() == b"abcdefghij" + + +class TestDownloadCompleteness: + """A short stream must be retried, never promoted as the finished file.""" + + def test_truncated_stream_raises(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = str(tmp_path / "short.bin") + + mock_response = Mock() + mock_response.headers = {"Content-Length": "10"} + mock_response.iter_content = Mock(return_value=[b"only4"]) + + with pytest.raises(IncompleteDownloadError, match="expected 10 bytes"): + dl._stream_download(mock_response, out_path, 0, 1024) + + def test_truncated_stream_is_retried_and_resumed(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = str(tmp_path / "resumed.bin") + + first = TestDownloadWithRetry._make_response( + 200, {"Content-Length": "10"}, [b"abcd"] + ) + second = TestDownloadWithRetry._make_response( + 206, {"Content-Length": "6"}, [b"efghij"] + ) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[first, second], + ), + patch("babel_explorer.core.downloader.time.sleep"), + ): + dl._download_with_retry("https://example.com/file", out_path, 1024) + + assert open(out_path, "rb").read() == b"abcdefghij" + + def test_encoded_body_skips_the_size_check(self, tmp_path): + """With Content-Encoding set, iter_content yields decoded bytes whose count + has nothing to do with Content-Length.""" + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = str(tmp_path / "gzipped.bin") + + mock_response = Mock() + mock_response.headers = {"Content-Length": "4", "Content-Encoding": "gzip"} + mock_response.iter_content = Mock(return_value=[b"decompressed"]) + + dl._stream_download(mock_response, out_path, 0, 1024) + assert open(out_path, "rb").read() == b"decompressed" + + def test_416_with_a_shorter_remote_file_restarts(self, tmp_path): + """A remote rebuild that shrank the file also answers 416; the over-long + local file must not be promoted as complete.""" + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = tmp_path / "shrunk.bin" + out_path.write_bytes(b"the old, longer file") # 20 bytes + + too_long = TestDownloadWithRetry._make_response(416) + fresh = TestDownloadWithRetry._make_response( + 200, {"Content-Length": "5", "ETag": '"new"'}, [b"short"] + ) + head = MagicMock(status_code=200, headers={"Content-Length": "5"}) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[too_long, fresh], + ), + patch("babel_explorer.core.downloader.requests.head", return_value=head), + ): + headers = dl._download_with_retry( + "https://example.com/file", str(out_path), 1024 + ) + + assert out_path.read_bytes() == b"short" + assert headers["ETag"] == '"new"' + + +class TestFullContentLength: + """A 206 Content-Length is a range length, not the file's length.""" + + def test_partial_response_records_the_total_from_content_range(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + file_path = str(tmp_path / "resumed.parquet") + with open(file_path, "wb") as f: + f.write(b"0123456789") + + dl._save_meta( + file_path, + {"Content-Length": "6", "Content-Range": "bytes 4-9/10", "ETag": '"e"'}, + ) + with open(file_path + ".meta") as f: + assert json.load(f)["content_length"] == 10 + + def test_partial_response_with_unknown_total_falls_back_to_the_file(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + file_path = str(tmp_path / "unknown_total.parquet") + with open(file_path, "wb") as f: + f.write(b"0123456789") + + dl._save_meta( + file_path, {"Content-Length": "6", "Content-Range": "bytes 4-9/*"} + ) + with open(file_path + ".meta") as f: + assert json.load(f)["content_length"] == 10 + + class TestGetDownloadedFileCaching: """Tests that repeated calls within the freshness window avoid redundant downloads.""" From 1a43d2fa0802fd1809c5f3c6720978a70e5dfe1a Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 31 Aug 2026 15:56:07 -0400 Subject: [PATCH 107/123] Keep Identifiers.parquet's own label column in json/tsv/csv output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `record_to_dict` dropped any key ending in `label` whose value was falsy, to implement the "omit an absent label rather than emit an empty one" convention. But `extra_fields` is flattened in first, so Identifiers.parquet's own `label` column — and any other `*_label` column Babel adds — was dropped whenever it was empty or NULL. `ids --format json` then emitted some records with a `label` key and some without, and a consumer doing `row["label"]` got a KeyError. The rule only ever meant the NodeNorm-derived labels, so name them: nodenorm_label, subj_label, obj_label. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/formatting.py | 14 +++++++++++--- tests/test_formatting.py | 26 +++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py index 1a2c629..b658a0b 100644 --- a/src/babel_explorer/formatting.py +++ b/src/babel_explorer/formatting.py @@ -14,6 +14,11 @@ from rich.console import Console from rich.markup import escape +#: Label fields populated from NodeNorm, which are omitted from machine-readable +#: output when empty. IdentifierRecord spells it nodenorm_label; +#: LabeledCrossReference spells it subj_label/obj_label. +_NODENORM_LABEL_FIELDS = ("nodenorm_label", "subj_label", "obj_label") + def record_to_dict(record) -> dict[str, Any]: """Convert a dataclass (or plain dict) to a flat dict. @@ -29,9 +34,12 @@ def record_to_dict(record) -> dict[str, Any]: d[col] = val # An absent label is omitted rather than emitted as "", matching the console # convention and keeping TSV/CSV columns stable when labels were not requested. - # Keyed on the concept, not one field name: LabeledCrossReference spells it - # subj_label/obj_label, and those must follow the same rule. - for key in [k for k, v in d.items() if k.endswith("label") and not v]: + # Listed explicitly rather than matched on a "label" suffix: extra_fields has + # already been flattened in above, and Identifiers.parquet has its own `label` + # column (Babel may add more) whose empty values are real data, not an absent + # NodeNorm lookup. Dropping those would make `label` present on some rows of a + # json/tsv/csv run and missing on others. + for key in [k for k in _NODENORM_LABEL_FIELDS if k in d and not d[k]]: del d[key] return d diff --git a/tests/test_formatting.py b/tests/test_formatting.py index 593df38..2a6629f 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -348,7 +348,31 @@ def test_nodenorm_label_does_not_collide_with_the_parquet_label_column(self): class TestLabelOmissionInRecords: - """The omit-when-absent rule is keyed on the concept, not one field name.""" + """The omit-when-absent rule covers the NodeNorm label fields, and only those.""" + + def test_an_empty_parquet_label_column_is_kept(self): + """Identifiers.parquet's own `label` column is data, not an absent lookup. + + Dropping it when empty would give some rows of a json/tsv/csv run a `label` + key and others none, so `row["label"]` raises KeyError downstream. + """ + rec = IdentifierRecord(curie="A:1", extra_fields=(("label", ""),)) + d = record_to_dict(rec) + assert d["label"] == "" + assert "nodenorm_label" not in d + + def test_json_rows_all_carry_the_parquet_label_column(self): + out = io.StringIO() + write_records( + [ + IdentifierRecord(curie="A:1", extra_fields=(("label", ""),)), + IdentifierRecord(curie="B:2", extra_fields=(("label", "asthma"),)), + ], + "json", + file=out, + ) + rows = json.loads(out.getvalue()) + assert [r["label"] for r in rows] == ["", "asthma"] def test_empty_subj_and_obj_labels_are_dropped(self): xref = LabeledCrossReference( From 040db7585385c3237e755b435046a96b5eb23baa Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 31 Aug 2026 15:56:07 -0400 Subject: [PATCH 108/123] Document the partial-download rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records why a leftover `.tmp` is never resumed across runs, why in-run resumes are conditional, where the two size checks sit, and why a failed HEAD does not refresh `last_checked` — so none of it gets "optimised" back into a cross-run resume or a fail-open freshness stamp. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 074a14c..888c79e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,10 +48,15 @@ When the release changes, `BabelDownloader.sync_cache_version()` clears `last_ch `.meta` sidecars in `/duckdb/` — never the Parquet files — so the existing ETag path re-checks each cached file and re-downloads only what actually changed. The stored ETag is kept deliberately: deleting the sidecar outright skips the HEAD and forces an unconditional -multi-gigabyte re-download. Partial `.tmp` downloads *are* deleted, because they resume by byte -offset with no ETag validation and would otherwise splice two releases into one corrupt Parquet. -This keeps `Concord.parquet` and `Identifiers.parquet` from being read together across two -different Babel releases. +multi-gigabyte re-download. Partial `.tmp` downloads *are* deleted, so no prefix from the previous +release survives into the next one. This keeps `Concord.parquet` and `Identifiers.parquet` from +being read together across two different Babel releases. + +If a HEAD request fails, `_remote_unchanged()` returns `None` — "could not check", distinct from +`True`/"confirmed unchanged". The cached file is still used, but `last_checked` is deliberately +**not** refreshed, so the next run checks again. Restamping it there would let one flaky HEAD pin +the previous release's Parquet as freshly validated for the whole freshness window, immediately +after `sync_cache_version()` cleared `last_checked` for a new release. `xrefs` fails when NodeNorm's `status` endpoint reports a different `babel_version` than the Babel being queried, since labels and cliques would not match the cross-references. Pass @@ -59,6 +64,30 @@ being queried, since labels and cliques would not match the cross-references. Pa (`xrefs` without `--labels`, including `--recurse`, which is served entirely by DuckDB; and `ids` without `--labels`) or when either version is unavailable. +## Partial downloads + +Downloads land in a sibling `.tmp` file and are promoted with `os.replace`. Three rules keep a +`.tmp` from becoming a corrupt Parquet that then passes every freshness check — a failure that is +permanent, because the file gets stamped with the *correct* ETag: + +- **A `.tmp` is never resumed across runs.** `get_downloaded_file()` deletes any it finds before + starting, and cleans up on `BaseException` so a Ctrl-C leaves nothing behind. Resume is by byte + offset, the only way to reach the download at all is that the remote bytes *changed*, and an + orphaned `.tmp` carries no record of which version its bytes came from. Restarting costs a + re-download; splicing costs silent data corruption. Do not "optimise" this back into a + cross-run resume without persisting the validator alongside the `.tmp`. +- **In-run resumes send `If-Range`** with the validator from the response they started writing + from, so a file rebuilt mid-download restarts (HTTP 200) instead of splicing. +- **Sizes are checked, twice.** A stream that ends short of `Content-Length` raises + `IncompleteDownloadError` and is retried, rather than being promoted as complete; and an HTTP + 416 is only treated as "already complete" once the local size matches the remote + `Content-Length`, since 416 also means the remote file *shrank* below the resume offset. + +`_save_meta()` records the length of the whole file, taken from `Content-Range` rather than a 206 +response's `Content-Length` (which is only the range's length). Storing the partial length would +make the Last-Modified fallback in `_remote_unchanged()` compare it against the full remote length +forever, re-downloading an unchanged multi-gigabyte file on every freshness expiry. + ## Commands ### Running the Application From b55081fdb54285c11c71d6c0f122ecbd7133ad57 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 31 Aug 2026 22:31:34 -0400 Subject: [PATCH 109/123] Explain why a .tmp is deleted in two places The delete in `get_downloaded_file` is the safety guarantee; the sweep in `sync_cache_version` is housekeeping for files that are never re-downloaded and so never reach it. Neither covers the other's case, and dropping the wrong one reintroduces silent Parquet corruption, so say which is which. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 6 ++++++ src/babel_explorer/core/downloader.py | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 888c79e..e736078 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,6 +52,12 @@ multi-gigabyte re-download. Partial `.tmp` downloads *are* deleted, so no prefix release survives into the next one. This keeps `Concord.parquet` and `Identifiers.parquet` from being read together across two different Babel releases. +A `.tmp` is deleted in two places, on purpose. The delete in `get_downloaded_file()` is the safety +guarantee (see [Partial downloads](#partial-downloads)); the sweep in `sync_cache_version()` is +housekeeping that reclaims gigabytes belonging to a release nobody will ask for again, including +for files that are never re-downloaded and so never reach `get_downloaded_file()`. Dropping the +sweep only wastes disk; dropping the other reintroduces silent Parquet corruption. + If a HEAD request fails, `_remote_unchanged()` returns `None` — "could not check", distinct from `True`/"confirmed unchanged". The cached file is still used, but `last_checked` is deliberately **not** refreshed, so the next run checks again. Restamping it there would let one flaky HEAD pin diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index a039613..ea13f4f 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -123,11 +123,20 @@ def sync_cache_version(self): destroyed if the version cannot be trusted, and an interrupted refresh self-heals: a ``.meta`` file is only written after a successful download. - Partial ``.tmp`` downloads are removed, because they are resumed by byte offset - and the bytes already on disk belong to the previous release. ``get_downloaded_file`` - also discards any ``.tmp`` it finds before starting a download, which covers the - cases this sweep cannot see (a content change within one release, or a file that is - never re-downloaded); clearing them here keeps stale gigabytes off disk as well. + Partial ``.tmp`` downloads are removed, because the bytes already on disk belong + to the previous release. This is deliberately the *second* of two deletes, and the + redundancy is the point: + + * ``get_downloaded_file`` discarding a leftover ``.tmp`` before each download is + the one that makes resume safe. It is the only one that covers a content change + within a single release, or a ``local_path`` this method never looked at. + * This sweep is housekeeping. It reclaims gigabytes belonging to a release nobody + will ask for again, including for files that are never re-downloaded and so + never reach ``get_downloaded_file``. + + Neither is redundant with the other for the case it owns. Removing this sweep only + wastes disk; removing the one in ``get_downloaded_file`` reintroduces silent + Parquet corruption — see the comment there before touching either. """ version = self.babel_version if version is None: From 76ae0409164dd8f239210f07b287f8098b09f6a6 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 02:01:17 -0400 Subject: [PATCH 110/123] Compose the Babel URL from a releases directory and a version BABEL_URL was one opaque knob that had to carry both "which server" and "which release". Splitting it matches how both Babel trees are actually laid out -- one subdirectory per release plus a `latest/` symlink -- and makes pinning a release a one-word change rather than a URL edit: BABEL_RELEASES_URL=https://stars.renci.org/var/babel/ (--babel-releases-url) BABEL_VERSION=latest (--babel-version) The effective Babel URL is the two composed. `--babel-url` still takes a complete URL and overrides both, for a tree that does not follow that layout, but it is now **command line only**: with two variables already feeding the composed URL, a third that silently outranked both would make "which release am I actually querying?" unanswerable from the environment alone. The defaults compose byte-identically to the URL they replace, so nothing about the shipped behaviour changes. Falling out of this: - `resolve_babel_version`'s final-path-segment fallback now strips exactly BABEL_VERSION, so a pinned release still resolves when VERSION.txt is unreachable. `latest` still yields None, as before. - The skew message can now suggest pinning --babel-version to the release NodeNorm was built from, which fixes the mismatch rather than suppressing it. - `--babel-version` rejects a value containing "://" or "..", since anyone with muscle memory from BABEL_URL will eventually paste a whole URL into it. - `cli()` warns if BABEL_URL is still set, so a stale .env does not silently send someone to the wrong release. `compose_babel_url` lives in core/downloader.py rather than cli.py because tests/constants.py needs the identical composition and must not import Click to get it. TestUrlConfiguration now stubs load_dotenv: it runs inside cli(), after CliRunner(env=...) has cleared a variable, so a real .env would otherwise leak into the assertions -- and every Translator developer is about to have BABEL_RELEASES_URL in theirs. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/cli.py | 165 ++++++++++++++++++++++---- src/babel_explorer/core/downloader.py | 25 +++- tests/conftest.py | 4 +- tests/constants.py | 21 ++-- tests/test_cli.py | 116 +++++++++++++++++- tests/test_downloader.py | 33 +++++- 6 files changed, 321 insertions(+), 43 deletions(-) diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index d4a7fbd..7826bef 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -1,6 +1,7 @@ """Command-line interface for babel-explorer.""" import logging +import os import re from itertools import combinations @@ -15,7 +16,11 @@ build_depth_map, find_shortest_path, ) -from babel_explorer.core.downloader import BabelDownloader, MissingBabelFileError +from babel_explorer.core.downloader import ( + BabelDownloader, + MissingBabelFileError, + compose_babel_url, +) from babel_explorer.core.nodenorm import NodeNorm from babel_explorer.formatting import ( curie_with_label, @@ -27,14 +32,33 @@ ) +def _validate_babel_version(ctx, param, value): + """Reject a --babel-version that is really a URL or a path traversal. + + Anyone with muscle memory from the old BABEL_URL variable will eventually put a + complete URL here, which would compose into nonsense with no useful diagnostic. + """ + if value is None: + return value + if "://" in value: + raise click.BadParameter( + f"{value!r} looks like a complete URL. Pass it to --babel-url instead, " + f"or give --babel-version just the release name (e.g. '2025dec11')." + ) + if ".." in value: + raise click.BadParameter(f"{value!r} may not contain '..'.") + return value + + def babel_options(f): - """Decorator adding the Babel source options: --local-dir, --babel-url, - --check-download and --allow-version-mismatch.""" + """Decorator adding the Babel source options: --local-dir, --babel-releases-url, + --babel-version, --babel-url, --check-download and --allow-version-mismatch.""" f = click.option( "--allow-version-mismatch", is_flag=True, envvar="BABEL_ALLOW_VERSION_MISMATCH", - help="Proceed even if NodeNorm was built from a different Babel release than --babel-url", + help="Proceed even if NodeNorm was built from a different Babel release than " + "the one being queried", )(f) f = click.option( "--check-download", @@ -48,19 +72,44 @@ def babel_options(f): f = click.option( "--babel-url", type=str, - default="https://stars.renci.org/var/babel/latest/", + default=None, + # Deliberately NO envvar=. BABEL_RELEASES_URL + BABEL_VERSION is the only + # environment-driven path to a Babel URL, so there is never a question of which + # variable wins. This is a per-run escape hatch, not configuration. Do not add one. + help="Complete URL of one Babel release, overriding --babel-releases-url and " + "--babel-version. Command line only: there is no BABEL_URL environment variable. " + "[default: --babel-releases-url + --babel-version]", + )(f) + f = click.option( + "--babel-version", + type=str, + default="latest", + show_default=True, + show_envvar=True, + envvar="BABEL_VERSION", + callback=_validate_babel_version, + help="Babel release to use: the name of a subdirectory under --babel-releases-url " + "(e.g. '2025dec11'). 'latest' follows whatever the server currently publishes.", + )(f) + f = click.option( + "--babel-releases-url", + type=str, + default="https://stars.renci.org/var/babel/", show_default=True, - envvar="BABEL_URL", - help="Base URL of the Babel server", + show_envvar=True, + envvar="BABEL_RELEASES_URL", + help="URL of a directory holding one subdirectory per Babel release.", )(f) f = click.option( "--local-dir", type=str, default="data", show_default=True, + show_envvar=True, envvar="BABEL_LOCAL_DIR", help="Local location to save Babel download files to. Holds one Babel release at " - "a time; cached files are refreshed automatically when --babel-url points at a new one.", + "a time; cached files are refreshed automatically when the effective Babel URL " + "points at a new one.", )(f) return f @@ -77,10 +126,48 @@ def nodenorm_options(f): )(f) -def make_downloader(babel_url: str, local_dir: str, check_download: str): - """Build a BabelDownloader and point its cache at the Babel release behind *babel_url*.""" +def resolve_babel_url( + babel_url: str | None, babel_releases_url: str, babel_version: str +) -> str: + """The effective Babel URL: the one release this run will query. + + ``--babel-url`` is a complete URL and wins outright; otherwise the release is + composed from the releases directory and the version. ``--babel-url`` has no + matching environment variable on purpose — with two variables already feeding the + composed URL, a third that silently outranked both would make "which release am I + actually querying?" unanswerable from the environment alone. + """ + if babel_url: + # Warn only when --babel-version was actually typed. A developer with + # BABEL_VERSION permanently in .env would otherwise be warned on every + # --babel-url run, which just teaches them to ignore warnings. + ctx = click.get_current_context(silent=True) + if ctx is not None and ctx.get_parameter_source("babel_version") == ( + click.core.ParameterSource.COMMANDLINE + ): + click.echo( + f"Warning: --babel-url overrides --babel-version, so " + f"{babel_version!r} is ignored.", + err=True, + ) + return babel_url.strip().rstrip("/") + "/" + return compose_babel_url(babel_releases_url, babel_version) + + +def make_downloader( + babel_url: str | None, + babel_releases_url: str, + babel_version: str, + local_dir: str, + check_download: str, +): + """Build a BabelDownloader and point its cache at the effective Babel release. + + Composition happens here rather than at each call site so a future command cannot + take the options and forget to resolve them. + """ downloader = BabelDownloader( - babel_url, + resolve_babel_url(babel_url, babel_releases_url, babel_version), local_path=local_dir, freshness_seconds=parse_duration(check_download), ) @@ -97,19 +184,22 @@ def check_babel_versions( cliques would come from one Babel while the cross-references come from another. Skipped when either version is unavailable. """ - babel_version = downloader.babel_version + # Named for the release the *server* reports, to keep it distinct from the + # babel_version parameter the commands take, which is the release the user asked for. + downloader_version = downloader.babel_version nodenorm_version = nodenorm.get_babel_version() if ( - babel_version + downloader_version and nodenorm_version - and babel_version != nodenorm_version + and downloader_version != nodenorm_version and not allow_version_mismatch ): raise click.ClickException( f"NodeNorm at {nodenorm.nodenorm_url} was built from Babel {nodenorm_version}, " - f"but {downloader.url_base} is Babel {babel_version}. Labels and cliques would " + f"but {downloader.url_base} is Babel {downloader_version}. Labels and cliques would " f"not match the cross-references. Point --nodenorm-url at a matching NodeNorm, " - f"or pass --allow-version-mismatch to proceed anyway." + f"pin --babel-version to the release NodeNorm was built from, or pass " + f"--allow-version-mismatch to proceed anyway." ) @@ -255,6 +345,15 @@ def cli(): logging.basicConfig(level=logging.INFO) # Runs before subcommand parameters are parsed, so .env feeds the envvar= defaults. load_dotenv() + # BABEL_URL was the single Babel setting before BABEL_RELEASES_URL + BABEL_VERSION. + # It is now inert, and silently ignoring it would send someone to the wrong release + # with no clue why. Checked after load_dotenv() so a stale .env is caught too. + if os.environ.get("BABEL_URL"): + click.echo( + "Warning: BABEL_URL is no longer used. Set BABEL_RELEASES_URL and " + "BABEL_VERSION instead, or pass --babel-url for a single run.", + err=True, + ) @cli.command("xrefs") @@ -271,7 +370,9 @@ def cli(): @format_option def xrefs( curies: list[str], - babel_url: str, + babel_url: str | None, + babel_releases_url: str, + babel_version: str, nodenorm_url: str, local_dir: str, recurse: bool, @@ -290,8 +391,13 @@ def xrefs( :param curies: A list of CURIEs (Compact URI) for which cross-references need to be retrieved. :type curies: list[str] - :param babel_url: Base URL of the Babel server from which to download DuckDB files. - :type babel_url: str + :param babel_url: Complete URL of one Babel release, overriding the two below. + ``None`` unless ``--babel-url`` was passed. + :type babel_url: str | None + :param babel_releases_url: URL of a directory holding one subdirectory per release. + :type babel_releases_url: str + :param babel_version: Which release subdirectory to query, or ``latest``. + :type babel_version: str :return: None """ @@ -306,7 +412,9 @@ def xrefs( ) recurse = True - downloader = make_downloader(babel_url, local_dir, check_download) + downloader = make_downloader( + babel_url, babel_releases_url, babel_version, local_dir, check_download + ) nodenorm = NodeNorm(nodenorm_url) # NodeNorm is only consulted for labels; --recurse is served entirely by the # recursive DuckDB query, so its results cannot disagree with NodeNorm's release. @@ -353,7 +461,9 @@ def xrefs( @format_option def ids( curies: list[str], - babel_url: str, + babel_url: str | None, + babel_releases_url: str, + babel_version: str, nodenorm_url: str, local_dir: str, labels: bool, @@ -370,12 +480,19 @@ def ids( :param curies: A list of CURIEs (Compact URI) for which cross-references need to be retrieved. :type curies: list[str] - :param babel_url: Base URL of the Babel server - :type babel_url: str + :param babel_url: Complete URL of one Babel release, overriding the two below. + ``None`` unless ``--babel-url`` was passed. + :type babel_url: str | None + :param babel_releases_url: URL of a directory holding one subdirectory per release. + :type babel_releases_url: str + :param babel_version: Which release subdirectory to query, or ``latest``. + :type babel_version: str :return: None """ - downloader = make_downloader(babel_url, local_dir, check_download) + downloader = make_downloader( + babel_url, babel_releases_url, babel_version, local_dir, check_download + ) nodenorm = NodeNorm(nodenorm_url) # NodeNorm is only consulted for labels, so only then can its Babel release differ. if labels: diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index ea13f4f..976b67e 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -31,12 +31,28 @@ class IncompleteDownloadError(RuntimeError): """ +def compose_babel_url(releases_url: str, version: str) -> str: + """Join a releases-directory URL and a release name into one Babel base URL. + + Normalised the way ``BabelDownloader`` wants it — exactly one trailing slash — so + callers can join relative paths straight onto the result. A releases URL with no + trailing slash would otherwise compose ``.../babellatest/`` and 404 everything. + + Lives here rather than in ``cli.py`` because ``tests/constants.py`` needs the same + composition and should not import Click to get it. + """ + return releases_url.strip().rstrip("/") + "/" + version.strip().strip("/") + "/" + + def resolve_babel_version(url_base: str, timeout: int = 30) -> str | None: """ Resolve the Babel version behind a Babel base URL. Reads ``VERSION.txt`` (present on all full Babel releases, e.g. ``Babel 2026jul22``), - falling back to the final path segment for older trees that predate it. + falling back to the final path segment for older trees that predate it. Under the + ``BABEL_RELEASES_URL`` + ``BABEL_VERSION`` scheme that final segment *is* the + requested version, so a pinned release survives a missing ``VERSION.txt`` while + ``latest`` still resolves to ``None``. :return: The version string, or ``None`` if it cannot be determined. """ @@ -458,9 +474,10 @@ def _download_with_retry(self, url, local_path, chunk_size): # do not currently publish the DuckDB Parquet files. raise MissingBabelFileError( f"This Babel release ({self.babel_version or self.url_base}) does not " - f"publish {url[len(self.url_base) :]}. Translator team members should " - f"contact the Babel developers for the Translator-specific URL and set " - f"BABEL_URL in .env." + f"publish {url[len(self.url_base) :]}. Translator team members " + f"should contact the Babel developers for the Translator-specific " + f"releases URL and set BABEL_RELEASES_URL in .env, or pass " + f"--babel-url with a complete release URL for a one-off run." ) else: response.raise_for_status() diff --git a/tests/conftest.py b/tests/conftest.py index 1b54ee5..d5b729b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -72,8 +72,8 @@ def test_data_dir(): def shared_downloader(test_data_dir) -> BabelDownloader: """A BabelDownloader pointed at the test data directory. - Skips the whole session when BABEL_URL points at a Babel release that does not - publish the DuckDB Parquet files (as the public releases currently do not). + Skips the whole session when the composed Babel URL points at a release that does + not publish the DuckDB Parquet files (as the public releases currently do not). """ probe_url = BABEL_URL + CONCORD_FILE try: diff --git a/tests/constants.py b/tests/constants.py index 58750c6..025a289 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -5,17 +5,22 @@ from dotenv import load_dotenv -# Integration tests run against whatever BABEL_URL points at, so a Translator developer -# with a .env exercises them while public contributors and CI fall back to the public -# release (which does not yet publish the DuckDB Parquet files, so those tests skip). +from babel_explorer.core.downloader import compose_babel_url + +# Integration tests run against whatever BABEL_RELEASES_URL and BABEL_VERSION compose to, +# so a Translator developer with a .env exercises them while public contributors and CI +# fall back to the public release (which does not yet publish the DuckDB Parquet files, +# so those tests skip). load_dotenv() -# Normalised the same way BabelDownloader does, so tests that join paths onto it -# directly agree with the downloader instead of quietly requesting ".../latestduckdb/". -BABEL_URL = ( - os.environ.get("BABEL_URL", "https://stars.renci.org/var/babel/latest/").rstrip("/") - + "/" +BABEL_RELEASES_URL = os.environ.get( + "BABEL_RELEASES_URL", "https://stars.renci.org/var/babel/" ) +BABEL_VERSION = os.environ.get("BABEL_VERSION", "latest") + +# Composed exactly the way the CLI composes it, so tests that join paths onto it directly +# agree with the downloader instead of quietly requesting ".../latestduckdb/". +BABEL_URL = compose_babel_url(BABEL_RELEASES_URL, BABEL_VERSION) NODENORM_URL = os.environ.get( "NODENORM_URL", "https://nodenormalization-sri.renci.org/" ) diff --git a/tests/test_cli.py b/tests/test_cli.py index fdbd909..4af4e5f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -518,6 +518,11 @@ class TestUrlConfiguration: def _invoke(args, env): runner = CliRunner() with ( + # load_dotenv() runs inside cli(), i.e. after CliRunner(env=...) has cleared a + # variable and before Click reads envvars — so a real .env would leak into + # these assertions. Every Translator developer is about to have + # BABEL_RELEASES_URL in theirs, so neutralise it rather than hope. + patch("babel_explorer.cli.load_dotenv"), patch("babel_explorer.cli.BabelDownloader") as mock_dl, patch("babel_explorer.cli.BabelXRefs") as mock_bx, patch("babel_explorer.cli.NodeNorm") as mock_nn, @@ -529,7 +534,12 @@ def _invoke(args, env): def test_defaults_are_public(self): mock_dl, mock_nn = self._invoke( - ["xrefs", "A:1"], env={"BABEL_URL": None, "NODENORM_URL": None} + ["xrefs", "A:1"], + env={ + "BABEL_RELEASES_URL": None, + "BABEL_VERSION": None, + "NODENORM_URL": None, + }, ) assert mock_dl.call_args[0][0] == "https://stars.renci.org/var/babel/latest/" assert mock_nn.call_args[0][0] == "https://nodenormalization-sri.renci.org/" @@ -538,22 +548,120 @@ def test_env_overrides_defaults(self): mock_dl, mock_nn = self._invoke( ["xrefs", "A:1"], env={ - "BABEL_URL": "https://example.com/babel/", + "BABEL_RELEASES_URL": "https://example.com/babel/", + "BABEL_VERSION": "2025dec11", "BABEL_LOCAL_DIR": "/tmp/babel-cache", "NODENORM_URL": "https://example.com/nn/", }, ) - assert mock_dl.call_args[0][0] == "https://example.com/babel/" + assert mock_dl.call_args[0][0] == "https://example.com/babel/2025dec11/" assert mock_dl.call_args.kwargs["local_path"] == "/tmp/babel-cache" assert mock_nn.call_args[0][0] == "https://example.com/nn/" + def test_releases_url_without_trailing_slash_still_composes(self): + mock_dl, _ = self._invoke( + ["xrefs", "A:1"], + env={ + "BABEL_RELEASES_URL": "https://example.com/babel", + "BABEL_VERSION": "2025dec11", + }, + ) + assert mock_dl.call_args[0][0] == "https://example.com/babel/2025dec11/" + def test_flag_beats_env(self): mock_dl, _ = self._invoke( ["xrefs", "A:1", "--babel-url", "https://flag.example.com/"], - env={"BABEL_URL": "https://env.example.com/"}, + env={ + "BABEL_RELEASES_URL": "https://env.example.com/", + "BABEL_VERSION": "2025dec11", + }, ) assert mock_dl.call_args[0][0] == "https://flag.example.com/" + def test_babel_version_flag_beats_env(self): + mock_dl, _ = self._invoke( + ["xrefs", "A:1", "--babel-version", "2026jul22"], + env={ + "BABEL_RELEASES_URL": "https://example.com/babel/", + "BABEL_VERSION": "2025dec11", + }, + ) + assert mock_dl.call_args[0][0] == "https://example.com/babel/2026jul22/" + + def test_babel_url_has_no_environment_variable(self): + """The design decision, pinned: BABEL_URL in the environment does nothing. + + Two variables already feed the composed URL; a third that silently outranked + both would make the effective release unreadable from the environment alone. + """ + mock_dl, _ = self._invoke( + ["xrefs", "A:1"], + env={ + "BABEL_URL": "https://ignored.example.com/", + "BABEL_RELEASES_URL": None, + "BABEL_VERSION": None, + }, + ) + assert mock_dl.call_args[0][0] == "https://stars.renci.org/var/babel/latest/" + + def test_stale_babel_url_warns(self): + """Ignoring it silently would send someone to the wrong release with no clue.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.load_dotenv"), + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke( + cli, ["xrefs", "A:1"], env={"BABEL_URL": "https://stale.example.com/"} + ) + assert result.exit_code == 0, result.output + assert "BABEL_URL is no longer used" in result.output + + def test_babel_url_with_typed_version_warns(self): + mock_dl, _ = self._invoke( + [ + "xrefs", + "A:1", + "--babel-url", + "https://flag.example.com/", + "--babel-version", + "2026jul22", + ], + env={}, + ) + assert mock_dl.call_args[0][0] == "https://flag.example.com/" + + def test_babel_url_with_env_version_is_silent(self): + """Warning on an env-supplied version would fire on every --babel-url run.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.load_dotenv"), + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke( + cli, + ["xrefs", "A:1", "--babel-url", "https://flag.example.com/"], + env={"BABEL_VERSION": "2026jul22"}, + ) + assert result.exit_code == 0, result.output + assert "is ignored" not in result.output + + @pytest.mark.parametrize( + "bad", ["https://example.com/babel/latest/", "../../etc/passwd"] + ) + def test_babel_version_rejects_urls_and_traversal(self, bad): + runner = CliRunner() + with patch("babel_explorer.cli.load_dotenv"): + result = runner.invoke(cli, ["xrefs", "A:1", "--babel-version", bad]) + assert result.exit_code != 0 + assert "--babel-version" in result.output or "may not contain" in result.output + class TestMissingBabelFileReporting: """A missing Parquet file should read as an error, not a traceback.""" diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 181c39b..4f75aef 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -19,16 +19,47 @@ BabelDownloader, IncompleteDownloadError, MissingBabelFileError, + compose_babel_url, resolve_babel_version, ) from tests.constants import BABEL_URL, CONCORD_FILE def test_babel_url_is_normalised_for_direct_path_joins(): - """A slashless BABEL_URL would probe ".../latestduckdb/..." and 404-skip everything.""" + """The env-driven URL must end in "/" too, or the skip probe requests + ".../latestduckdb/Concord.parquet", 404s, and silently skips every integration test.""" assert BABEL_URL.endswith("/") +class TestComposeBabelUrl: + """Composition has to absorb the slashes users will and will not type.""" + + @pytest.mark.parametrize( + "releases, version, expected", + [ + ("https://ex.com/babel/", "latest", "https://ex.com/babel/latest/"), + ("https://ex.com/babel", "latest", "https://ex.com/babel/latest/"), + ("https://ex.com/babel//", "2025dec11", "https://ex.com/babel/2025dec11/"), + ("https://ex.com/babel", "/2025dec11/", "https://ex.com/babel/2025dec11/"), + (" https://ex.com/babel ", "latest", "https://ex.com/babel/latest/"), + ( + "https://ex.com/babel", + " 2025dec11 ", + "https://ex.com/babel/2025dec11/", + ), + ], + ) + def test_normalisation(self, releases, version, expected): + assert compose_babel_url(releases, version) == expected + + def test_public_default_composes_to_the_historical_url(self): + """The default pair must reproduce the single URL this option pair replaced.""" + assert ( + compose_babel_url("https://stars.renci.org/var/babel/", "latest") + == "https://stars.renci.org/var/babel/latest/" + ) + + def _version_response(text): """A mock requests response serving *text* as the body of VERSION.txt.""" response = Mock() From e1473e67a490d0eac93f69d1d57eb6328da2d820 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 02:01:52 -0400 Subject: [PATCH 111/123] Rename .env.example to env.default and document the new settings `.env.example` is hidden on Linux and reads like a sample rather than the defaults it actually holds. `env.default` is visible in a plain `ls` and says what it is. It also gains BABEL_CHECK_DOWNLOAD, which the README documented but the old template omitted. The README and CLAUDE.md are updated throughout for BABEL_RELEASES_URL / BABEL_VERSION, and both now record why --babel-url deliberately has no envvar= so nobody adds one back. CLAUDE.md also notes that the .babel-version marker records the release the server *resolved* to rather than the one requested, so `latest` and an equivalent pinned version share a cache -- that is intended, not a bug to fix. The missing-Parquet caveat is no longer addressed only to Translator members: it now says plainly that the shipped defaults cannot serve data yet, and links the tracking issue. Co-Authored-By: Claude Opus 5 --- .env.example | 17 ----------------- CLAUDE.md | 48 ++++++++++++++++++++++++++++++++++-------------- README.md | 40 ++++++++++++++++++++++++++-------------- env.default | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 45 deletions(-) delete mode 100644 .env.example create mode 100644 env.default diff --git a/.env.example b/.env.example deleted file mode 100644 index bbd24c1..0000000 --- a/.env.example +++ /dev/null @@ -1,17 +0,0 @@ -# Copy this file to .env and edit as needed. Every value here can also be set as an -# environment variable, or overridden per-run by the matching command-line option. - -# Public Babel release. "latest" is resolved through VERSION.txt; when the release -# behind it changes, babel-explorer refreshes the cached files automatically. -BABEL_URL=https://stars.renci.org/var/babel/latest/ - -# Where downloaded Babel files are cached. This holds one Babel release at a time. -# Point it somewhere per-release if you need to keep several around. -BABEL_LOCAL_DIR=data - -NODENORM_URL=https://nodenormalization-sri.renci.org/ - -# NOTE: public Babel releases do not currently publish the DuckDB Parquet files -# (duckdb/Concord.parquet, duckdb/Identifiers.parquet) that babel-explorer needs. -# Translator team members should contact the Babel developers for the -# Translator-specific URL and set BABEL_URL to it here. diff --git a/CLAUDE.md b/CLAUDE.md index e736078..6d8d235 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ uv sync uv sync --group dev # Configure the Babel and NodeNorm endpoints -cp .env.example .env +cp env.default .env # Run the CLI uv run babel-explorer --help @@ -26,22 +26,39 @@ uv run babel-explorer --help ## Configuration -`BABEL_URL`, `BABEL_LOCAL_DIR`, `BABEL_CHECK_DOWNLOAD`, `NODENORM_URL`, and +`BABEL_RELEASES_URL`, `BABEL_VERSION`, `BABEL_LOCAL_DIR`, `BABEL_CHECK_DOWNLOAD`, `NODENORM_URL`, and `BABEL_ALLOW_VERSION_MISMATCH` are read from `.env` (via `python-dotenv`, loaded in the `cli()` group) or the environment. Each is also a command-line option, and precedence runs **flag > environment variable > `.env` > built-in default**. -`.env.example` ships with the **public** Babel URL only. Public Babel releases do not currently +The release actually queried — the **effective Babel URL** — is +`BABEL_RELEASES_URL.rstrip("/") + "/" + BABEL_VERSION + "/"`, composed by `resolve_babel_url()` +(`cli.py`) on top of the pure `compose_babel_url()` (`core/downloader.py`). `compose_babel_url` +lives in the downloader rather than the CLI because `tests/constants.py` needs the same +composition and must not import Click to get it. + +`--babel-url` overrides the composed pair with a complete URL, for a tree that does not follow the +releases-directory layout. It has **no `envvar=`, on purpose**: two variables already feed the +composed URL, and a third that silently outranked both would make "which release am I querying?" +unanswerable from the environment alone. Do not add one. `BABEL_URL` was the single pre-refactor +setting and is now inert; `cli()` warns if it is still set so it does not fail silently. + +`env.default` ships with the **public** Babel URL only. Public Babel releases do not currently publish the DuckDB Parquet files this tool needs, so Translator team members must contact the -Babel developers for the Translator-specific URL and set `BABEL_URL` to it. Never commit that URL -to this repository. +Babel developers for the Translator-specific releases URL and set `BABEL_RELEASES_URL` to it. +Never commit that URL to this repository. ## Babel versions -The Babel version behind `--babel-url` is resolved by `resolve_babel_version()` +The Babel version behind the effective Babel URL is resolved by `resolve_babel_version()` (`core/downloader.py`), which reads `VERSION.txt` (`Babel 2026jul22`) and falls back to the final path segment for older trees that predate it. `latest/` resolves to whatever release it currently -points at. +points at. That fallback segment is now exactly `BABEL_VERSION`, so a pinned release still +resolves when `VERSION.txt` is unreachable, while `latest` yields `None` as before. + +The `.babel-version` marker records the release the server *resolved* to, not the one requested, so +`BABEL_VERSION=latest` and `BABEL_VERSION=2025dec11` share a cache while they name the same +release. That is deliberate — do not "fix" it into a spurious refresh. `BABEL_LOCAL_DIR` holds **one Babel release at a time**, recorded in a `.babel-version` marker. When the release changes, `BabelDownloader.sync_cache_version()` clears `last_checked` from the @@ -118,7 +135,10 @@ uv run babel-explorer ids MONDO:0004979 --labels uv run babel-explorer test-concord MONDO:0004979 HP:0000001 # Use a custom Babel server or local directory (overrides .env) -uv run babel-explorer xrefs MONDO:0004979 --local-dir data --babel-url https://stars.renci.org/var/babel/latest/ +uv run babel-explorer xrefs MONDO:0004979 --local-dir data --babel-version 2025dec11 + +# Override the composed URL entirely (command line only; there is no BABEL_URL env var) +uv run babel-explorer xrefs MONDO:0004979 --babel-url https://stars.renci.org/var/babel/latest/ ``` ### Development Commands @@ -210,7 +230,7 @@ than silently emitting the full cross-reference list. ### Data Flow -1. User provides CURIEs via CLI; `BABEL_URL` / `NODENORM_URL` come from `.env` or the environment +1. User provides CURIEs via CLI; `BABEL_RELEASES_URL` + `BABEL_VERSION` / `NODENORM_URL` come from `.env` or the environment, and are composed into the effective Babel URL 2. BabelDownloader resolves the Babel version, refreshes the cache if it changed, and ensures required Parquet files are downloaded 3. BabelXRefs queries files using DuckDB 4. If `--labels` is set, NodeNorm is queried for additional metadata (`--recurse` alone does not consult NodeNorm — the recursive expansion is a single DuckDB query) @@ -239,9 +259,9 @@ uv run pytest --collect-only -q -m "not integration" # unit test count uv run pytest --collect-only -q # full count ``` -**Integration tests skip when `BABEL_URL` points at a Babel release that does not publish +**Integration tests skip when the composed Babel URL points at a release that does not publish `duckdb/Concord.parquet`**, which is the case for every public release right now. A run reporting -a couple of dozen skips is the expected result without a Translator `BABEL_URL` in `.env`, not a +a couple of dozen skips is the expected result without a Translator `BABEL_RELEASES_URL` in `.env`, not a broken test environment. ### Test Infrastructure @@ -260,8 +280,8 @@ broken test environment. ## Important Notes - **Data directory**: The `data/` directory is gitignored and contains downloaded Parquet files and generated DuckDB databases -- **Babel versions**: The Babel release comes from whatever `--babel-url` / `BABEL_URL` points at; see [Babel versions](#babel-versions) above -- **`.env`**: gitignored. Only `.env.example` is committed, and it must never contain the Translator-specific Babel URL +- **Babel versions**: The Babel release comes from `BABEL_RELEASES_URL` + `BABEL_VERSION`, or from `--babel-url` when given; see [Babel versions](#babel-versions) above +- **`.env`**: gitignored. Only `env.default` is committed, and it must never contain the Translator-specific Babel URL ## File Locations @@ -269,5 +289,5 @@ broken test environment. - Tests: `tests/` - Test CURIEs: `tests/data/valid_curies.txt` - Downloaded Babel files: `/duckdb/*.parquet` (default `data/duckdb/`) -- Endpoint configuration: `.env` (gitignored), template in `.env.example` +- Endpoint configuration: `.env` (gitignored), template in `env.default` - Entry point: `src/babel_explorer/cli.py` diff --git a/README.md b/README.md index 713be82..0cbd1d5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This project uses [uv](https://docs.astral.sh/uv/) for package management: ```bash uv sync --group dev -cp .env.example .env +cp env.default .env ``` ## Configuration @@ -18,28 +18,40 @@ cp .env.example .env | Variable | Default | Purpose | |---|---|---| -| `BABEL_URL` | `https://stars.renci.org/var/babel/latest/` | Babel release to query | +| `BABEL_RELEASES_URL` | `https://stars.renci.org/var/babel/` | Directory holding one subdirectory per Babel release | +| `BABEL_VERSION` | `latest` | Which release subdirectory to query | | `BABEL_LOCAL_DIR` | `data` | Where downloaded Babel files are cached | | `BABEL_CHECK_DOWNLOAD` | `3h` | How often to re-check downloads | | `NODENORM_URL` | `https://nodenormalization-sri.renci.org/` | NodeNorm instance for labels and cliques | Each has a matching command-line option, and precedence runs **flag > environment variable > -`.env` > default**. +`.env` > default**. The release actually queried — the *effective Babel URL* — is +`BABEL_RELEASES_URL` + `BABEL_VERSION` + `/`. -> **Translator team members:** public Babel releases do not currently publish the DuckDB Parquet -> files (`duckdb/Concord.parquet`, `duckdb/Identifiers.parquet`) that babel-explorer needs, so the -> default `BABEL_URL` will report that the files are missing. Contact the Babel developers for the -> Translator-specific URL and set `BABEL_URL` to it in your `.env`. +`--babel-url` is the one exception. It takes a complete URL and overrides the composed pair, for a +tree that does not follow the releases-directory layout. It is **command-line only**: there is no +`BABEL_URL` environment variable, so the environment can never disagree with itself about which +release is in effect. + +> **Public releases cannot serve data yet.** Public Babel releases do not currently publish the +> DuckDB Parquet files (`duckdb/Concord.parquet`, `duckdb/Identifiers.parquet`) that babel-explorer +> needs, so the shipped defaults will report that the files are missing. Translator team members +> should contact the Babel developers for the Translator-specific releases URL and set +> `BABEL_RELEASES_URL` to it in their `.env`, or pass `--babel-url ` for a single +> run. Tracked in [#16](https://github.com/TranslatorSRI/babel-explorer/issues/16). ### Babel versions -`BABEL_LOCAL_DIR` holds one Babel release at a time. When `BABEL_URL` starts pointing at a -different release, babel-explorer notices and re-downloads the files that changed — you do not -need to clear the cache by hand. +`BABEL_LOCAL_DIR` holds one Babel release at a time. When the effective Babel URL starts pointing +at a different release, babel-explorer notices and re-downloads the files that changed — you do not +need to clear the cache by hand. The cache marker records the release the server *resolved* to, so +`BABEL_VERSION=latest` and `BABEL_VERSION=2025dec11` share a cache while they name the same +release. `xrefs` refuses to run when NodeNorm was built from a different Babel release than the one being -queried, since the labels and cliques would not match the cross-references. Pass -`--allow-version-mismatch` to override. +queried, since the labels and cliques would not match the cross-references. Either pin +`BABEL_VERSION` to the release NodeNorm reports, point `--nodenorm-url` at a matching NodeNorm, or +pass `--allow-version-mismatch` to override. ## Usage @@ -69,8 +81,8 @@ uv run babel-explorer test-concord MONDO:0004979 HP:0000001 Tests are split into fast **unit tests** (mocked, no network) and slower **integration tests** (real file downloads and API calls), controlled by pytest markers. -Integration tests run against whatever `BABEL_URL` points at, and skip when that release does not -publish the DuckDB Parquet files. +Integration tests run against whatever `BABEL_RELEASES_URL` and `BABEL_VERSION` compose to, and +skip when that release does not publish the DuckDB Parquet files. ```bash # Unit tests only — fast, no network required diff --git a/env.default b/env.default new file mode 100644 index 0000000..922c778 --- /dev/null +++ b/env.default @@ -0,0 +1,33 @@ +# Copy this file to .env and edit as needed. Every value here can also be set as an +# environment variable, or overridden per-run by the matching command-line option. +# Precedence: command-line flag > environment variable > .env > built-in default. + +# Directory holding one subdirectory per Babel release. The release actually queried is +# BABEL_RELEASES_URL + BABEL_VERSION + "/". +BABEL_RELEASES_URL=https://stars.renci.org/var/babel/ + +# Which release to use: a subdirectory name such as 2025dec11, or "latest" to follow +# whatever the server currently publishes. "latest" is resolved through VERSION.txt, so +# when the release behind it changes, babel-explorer refreshes the cached files +# automatically. +BABEL_VERSION=latest + +# There is deliberately no BABEL_URL variable. To query a complete URL that does not fit +# the releases-directory + version layout, pass --babel-url on the command line; it +# overrides both settings above for that run. + +# Where downloaded Babel files are cached. This holds one Babel release at a time. +# Point it somewhere per-release if you need to keep several around. +BABEL_LOCAL_DIR=data + +# How often to re-check downloads: e.g. 3h, 30m, 1d, 0, never. +# "never" always uses cached files; "0" forces a re-check every time. +BABEL_CHECK_DOWNLOAD=3h + +NODENORM_URL=https://nodenormalization-sri.renci.org/ + +# NOTE: public Babel releases do not currently publish the DuckDB Parquet files +# (duckdb/Concord.parquet, duckdb/Identifiers.parquet) that babel-explorer needs, so the +# defaults above will report that the files are missing. Translator team members should +# contact the Babel developers for the Translator-specific releases URL and set +# BABEL_RELEASES_URL to it here. Never commit that URL to this repository. From 2cb4f73b696629056cb0f5173b01a054ab239288 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 02:01:52 -0400 Subject: [PATCH 112/123] Add package metadata and a changelog for the 0.1.0 release The repository is MIT licensed and has been since the first commit, but the built wheel carried no license, no author and no classifiers. Anyone inspecting the package could not tell what they were allowed to do with it. CHANGELOG.md records what 0.1.0 contains and, just as importantly, what it cannot do yet: the shipped defaults cannot query data (#16) and --labels fails the version check against public NodeNorm (#17). Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++++ pyproject.toml | 10 ++++++++++ 2 files changed, 49 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..67b5ce9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,39 @@ +# Changelog + +All notable changes to babel-explorer are documented here. This project follows +[semantic versioning](https://semver.org/). + +## 0.1.0 — 2026-09-01 + +First release. A Click CLI for querying Babel intermediate files through DuckDB, with optional +label enrichment from NodeNorm. + +### Added + +- `xrefs` — cross-references for one or more CURIEs, with `--recurse` for transitive expansion + (a single `WITH RECURSIVE` DuckDB query), `--paths` for the shortest paths connecting the given + CURIEs, and `--labels` for NodeNorm labels and Biolink types. +- `ids` — identifier records from `Identifiers.parquet`, with `--labels`. +- `test-concord` — compare a proposed concordance change against NodeNorm's current cliques. +- `--format json|tsv|csv` on `xrefs` and `ids` for machine-readable output. +- `BabelDownloader`: streaming downloads with ETag-based freshness checking, resumable retries, + and a cache that holds one Babel release at a time and refreshes itself when that release + changes. +- Configuration from `.env` or the environment — `BABEL_RELEASES_URL`, `BABEL_VERSION`, + `BABEL_LOCAL_DIR`, `BABEL_CHECK_DOWNLOAD`, `NODENORM_URL`, `BABEL_ALLOW_VERSION_MISMATCH` — + with `env.default` as the committed template. Precedence: flag > environment > `.env` > default. +- A version-skew check that refuses to mix labels from one Babel release with cross-references + from another, overridable with `--allow-version-mismatch`. + +### Known limitations + +- **The shipped defaults cannot query data yet.** Public Babel releases do not publish + `duckdb/Concord.parquet` or `duckdb/Identifiers.parquet`. Translator team members can set + `BABEL_RELEASES_URL` to an internal releases URL; everyone else gets a clear error rather than + results. Tracked in [#16](https://github.com/TranslatorSRI/babel-explorer/issues/16). +- **`--labels` fails against the public defaults.** NodeNorm dev reports Babel `2025sep1` while + public `latest` is `2025dec11`, so the skew check fires. Pin `BABEL_VERSION` to the release + NodeNorm was built from, or pass `--allow-version-mismatch`. Tracked in + [#17](https://github.com/TranslatorSRI/babel-explorer/issues/17). +- 28 of 288 tests are integration tests that skip without a Babel release publishing the Parquet + files. Tracked in [#18](https://github.com/TranslatorSRI/babel-explorer/issues/18). diff --git a/pyproject.toml b/pyproject.toml index fe6e3a8..707c00c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,17 @@ name = "babel-explorer" version = "0.1.0" description = "Tool for querying and exploring Babel APIs and intermediate files" readme = "README.md" +license = "MIT" +license-files = ["LICENSE"] +authors = [{ name = "Gaurav Vaidya", email = "gaurav@ggvaidya.com" }] requires-python = ">=3.11" +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] dependencies = [ "click>=8.3.1", "duckdb>=1.4.2", From 37e9fac193c1e63f82490fe3116c801bb4ba1ff3 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 02:19:34 -0400 Subject: [PATCH 113/123] Stop claiming "not slow" means no multi-gigabyte download Verifying the release end to end against a current Babel turned up that Concord.parquet is 4.6 GB in 2026jul22, not the ~626 MB the fixture docstring claimed. Its tests are not marked `slow`, so `pytest -m "integration and not slow"` -- documented in both README.md and CLAUDE.md as avoiding 2GB+ downloads -- would in fact pull 4.6 GB. Marking those tests slow instead would leave the non-slow integration set covering nothing that touches real data, so the marker keeps its meaning and the docs are corrected: `slow` means Identifiers.parquet, and `not slow` is not a promise of "small". Called out explicitly because it changes the sizing for #18, where CI would start pulling these files for real. Hard byte figures are dropped from the fixture docstrings for the same reason the file already gives for test counts: they drift silently and then mislead. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 9 +++++++-- README.md | 2 +- pyproject.toml | 2 +- tests/conftest.py | 11 +++++++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6d8d235..7ec2ac8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,7 +150,7 @@ uv run pytest -v # Run unit tests only (fast, no network) uv run pytest -v -m "not integration" -# Run integration tests without 2GB+ downloads +# Run integration tests without the Identifiers.parquet download uv run pytest -v -m "integration and not slow" # Run a single test file @@ -250,7 +250,12 @@ than silently emitting the full cross-reference list. Tests live in `tests/` and are split into fast **unit tests** (mocked, no network) and slower **integration tests** (real downloads and API calls). Pytest markers control which tests run: - **`@pytest.mark.integration`** — requires network access (downloads Parquet files or calls NodeNorm API) -- **`@pytest.mark.slow`** — downloads very large files (2 GB+) +- **`@pytest.mark.slow`** — downloads `Identifiers.parquet`, the largest file Babel publishes + +Note that `not slow` is *not* the same as "small". `Concord.parquet` is itself multi-gigabyte in +current releases (4.6 GB in `2026jul22`) and its tests are not marked slow, because excluding them +would leave the non-slow integration set covering nothing that touches real data. Budget for that +before pointing CI at a Babel that publishes the Parquet files (see issue #18). Do not record per-file test counts here — they drift silently and then mislead. Get them on demand: diff --git a/README.md b/README.md index 0cbd1d5..d769b8a 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ skip when that release does not publish the DuckDB Parquet files. # Unit tests only — fast, no network required uv run pytest -v -m "not integration" -# Integration tests without 2GB+ downloads +# Integration tests without the Identifiers.parquet download uv run pytest -v -m "integration and not slow" # Full suite including large file downloads diff --git a/pyproject.toml b/pyproject.toml index 707c00c..5d12da4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,5 +59,5 @@ ignore = [ addopts = "-n auto" markers = [ "integration: tests requiring network access (deselect with '-m \"not integration\"')", - "slow: tests downloading very large files 2GB+ (deselect with '-m \"not slow\"')", + "slow: tests downloading Identifiers.parquet, the largest file Babel publishes (deselect with '-m \"not slow\"')", ] diff --git a/tests/conftest.py b/tests/conftest.py index d5b729b..f39ae03 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -87,7 +87,11 @@ def shared_downloader(test_data_dir) -> BabelDownloader: @pytest.fixture(scope="session") def downloaded_concord(shared_downloader, test_data_dir) -> str: - """Download duckdb/Concord.parquet (~626 MB). Returns the local path.""" + """Download duckdb/Concord.parquet. Returns the local path. + + Multi-gigabyte in current releases and growing; do not record a figure here, + it drifts silently and then misleads. + """ lock_path = os.path.join(test_data_dir, "concord.lock") with FileLock(lock_path): return shared_downloader.get_downloaded_file(CONCORD_FILE) @@ -112,7 +116,10 @@ def downloaded_parquet_files(downloaded_concord, downloaded_metadata) -> dict[st @pytest.fixture(scope="session") def downloaded_identifiers(shared_downloader, test_data_dir) -> str: - """Download duckdb/Identifiers.parquet (2 GB+). Returns the local path.""" + """Download duckdb/Identifiers.parquet, the largest file Babel publishes. + + Every test that reaches this is marked ``slow``. + """ lock_path = os.path.join(test_data_dir, "identifiers.lock") with FileLock(lock_path): return shared_downloader.get_downloaded_file(IDENTIFIERS_FILE) From 89dac95c5d1c1c8f9c6035e154bd37d01ff7f75a Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 02:58:24 -0400 Subject: [PATCH 114/123] Test that the committed config template cannot leak a private URL "Never commit the Translator-specific URL" has been a documented rule for as long as the repository has had a config template, and nothing enforced it. The URL was in fact the hardcoded default from the initial commit until history was rewritten to remove it, so the rule has already failed once in practice. TestCommittedConfigTemplate reads env.default and asserts that every host in it is public, that its defaults still compose to the CLI's default URL, that it documents exactly the settings the CLI reads, and that it does not resurrect BABEL_URL. Verified non-vacuous: injecting the internal URL fails it. Also pins the missing-Parquet error's wording. That message is where most people first learn the configuration scheme exists; it named BABEL_URL for as long as that variable did, and nothing would have caught the wording going stale when the variable was replaced. .gitignore gains .env.* alongside .env, because a .env.backup or .env.local holding that URL is precisely what a blanket `git add` would sweep up. env.default does not match the pattern and stays tracked. CLAUDE.md records what the history rewrite means for anyone with an older clone or an old PR link, and points at the test as the enforcement mechanism. Co-Authored-By: Claude Opus 5 --- .gitignore | 4 +++ CLAUDE.md | 15 +++++++++ tests/test_cli.py | 70 +++++++++++++++++++++++++++++++++++++++- tests/test_downloader.py | 24 ++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f3689c0..62355d6 100644 --- a/.gitignore +++ b/.gitignore @@ -143,7 +143,11 @@ celerybeat.pid *.sage.py # Environments +# .env.* as well as .env: a .env.backup or .env.local holding the Translator-specific +# releases URL is exactly what must never be committed, and a blanket `git add` would +# take it. env.default does not match this pattern and stays tracked. .env +.env.* .envrc .venv env/ diff --git a/CLAUDE.md b/CLAUDE.md index 7ec2ac8..ef53a11 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -282,6 +282,21 @@ broken test environment. - **`LabeledCrossReference`** — Extends CrossReference with labels and biolink types from NodeNorm - **`IdentifierRecord`** — Frozen dataclass for Identifiers.parquet rows (curie + dynamic extra fields, plus `nodenorm_label` under `--labels`). Returned by `BabelXRefs.get_curie_ids()`. The NodeNorm label is *not* called `label`: Identifiers.parquet has its own `label` column, which lands in `extra_fields` and would collide with it once the record is flattened for json/tsv/csv. +## Repository history was rewritten on 2026-09-01 + +Every commit was rewritten to remove an internal Babel URL that had been the hardcoded default +since the initial commit. Consequences a future contributor will trip over: + +- **A clone taken before that date has divergent history.** Every SHA changed except `gh-pages`. + Re-clone; do not try to merge or rebase the old history back together. +- **PRs #1, #4, #6, #7 and #11 are dead.** GitHub refuses to reopen a PR whose original head + commits no longer exist, so they were recreated as #20-#24. Old PR links and commit SHAs in + issue comments point at nothing. +- **`.env.*` is gitignored, `env.default` is not.** The URL leaked in the first place because it + was a default in source rather than configuration. `TestCommittedConfigTemplate` + (`tests/test_cli.py`) now fails if a non-public host appears in `env.default`; that test is the + enforcement, so do not weaken it to accommodate a convenient default. + ## Important Notes - **Data directory**: The `data/` directory is gitignored and contains downloaded Parquet files and generated DuckDB databases diff --git a/tests/test_cli.py b/tests/test_cli.py index 4af4e5f..740c6fd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,6 +5,8 @@ """ import json +import pathlib +import re from unittest.mock import MagicMock, patch import click @@ -13,7 +15,10 @@ from babel_explorer.cli import cli, parse_duration from babel_explorer.core.babel_xrefs import CrossReference, IdentifierRecord -from babel_explorer.core.downloader import MissingBabelFileError +from babel_explorer.core.downloader import ( + MissingBabelFileError, + compose_babel_url, +) from babel_explorer.core.nodenorm import Identifier # ========================================================================== @@ -663,6 +668,69 @@ def test_babel_version_rejects_urls_and_traversal(self, bad): assert "--babel-version" in result.output or "may not contain" in result.output +class TestCommittedConfigTemplate: + """env.default is the only config file that ships, so it is the one that can leak. + + CLAUDE.md and README.md both say the Translator-specific URL must never be committed. + Until now nothing enforced it, and the URL did in fact sit in this repository's git + history from the initial commit until it was rewritten out on 2026-09-01. A rule with + no test is a rule that comes back. + """ + + TEMPLATE = pathlib.Path(__file__).resolve().parent.parent / "env.default" + + def _settings(self) -> dict[str, str]: + settings = {} + for line in self.TEMPLATE.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, _, value = line.partition("=") + settings[key.strip()] = value.strip().strip("\"'") + return settings + + def test_template_is_the_committed_one(self): + """.env.example was renamed; nothing should resurrect it alongside env.default.""" + assert self.TEMPLATE.is_file() + assert not (self.TEMPLATE.parent / ".env.example").exists() + + def test_documents_every_setting_the_cli_reads(self): + """A setting the CLI honours but the template omits is one nobody discovers.""" + assert set(self._settings()) == { + "BABEL_RELEASES_URL", + "BABEL_VERSION", + "BABEL_LOCAL_DIR", + "BABEL_CHECK_DOWNLOAD", + "NODENORM_URL", + } + + def test_defaults_match_the_cli_defaults(self): + """A template that disagrees with the code silently changes what `cp` gives you.""" + settings = self._settings() + assert settings["BABEL_RELEASES_URL"] == "https://stars.renci.org/var/babel/" + assert settings["BABEL_VERSION"] == "latest" + assert ( + compose_babel_url(settings["BABEL_RELEASES_URL"], settings["BABEL_VERSION"]) + == "https://stars.renci.org/var/babel/latest/" + ) + + def test_defines_no_babel_url(self): + """BABEL_URL is inert. Shipping it would send people to a setting that does nothing.""" + assert "BABEL_URL" not in self._settings() + + def test_carries_no_non_public_url(self): + """The guard that matters: only public hosts, and never the internal outputs tree.""" + text = self.TEMPLATE.read_text() + for host in re.findall(r"https?://([^/\s\"']+)", text): + assert host in { + "stars.renci.org", + "nodenormalization-sri.renci.org", + }, f"{host} is not a public endpoint" + for path in re.findall(r"https?://\S+", text): + assert "/var/babel/" in path or "nodenormalization" in path, ( + f"{path} is not the public Babel or NodeNorm endpoint" + ) + + class TestMissingBabelFileReporting: """A missing Parquet file should read as an error, not a traceback.""" diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 4f75aef..d3be5b4 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -246,6 +246,30 @@ def test_404_raises_immediately(self, tmp_path): assert mock_get.call_count == 1, "a 404 must not be retried" + def test_404_message_names_the_current_setting(self, tmp_path): + """This message is where most people learn the config scheme exists. + + It named BABEL_URL for as long as that variable did; nothing caught the + wording when the variable was replaced. Pin it to the setting that works. + """ + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + # Preset so the patched requests.get is not also asked to resolve VERSION.txt. + dl.babel_version = "2025dec11" + response = MagicMock(status_code=404) + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + + with patch( + "babel_explorer.core.downloader.requests.get", return_value=response + ): + with pytest.raises(MissingBabelFileError) as excinfo: + dl.get_downloaded_file(CONCORD_FILE) + + message = str(excinfo.value) + assert "BABEL_RELEASES_URL" in message + assert "--babel-url" in message + assert "set BABEL_URL" not in message + # ========================================================================== # Unit Tests — no network required From f7ab3f76bf7fcf535e26052ec241617674e3b48c Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 03:33:34 -0400 Subject: [PATCH 115/123] Re-check cached Parquet files whenever the Babel release changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two paths let a cache holding the previous release be used against a marker naming the new one, so Concord.parquet and Identifiers.parquet could be read together across two Babel builds — the exact failure the version marker exists to prevent. _is_within_freshness() returned True on float("inf") before looking at last_checked, so the clearing sync_cache_version() does was a no-op under --check-download never: the old release's Parquet came back with no network call at all. Test the missing last_checked first. "never" means "do not re-check within a release", not "ignore a release change". sync_cache_version() also stamped the new release into .babel-version up front, so a run interrupted between the two Parquet files left a marker claiming a release the cache only half held; the next run saw it match and skipped the refresh. Hand the release to _write_version_marker_if_synced() instead, which writes it only once no .meta sidecar is still missing its last_checked. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 16 +++++ src/babel_explorer/core/downloader.py | 69 ++++++++++++++++++-- tests/test_downloader.py | 93 +++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ef53a11..eed0799 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,6 +69,22 @@ multi-gigabyte re-download. Partial `.tmp` downloads *are* deleted, so no prefix release survives into the next one. This keeps `Concord.parquet` and `Identifiers.parquet` from being read together across two different Babel releases. +`sync_cache_version()` does **not** write the new release to `.babel-version` itself. The marker +claims "the local cache holds this release", which is only true once every cached file has been +re-validated against it, so it is written by `_write_version_marker_if_synced()` after a download +instead — once no `.meta` sidecar in `duckdb/` is still missing its `last_checked`. Stamping it up +front would leave a run interrupted between `Concord.parquet` and `Identifiers.parquet` with a +marker naming the new release over a half-old cache, and the next run would see a marker that +matches and skip the refresh entirely. A cached file nobody asks for holds the marker back +indefinitely, costing one HEAD per run; that is correct, not a bug — the file really is still from +the previous release. + +`--check-download never` (`freshness_seconds=inf`) suppresses re-checks *within* a release, not +across one. `_is_within_freshness()` tests for a missing `last_checked` **before** the `inf` +shortcut, so a sidecar the version change expired is never fresh. Reordering those two lines +re-opens the whole hole: `never` would hand back the previous release's Parquet with no network +call at all. + A `.tmp` is deleted in two places, on purpose. The delete in `get_downloaded_file()` is the safety guarantee (see [Partial downloads](#partial-downloads)); the sweep in `sync_cache_version()` is housekeeping that reclaims gigabytes belonging to a release nobody will ask for again, including diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 976b67e..29e24b0 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -101,6 +101,9 @@ def __init__( self.freshness_seconds = freshness_seconds self.timeout = timeout self.logger = logging.getLogger(BabelDownloader.__name__) + # Release whose marker is waiting for the cache to catch up with it; see + # _write_version_marker_if_synced. + self._pending_version: str | None = None if local_path is None: local_path = tempfile.gettempdir() @@ -153,6 +156,10 @@ def sync_cache_version(self): Neither is redundant with the other for the case it owns. Removing this sweep only wastes disk; removing the one in ``get_downloaded_file`` reintroduces silent Parquet corruption — see the comment there before touching either. + + The new release is *not* written to the version marker here — it is handed to + ``_write_version_marker_if_synced``, which stamps it only once the cache actually + holds it. See that method for why writing it up front is unsafe. """ version = self.babel_version if version is None: @@ -190,8 +197,44 @@ def sync_cache_version(self): os.remove(tmp_path) if cached_version != version: - with open(marker_path, "w") as f: - f.write(version + "\n") + self._pending_version = version + # Usually a no-op here (the sidecars were just expired), but it covers the + # cases with nothing to catch up on: a first run with an empty cache, and a + # cache written before this tool kept a marker at all. + self._write_version_marker_if_synced() + + def _write_version_marker_if_synced(self): + """Record the pending release in ``.babel-version`` once the cache matches it. + + The marker claims "the local cache holds this release", and that only becomes + true once every cached file has been re-validated against it — so it is written + here, after downloads, rather than by ``sync_cache_version`` the moment the change + is noticed. Stamping it up front leaves a run that is interrupted after + ``Concord.parquet`` is refreshed but before ``Identifiers.parquet`` is with a + marker naming the new release over a half-old cache; the next run then sees a + marker that matches, does no version-driven refresh, and reads the two Parquet + files together across two different Babel releases. + + "Re-validated" is exactly "the sidecar has a ``last_checked`` again": + ``sync_cache_version`` cleared it from all of them, and only a confirmed-unchanged + HEAD or a completed download puts it back. A cached file that nobody asks for + therefore holds the marker back indefinitely, at a cost of one HEAD per run. That + is the honest answer rather than a bug — that file really is still from the + previous release. + """ + version = self._pending_version + if version is None: + return + + duckdb_dir = os.path.join(self.local_path, "duckdb") + for meta_path in glob.glob(os.path.join(duckdb_dir, "*.meta")): + meta = self._load_meta(meta_path.removesuffix(".meta")) + if meta is None or "last_checked" not in meta: + return + + with open(os.path.join(self.local_path, VERSION_MARKER), "w") as f: + f.write(version + "\n") + self._pending_version = None def _get_meta_path(self, local_path): """Return the sidecar metadata file path for a given local file.""" @@ -263,6 +306,15 @@ def _is_within_freshness(self, meta, freshness_seconds): """ Return True if last_checked is within freshness_seconds of now. + A file that has never been validated against the current release is never + fresh, whatever the window. That case is tested *before* the ``float('inf')`` + shortcut, and the order matters: ``sync_cache_version`` clears ``last_checked`` + from every sidecar when the Babel release changes, and if ``inf`` short-circuited + ahead of that, ``--check-download never`` would hand back the previous release's + Parquet without a single network call — while the version marker went on to name + the new release, so the mismatch would never be noticed again. ``never`` means + "do not re-check for changes *within* a release", not "ignore a release change". + Args: meta: dict loaded from .meta file freshness_seconds: Number of seconds; float('inf') means always fresh @@ -270,11 +322,11 @@ def _is_within_freshness(self, meta, freshness_seconds): Returns: bool """ - if freshness_seconds == float("inf"): - return True last_checked_str = meta.get("last_checked") if not last_checked_str: return False + if freshness_seconds == float("inf"): + return True try: last_checked = datetime.fromisoformat(last_checked_str) age = (datetime.now(UTC) - last_checked).total_seconds() @@ -527,6 +579,15 @@ def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): Returns: str: Local path to the downloaded file """ + local_path = self._fetch_file(dirpath, chunk_size) + # This file may have been the last one still holding the cache back from a + # release change sync_cache_version spotted. If so, this is where the version + # marker finally gets written. + self._write_version_marker_if_synced() + return local_path + + def _fetch_file(self, dirpath: str, chunk_size: int): + """Do the actual fetching for ``get_downloaded_file``, which see.""" local_path_to_download_to = os.path.join(self.local_path, dirpath) os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) diff --git a/tests/test_downloader.py b/tests/test_downloader.py index d3be5b4..efd8f1e 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -175,8 +175,52 @@ def test_changed_version_expires_meta_but_keeps_etag_and_parquet(self, tmp_path) "dropping the ETag would force an unconditional multi-gigabyte re-download" ) assert parquet.exists(), "the Parquet file itself must never be deleted" + + def test_changed_version_leaves_marker_until_the_cache_catches_up(self, tmp_path): + """A marker written up front makes an interrupted refresh look complete. + + If the marker named the new release straight away and the run died after + Concord was refreshed but before Identifiers was, the next run would see a + matching marker, skip the version-driven refresh entirely, and read the two + Parquet files together across two Babel releases. + """ + self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2025nov19" + + def test_marker_written_once_every_sidecar_is_revalidated(self, tmp_path): + _, meta = self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + dl = self._downloader(tmp_path, "2026jul22") + dl.sync_cache_version() + + # What a confirmed-unchanged HEAD or a completed download leaves behind. + dl._write_meta(str(meta).removesuffix(".meta"), json.loads(meta.read_text())) + dl._write_version_marker_if_synced() + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2026jul22" + def test_marker_withheld_while_one_cached_file_is_still_stale(self, tmp_path): + """Every cached file must be re-validated, not just the one that was asked for.""" + _, meta = self._seed_cache(tmp_path) + other = tmp_path / "duckdb" / "Identifiers.parquet.meta" + other.write_text( + json.dumps({"etag": '"def"', "last_checked": "2026-07-22T00:00:00+00:00"}) + ) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + dl = self._downloader(tmp_path, "2026jul22") + dl.sync_cache_version() + + dl._write_meta(str(meta).removesuffix(".meta"), json.loads(meta.read_text())) + dl._write_version_marker_if_synced() + + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2025nov19" + def test_changed_version_removes_partial_downloads(self, tmp_path): """A .tmp from the previous release must not be resumed against the new one.""" self._seed_cache(tmp_path) @@ -462,6 +506,16 @@ def test_returns_true_when_freshness_is_inf(self, tmp_path): meta = {"last_checked": old} assert dl._is_within_freshness(meta, float("inf")) is True + def test_returns_false_when_missing_last_checked_even_if_inf(self, tmp_path): + """`--check-download never` must not resurrect a sidecar the version change expired. + + sync_cache_version clears last_checked to force a re-check. If float('inf') + short-circuited ahead of that test, `never` would return the previous release's + Parquet with no network call at all. + """ + dl = self._make_dl(tmp_path) + assert dl._is_within_freshness({"etag": '"old"'}, float("inf")) is False + def test_returns_false_when_freshness_is_zero(self, tmp_path): dl = self._make_dl(tmp_path) just_now = datetime.now(UTC).isoformat() @@ -555,6 +609,45 @@ def test_tier1_returns_immediately_no_http(self, tmp_path): mock_get.assert_not_called() assert result == str(local) + def test_never_still_rechecks_after_a_release_change(self, tmp_path): + """`--check-download never` must not defeat the cross-release refresh. + + End-to-end version of the _is_within_freshness ordering: the cache holds the + previous release, sync_cache_version has expired the sidecar, and `never` still + has to issue the HEAD that notices the ETag changed. + """ + dl = self._make_dl(tmp_path, freshness=float("inf")) + test_file = "duckdb/test.parquet" + local = tmp_path / "duckdb" / "test.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data from the previous release") + + # No last_checked: exactly what sync_cache_version leaves behind. + with open(str(local) + ".meta", "w") as f: + json.dump({"etag": '"old"'}, f) + + mock_head_resp = Mock() + mock_head_resp.headers = {"ETag": '"new"'} + mock_head_resp.raise_for_status = Mock() + + def fake_download(url, tmp_path_, chunk_size): + """Stand in for the real download by writing the .tmp it would have left.""" + with open(tmp_path_, "wb") as f: + f.write(b"the new release") + return {"ETag": '"new"'} + + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp + ) as mock_head: + with patch.object( + dl, "_download_with_retry", side_effect=fake_download + ) as mock_download: + dl.get_downloaded_file(test_file) + + mock_head.assert_called_once() + mock_download.assert_called_once() + assert local.read_bytes() == b"the new release" + # --- Tier 2: stale .meta, ETag matches --- def test_tier2_head_check_no_redownload(self, tmp_path): From 9006a0dc94c15717c031cb8e76cacbca21d1a7c3 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 03:35:52 -0400 Subject: [PATCH 116/123] Reject `xrefs --paths` with one CURIE before downloading Concord.parquet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "at least two CURIEs" requirement lived in _print_paths, which runs after make_downloader(), the multi-gigabyte Concord.parquet download and the full recursive query — so `xrefs MONDO:0004979 --paths` fetched 4.6 GB and then said it needed another CURIE. --paths implies --recurse, so there is no cheap version of that mistake. Check it beside the --format guard, which was already up front. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 5 ++++- src/babel_explorer/cli.py | 26 ++++++++++++++++++-------- tests/test_cli.py | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index eed0799..d9e9e75 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -207,7 +207,10 @@ MONDO:0004979 "asthma" skos:exactMatch EFO:0000270 "asthma" This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels`, `ids --labels`, and `test-concord`. `--paths` is console-only; combining it with `--format json`/`tsv`/`csv` is rejected up front rather -than silently emitting the full cross-reference list. +than silently emitting the full cross-reference list. It also needs at least two CURIEs, and that +is checked in the same place, before `make_downloader()` — `--paths` implies `--recurse`, so +finding out inside `_print_paths()` would cost a multi-gigabyte download and a full recursive query +before rejecting the run. **When a label is absent, omit it entirely** — do not substitute a placeholder like `-` or `""`. A CURIE with no label renders as just the bare CURIE. diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index 7826bef..e7ac8a9 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -248,12 +248,13 @@ def _depth_of(curie: str, query_set: set, depth: int | None) -> int | None: def _print_paths(console, curies, xrefs_list, labels: bool) -> None: - """Print the shortest path between every pair of query CURIEs.""" - curie_list = list(curies) - if len(curie_list) < 2: - console.print("[yellow]--paths requires at least two CURIEs.[/yellow]") - return + """Print the shortest path between every pair of query CURIEs. + The caller checks the "at least two CURIEs" requirement up front — see ``xrefs`` — + so that a run that cannot produce a path never downloads Concord.parquet to find + that out. With fewer than two, ``combinations`` simply yields no pairs. + """ + curie_list = list(curies) query_set = set(curie_list) # One neighbour map for every pair: rebuilding it per pair re-walks the whole # recursive xref list C(n,2) times. @@ -402,14 +403,23 @@ def xrefs( :return: None """ if paths: - # Checked before anything is downloaded. Only the console renderer knows how to - # lay out paths; the other formats would silently emit the full recursive xref - # list instead, which looks like a successful --paths run but is not one. + # Both checks happen before anything is downloaded. --paths implies --recurse, + # so getting one of them wrong otherwise costs a multi-gigabyte Concord.parquet + # download and a full recursive query before the run is rejected. + # + # Only the console renderer knows how to lay out paths; the other formats would + # silently emit the full recursive xref list instead, which looks like a + # successful --paths run but is not one. if fmt != "console": raise click.UsageError( f"--paths is only supported with --format console, not --format {fmt}. " f"Drop --paths to emit the full cross-reference list as {fmt}." ) + if len(curies) < 2: + raise click.UsageError( + "--paths needs at least two CURIEs to find a path between. " + "Drop --paths to list the cross-references of a single CURIE." + ) recurse = True downloader = make_downloader( diff --git a/tests/test_cli.py b/tests/test_cli.py index 740c6fd..ec4b3b6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -791,6 +791,20 @@ def test_rejected_for_non_console_formats(self, fmt): # Rejected before anything is downloaded. mock_dl.assert_not_called() + def test_single_curie_rejected_before_downloading(self): + """--paths implies --recurse, so finding out late costs a 4.6 GB download.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs"), + patch("babel_explorer.cli.NodeNorm"), + ): + result = runner.invoke(cli, ["xrefs", "A:1", "--paths"]) + + assert result.exit_code != 0 + assert "--paths needs at least two CURIEs" in result.output + mock_dl.assert_not_called() + def test_allowed_for_console(self): runner = CliRunner() with ( From 99ccaf768178bdf121eb768f55c8dd66dfa416e6 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 03:36:17 -0400 Subject: [PATCH 117/123] Report an unreachable NodeNorm as an error rather than a traceback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BabelExplorerGroup.invoke mapped only MissingBabelFileError to a ClickException. NodeNorm deliberately lets requests.HTTPError and ConnectionError propagate so a failed lookup is not cached, and nothing caught them, so `xrefs --labels`, `ids --labels` and `test-concord` against a down or 5xx NodeNorm ended in a Python stack trace. Catch requests.RequestException alongside it. The failure lands mid-query rather than at startup because get_babel_version() swallows its own errors, so the version check passes against an unreachable NodeNorm. That is still the right trade — a NodeNorm that cannot report a version should not block a run — but it does mean the error has to be legible. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 6 ++++++ src/babel_explorer/cli.py | 16 +++++++++++++++- tests/test_cli.py | 26 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index d9e9e75..65f4fd8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,6 +97,12 @@ If a HEAD request fails, `_remote_unchanged()` returns `None` — "could not che the previous release's Parquet as freshly validated for the whole freshness window, immediately after `sync_cache_version()` cleared `last_checked` for a new release. +`BabelExplorerGroup.invoke()` turns `requests.RequestException` into a `ClickException`, so an +unreachable NodeNorm reports an error rather than a traceback. In practice only NodeNorm reaches +it: the downloader handles its own network failures, while NodeNorm deliberately lets HTTP errors +propagate so a failed lookup is not cached. `get_babel_version()` swallows its own errors, so an +unreachable NodeNorm passes the version check below and only fails part-way through the query. + `xrefs` fails when NodeNorm's `status` endpoint reports a different `babel_version` than the Babel being queried, since labels and cliques would not match the cross-references. Pass `--allow-version-mismatch` to proceed anyway. The check is skipped when NodeNorm is not consulted diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py index e7ac8a9..7c712c1 100644 --- a/src/babel_explorer/cli.py +++ b/src/babel_explorer/cli.py @@ -6,6 +6,7 @@ from itertools import combinations import click +import requests from dotenv import load_dotenv from rich.markup import escape @@ -331,13 +332,26 @@ def _print_paths(console, curies, xrefs_list, labels: bool) -> None: class BabelExplorerGroup(click.Group): - """Group that reports missing Babel files as a plain error rather than a traceback.""" + """Group that reports service failures as a plain error rather than a traceback.""" def invoke(self, ctx): try: return super().invoke(ctx) except MissingBabelFileError as e: raise click.ClickException(str(e)) from e + except requests.RequestException as e: + # In practice this is always NodeNorm: the downloader handles its own + # network failures (a failed HEAD falls back to the cached file, and + # _download_with_retry re-raises as RuntimeError after its last attempt), + # while NodeNorm deliberately lets HTTP errors propagate so a failed + # lookup is not cached. Note that get_babel_version() swallows its own + # errors, so an unreachable NodeNorm passes the version check and only + # fails here, part-way through a query. + raise click.ClickException( + f"NodeNorm request failed: {e}. Check that --nodenorm-url is " + f"reachable; `xrefs` and `ids` can also be run without --labels, " + f"which does not consult NodeNorm at all." + ) from e @click.group(cls=BabelExplorerGroup) diff --git a/tests/test_cli.py b/tests/test_cli.py index ec4b3b6..fe66820 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,6 +11,7 @@ import click import pytest +import requests from click.testing import CliRunner from babel_explorer.cli import cli, parse_duration @@ -771,6 +772,31 @@ def test_also_wrapped_for_ids(self): assert "Traceback" not in result.output +class TestNodeNormFailureIsNotATraceback: + """An unreachable NodeNorm must not end the run in a Python stack trace.""" + + def test_connection_error_becomes_a_click_error(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = "2026jul22" + # get_babel_version() swallows its own errors, so the version check passes + # and the failure only surfaces once the query is under way. + mock_nn.return_value.get_babel_version.return_value = None + mock_bx.return_value.get_curie_xrefs.side_effect = requests.ConnectionError( + "connection refused" + ) + result = runner.invoke(cli, ["xrefs", "A:1", "--labels"]) + + assert result.exit_code == 1 + assert "NodeNorm request failed" in result.output + assert "connection refused" in result.output + assert "Traceback" not in result.output + + class TestPathsFormatGuard: """--paths only has a renderer for the console format.""" From 8eb76335fa75807fd692dcefe46e872e19426f59 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 03:36:35 -0400 Subject: [PATCH 118/123] Spill DuckDB query temporaries into the cache dir, not the working directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BabelXRefs opened connections with a bare duckdb.connect(), whose default temp_directory is `.tmp` in the *current working directory*. Nothing is persisted, but "in-memory" is not "touches no disk": the recursive expansion materialises the whole Concord relation (4.6 GB in 2026jul22) plus a doubled edges relation, so a real `--recurse` run dropped gigabytes of spill wherever the user happened to be standing. Route every connection through BabelXRefs._connect(), which points temp_directory at /duckdb-spill/ — the directory the user already chose to hold multi-gigabyte Parquet files. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 10 ++++++++-- src/babel_explorer/core/babel_xrefs.py | 21 ++++++++++++++++++--- tests/test_babel_xrefs.py | 14 ++++++++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 65f4fd8..2cf5205 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -240,7 +240,11 @@ before rejecting the run. - Main query engine for cross-references - Uses DuckDB to query Parquet files (`Concord.parquet`, `Identifiers.parquet`) - Supports recursive expansion of cross-references via a single `WITH RECURSIVE` query - - Uses ephemeral in-memory DuckDB connections (nothing written to disk) + - Uses ephemeral in-memory DuckDB connections, opened by `BabelXRefs._connect()`. No + database is persisted, but larger-than-memory queries spill to disk: `_connect()` sets + `temp_directory` to `/duckdb-spill/`, because DuckDB's default is + `.tmp` in the *current working directory* and a `--recurse` run materialises the whole + multi-gigabyte Concord relation 3. **NodeNorm** (`src/babel_explorer/core/nodenorm.py`): - Integration with NodeNormalization API (https://nodenormalization-sri.renci.org/) @@ -266,7 +270,8 @@ before rejecting the run. - **Lazy downloading**: Files are only downloaded when first accessed - **LRU caching**: Heavy use of `@functools.lru_cache` to avoid redundant downloads and API calls - **Recursive expansion**: The `--recurse` flag recursively follows all cross-references to build complete graphs -- **DuckDB for querying**: In-memory SQL queries against Parquet files for fast lookups +- **DuckDB for querying**: In-memory SQL queries against Parquet files for fast lookups, spilling + to `/duckdb-spill/` rather than the working directory ## Testing @@ -334,5 +339,6 @@ since the initial commit. Consequences a future contributor will trip over: - Tests: `tests/` - Test CURIEs: `tests/data/valid_curies.txt` - Downloaded Babel files: `/duckdb/*.parquet` (default `data/duckdb/`) +- DuckDB query spill: `/duckdb-spill/` (default `data/duckdb-spill/`) - Endpoint configuration: `.env` (gitignored), template in `env.default` - Entry point: `src/babel_explorer/cli.py` diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 407639a..75f8f16 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -7,6 +7,7 @@ import dataclasses import logging +import os from collections import deque import duckdb @@ -161,6 +162,20 @@ def __init__(self, downloader: BabelDownloader, nodenorm: NodeNorm = None): self.nodenorm = nodenorm self._xref_cache: dict = {} + def _connect(self): + """Open an ephemeral in-memory DuckDB connection that spills into the cache dir. + + Nothing is persisted, but "in-memory" is not the same as "touches no disk": + DuckDB's default ``temp_directory`` is ``.tmp`` in the *current working + directory*, and the recursive expansion materialises a multi-gigabyte Concord + relation. Left at the default, a ``--recurse`` run drops gigabytes of spill files + wherever the user happened to be standing. Point it at the directory the Parquet + files already live in, which the user chose knowing it holds bulk data. + """ + spill_dir = os.path.join(self.downloader.local_path, "duckdb-spill") + os.makedirs(spill_dir, exist_ok=True) + return duckdb.connect(config={"temp_directory": spill_dir}) + def _require_nodenorm(self): if self.nodenorm is None: raise ValueError( @@ -187,7 +202,7 @@ def get_curie_ids( ) # Query the Parquet files using DuckDB (in-memory; nothing is persisted). - with duckdb.connect() as db: + with self._connect() as db: result = db.execute( "SELECT * FROM read_parquet($1) WHERE curie IN (SELECT unnest($2::VARCHAR[]))", [identifier_parquet, list(curies)], @@ -236,7 +251,7 @@ def _query_xrefs(self, curies: list[str], label_curies: bool = False) -> list: concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") - with duckdb.connect() as db: + with self._connect() as db: xref_tuples = db.execute( """ SELECT filename, subj, pred, obj FROM read_parquet($1) @@ -296,7 +311,7 @@ def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = Fal concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") - with duckdb.connect() as db: + with self._connect() as db: rows = db.execute( """ WITH RECURSIVE diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 29d4cb3..6938d7b 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -204,6 +204,20 @@ def test_init_with_nodenorm(self, tmp_path): bx = BabelXRefs(dl, nn) assert bx.nodenorm is nn + def test_connect_spills_inside_the_cache_directory(self, tmp_path): + """DuckDB's default temp_directory is `.tmp` in the *current* directory. + + The recursive expansion materialises a multi-gigabyte Concord relation, so + leaving that at the default drops gigabytes of spill wherever the user ran the + command. + """ + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + with BabelXRefs(dl)._connect() as db: + setting = db.execute("SELECT current_setting('temp_directory')").fetchone() + + assert setting[0] == str(tmp_path / "duckdb-spill") + assert (tmp_path / "duckdb-spill").is_dir() + class TestBabelXRefsMocked: """Mocked query tests — no DuckDB or Parquet files needed.""" From 4d229c4835f1772de87f1b55aeb7e58a546f6754 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 03:36:53 -0400 Subject: [PATCH 119/123] Describe the caching that exists, and stop counting tests in the changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md claimed `@functools.lru_cache` in three places; `grep -rn lru_cache src/` returns nothing. BabelDownloader caches on disk via ETags and `.meta` sidecars and memoises only `babel_version` (with `cached_property`); NodeNorm uses three plain per-instance dicts. A reader chasing LRU eviction semantics was chasing nothing. The changelog's "28 of 288 tests" was already wrong — collection reports 294 — and CLAUDE.md says not to record per-file test counts precisely because they drift silently and then mislead. The same reasoning applies here. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 5 +++-- CLAUDE.md | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67b5ce9..77fe73a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,5 +35,6 @@ label enrichment from NodeNorm. public `latest` is `2025dec11`, so the skew check fires. Pin `BABEL_VERSION` to the release NodeNorm was built from, or pass `--allow-version-mismatch`. Tracked in [#17](https://github.com/TranslatorSRI/babel-explorer/issues/17). -- 28 of 288 tests are integration tests that skip without a Babel release publishing the Parquet - files. Tracked in [#18](https://github.com/TranslatorSRI/babel-explorer/issues/18). +- The integration tests skip without a Babel release publishing the Parquet files, so a default + run exercises only the unit suite. Tracked in + [#18](https://github.com/TranslatorSRI/babel-explorer/issues/18). diff --git a/CLAUDE.md b/CLAUDE.md index 2cf5205..ea0fe74 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -231,7 +231,8 @@ before rejecting the run. 1. **BabelDownloader** (`src/babel_explorer/core/downloader.py`): - Downloads Babel intermediate files from a remote HTTP(S) server using Python's `requests` library (streaming downloads) - Caches files locally in a configurable directory (default: `data/`), one Babel release at a time - - Uses `@functools.lru_cache` to avoid re-downloading + - Caching is on disk, keyed by ETag and the `.meta` sidecars — not in memory. Only + `babel_version` is memoised, via `functools.cached_property` - Resolves the Babel version (`resolve_babel_version`) and refreshes the cache when it changes (`sync_cache_version`) - Raises `MissingBabelFileError` on a 404 for a `duckdb/` file, since public releases do not publish them - **Important**: Requires network access but no external tools like `wget` @@ -249,7 +250,8 @@ before rejecting the run. 3. **NodeNorm** (`src/babel_explorer/core/nodenorm.py`): - Integration with NodeNormalization API (https://nodenormalization-sri.renci.org/) - Fetches labels, biolink types, and equivalent identifiers for CURIEs - - Uses `@functools.lru_cache` for performance + - Caches normalisation results, identifiers and cliques in per-instance dicts; a new + `NodeNorm` object is the way to get uncached results - `get_babel_version()` reads the `status` endpoint to report which Babel release it was built from - Optional component for label enrichment @@ -268,7 +270,8 @@ before rejecting the run. ### Key Design Patterns - **Lazy downloading**: Files are only downloaded when first accessed -- **LRU caching**: Heavy use of `@functools.lru_cache` to avoid redundant downloads and API calls +- **Caching**: Downloads are cached on disk (ETag + `.meta` sidecar); NodeNorm results are cached + in per-instance dicts. Neither uses `functools.lru_cache` - **Recursive expansion**: The `--recurse` flag recursively follows all cross-references to build complete graphs - **DuckDB for querying**: In-memory SQL queries against Parquet files for fast lookups, spilling to `/duckdb-spill/` rather than the working directory From 8be1143f725bddf72ffab0d38c69c6aa8e77c397 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 08:25:00 -0400 Subject: [PATCH 120/123] Never resume a download the server gave no validator for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _download_with_retry sent a bare Range whenever a retry found bytes on disk, adding If-Range only if a validator happened to be known. A server that supplies neither an ETag nor a Last-Modified therefore got an unconditional resume — and that is precisely the case where a file rebuilt between attempts splices its tail onto the old version's prefix undetectably, since what lands is then stamped with the new validator and passes every later freshness check. Having a validator is now a precondition for resuming at all; without one the partial file is discarded and the download restarts. Two existing tests encoded the old behaviour by seeding a partial file with no validator, a state get_downloaded_file never produces. They now reach the resume through a first attempt that ends short, which is how production gets there. Copilot (suppressed): src/babel_explorer/core/downloader.py — an HTTP 416 whose HEAD carries no Content-Length was treated as "already complete". 416 is also how a server answers when the file shrank below the resume offset, and with no remote length there is nothing to tell the two apart, so it now restarts as well. The restart cannot loop: the retry has nothing on disk and so sends no Range. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 13 ++++- src/babel_explorer/core/downloader.py | 42 ++++++++++---- tests/test_downloader.py | 80 +++++++++++++++++++++++---- 3 files changed, 112 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ea0fe74..3fedae1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -121,12 +121,19 @@ permanent, because the file gets stamped with the *correct* ETag: orphaned `.tmp` carries no record of which version its bytes came from. Restarting costs a re-download; splicing costs silent data corruption. Do not "optimise" this back into a cross-run resume without persisting the validator alongside the `.tmp`. -- **In-run resumes send `If-Range`** with the validator from the response they started writing - from, so a file rebuilt mid-download restarts (HTTP 200) instead of splicing. +- **A resume requires a validator, and sends it as `If-Range`.** The validator is the ETag (else + Last-Modified) of the response the bytes on disk were written from, so a file rebuilt + mid-download restarts (HTTP 200) instead of splicing. A server that supplies neither leaves + nothing to make the resume conditional on, so `_download_with_retry()` discards the partial file + and restarts from zero rather than sending a bare `Range`. Do not relax that into "send `Range`, + add `If-Range` when we happen to have one": the case with no validator is exactly the one where + a splice cannot be detected afterwards. - **Sizes are checked, twice.** A stream that ends short of `Content-Length` raises `IncompleteDownloadError` and is retried, rather than being promoted as complete; and an HTTP 416 is only treated as "already complete" once the local size matches the remote - `Content-Length`, since 416 also means the remote file *shrank* below the resume offset. + `Content-Length`, since 416 also means the remote file *shrank* below the resume offset. A HEAD + that reports no `Content-Length` at all is "cannot confirm", not "complete", and restarts too — + which cannot loop, because the retry has nothing on disk and so sends no `Range`. `_save_meta()` records the length of the whole file, taken from `Content-Range` rather than a 206 response's `Content-Length` (which is only the range's length). Storing the partial length would diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py index 29e24b0..0897852 100644 --- a/src/babel_explorer/core/downloader.py +++ b/src/babel_explorer/core/downloader.py @@ -467,6 +467,8 @@ def _download_with_retry(self, url, local_path, chunk_size): # the old version's prefix — a splice that would pass every later ETag check. # A leftover .tmp from an earlier run carries no validator and is never # resumed; get_downloaded_file removes it before we are called. + # + # Having one is a *precondition* for resuming at all: see the restart below. validator = None for attempt in range(1, self.retries + 1): @@ -475,11 +477,27 @@ def _download_with_retry(self, url, local_path, chunk_size): if os.path.exists(local_path): resume_byte_pos = os.path.getsize(local_path) + if resume_byte_pos > 0 and not validator: + # Nothing to make the resume conditional on. A server that sent + # neither an ETag nor a Last-Modified leaves no way to ask for + # "the rest of *this* file", so a bare Range against a file + # rebuilt between attempts splices the new version's tail onto + # the old version's prefix — the very corruption If-Range exists + # to prevent, and just as permanent, since what lands gets + # stamped with the new validator and passes every later check. + # Such a server costs a restart instead. + self.logger.warning( + f"Restarting {local_path} from the beginning: the response it " + f"was written from carried no ETag or Last-Modified, so the " + f"resume cannot be made conditional" + ) + os.remove(local_path) + resume_byte_pos = 0 + headers = {} if resume_byte_pos > 0: headers["Range"] = f"bytes={resume_byte_pos}-" - if validator: - headers["If-Range"] = validator + headers["If-Range"] = validator self.logger.info(f"Resuming download from byte {resume_byte_pos}") # timeout is per-read (seconds without receiving bytes), not a total time limit. @@ -489,22 +507,26 @@ def _download_with_retry(self, url, local_path, chunk_size): if response.status_code == 416: # 416 also comes back when the remote file *shrank* below our # resume offset, so "the range is past the end" does not by - # itself mean the local file is the remote one. Check the size - # before promoting it, or a rebuild that produced a smaller - # Parquet leaves an over-long file with valid-looking metadata. + # itself mean the local file is the remote one. Only a remote + # length that matches the local one proves that, so a missing + # Content-Length is treated as "cannot confirm" and restarts + # too: promoting an unverified file leaves a wrong-length + # Parquet stamped with valid-looking metadata. The restart is + # safe from looping, since the retry sends no Range at all. head = requests.head(url, timeout=self.timeout) head.raise_for_status() remote_length = head.headers.get("Content-Length") if ( - remote_length is not None - and int(remote_length) != resume_byte_pos + remote_length is None + or int(remote_length) != resume_byte_pos ): self.logger.warning( f"Local file is {resume_byte_pos} bytes but the remote " - f"file is {remote_length}; discarding it and " - f"downloading afresh" + f"length is {remote_length or 'unknown'}; discarding it " + f"and downloading afresh" ) - os.remove(local_path) + if os.path.exists(local_path): + os.remove(local_path) continue self.logger.info(f"File already complete: {local_path}") # The 416 headers describe the error body, not the file; saving diff --git a/tests/test_downloader.py b/tests/test_downloader.py index efd8f1e..eb2aee5 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -929,11 +929,13 @@ def test_truncated_stream_is_retried_and_resumed(self, tmp_path): ) out_path = str(tmp_path / "resumed.bin") + # The ETag is what makes the resume conditional, and so allowed at all: + # without a validator the retry restarts from zero instead. first = TestDownloadWithRetry._make_response( - 200, {"Content-Length": "10"}, [b"abcd"] + 200, {"Content-Length": "10", "ETag": '"v1"'}, [b"abcd"] ) second = TestDownloadWithRetry._make_response( - 206, {"Content-Length": "6"}, [b"efghij"] + 206, {"Content-Length": "6", "ETag": '"v1"'}, [b"efghij"] ) with ( patch( @@ -966,8 +968,12 @@ def test_416_with_a_shorter_remote_file_restarts(self, tmp_path): url_base="https://example.com/", local_path=str(tmp_path), retries=3 ) out_path = tmp_path / "shrunk.bin" - out_path.write_bytes(b"the old, longer file") # 20 bytes + # Attempt 1 ends short, leaving 20 bytes and a validator behind. Only then is + # there a Range to send, and so a 416 to receive. + truncated = TestDownloadWithRetry._make_response( + 200, {"Content-Length": "30", "ETag": '"old"'}, [b"the old, longer file"] + ) too_long = TestDownloadWithRetry._make_response(416) fresh = TestDownloadWithRetry._make_response( 200, {"Content-Length": "5", "ETag": '"new"'}, [b"short"] @@ -976,9 +982,10 @@ def test_416_with_a_shorter_remote_file_restarts(self, tmp_path): with ( patch( "babel_explorer.core.downloader.requests.get", - side_effect=[too_long, fresh], + side_effect=[truncated, too_long, fresh], ), patch("babel_explorer.core.downloader.requests.head", return_value=head), + patch("babel_explorer.core.downloader.time.sleep"), ): headers = dl._download_with_retry( "https://example.com/file", str(out_path), 1024 @@ -1081,32 +1088,49 @@ def test_succeeds_on_second_attempt(self, tmp_path): dl._download_with_retry("https://example.com/file", out_path, 1024) assert os.path.exists(out_path) - def test_resume_sends_range_header(self, tmp_path): + def test_resume_without_a_validator_restarts_from_zero(self, tmp_path): + """A bare Range is a splice waiting to happen. + + With neither an ETag nor a Last-Modified there is no If-Range to send, so + nothing stops a server that rebuilt the file from handing back the new + version's tail to append to the old version's prefix — and the result would + be stamped with the new validator and pass every later check. Restarting + costs a re-download; resuming costs silent corruption. + """ dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) out_path = tmp_path / "partial.bin" out_path.write_bytes(b"partial") # 7 bytes - mock_response = self._make_response(206, {"Content-Length": "3"}, [b"end"]) + mock_response = self._make_response(200, {"Content-Length": "5"}, [b"whole"]) with patch( "babel_explorer.core.downloader.requests.get", return_value=mock_response ) as mock_get: dl._download_with_retry("https://example.com/file", str(out_path), 1024) _, kwargs = mock_get.call_args - assert kwargs["headers"] == {"Range": "bytes=7-"} + assert kwargs["headers"] == {}, "no validator means no conditional resume" + assert out_path.read_bytes() == b"whole" def test_http_416_file_already_complete(self, tmp_path): - dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) out_path = tmp_path / "complete.bin" - out_path.write_bytes(b"full file") + # Attempt 1 over-declares Content-Length and delivers the whole 9-byte file, + # so the size check retries it; attempt 2 then asks for bytes past the end of + # a file it already holds in full, which is what a 416 legitimately means. + over_declared = self._make_response( + 200, {"Content-Length": "20", "ETag": '"v1"'}, [b"full file"] + ) mock_response = self._make_response(416) head = MagicMock(status_code=200, headers={"Content-Length": "9"}) with ( patch( "babel_explorer.core.downloader.requests.get", - return_value=mock_response, + side_effect=[over_declared, mock_response], ), patch("babel_explorer.core.downloader.requests.head", return_value=head), + patch("babel_explorer.core.downloader.time.sleep"), ): headers = dl._download_with_retry( "https://example.com/file", str(out_path), 1024 @@ -1116,6 +1140,42 @@ def test_http_416_file_already_complete(self, tmp_path): # metadata would fail every later freshness check and re-download the file. assert headers == {"Content-Length": "9"} + def test_416_without_a_content_length_restarts(self, tmp_path): + """416 alone does not prove completeness — it is also how a shrunk file answers. + + With no remote length to compare against there is no way to tell the two + apart, so the local file must not be promoted as complete. + """ + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = tmp_path / "unverifiable.bin" + + # Attempt 1 ends short, leaving bytes and a validator behind, so attempt 2 + # sends the Range that draws the 416. + truncated = self._make_response( + 200, {"Content-Length": "30", "ETag": '"old"'}, [b"possibly stale bytes"] + ) + range_rejected = self._make_response(416) + fresh = self._make_response( + 200, {"Content-Length": "5", "ETag": '"new"'}, [b"fresh"] + ) + head = MagicMock(status_code=200, headers={}) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[truncated, range_rejected, fresh], + ), + patch("babel_explorer.core.downloader.requests.head", return_value=head), + patch("babel_explorer.core.downloader.time.sleep"), + ): + headers = dl._download_with_retry( + "https://example.com/file", str(out_path), 1024 + ) + + assert out_path.read_bytes() == b"fresh" + assert headers["ETag"] == '"new"' + def test_server_no_resume_restarts_download(self, tmp_path): """When server responds 200 (instead of 206), partial file is removed and download restarts.""" dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) From 892bc285916235e7caf945949a14592b85334bc9 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 08:25:12 -0400 Subject: [PATCH 121/123] Give BabelXRefs a public cache reset the integration tests can call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The integration tests cleared the cross-reference cache with `get_curie_xref.cache_clear()`, left over from when that method was decorated with functools.lru_cache. It is a plain method over a per-instance dict now, so all ten calls raised AttributeError — invisible because every one of those tests skips without a Babel release publishing the Parquet files, which is the case for every public release. Add BabelXRefs.clear_xref_cache() and call that instead, with a unit test so the method's existence is covered by the suite that actually runs. Co-Authored-By: Claude Opus 5 --- src/babel_explorer/core/babel_xrefs.py | 9 +++++++ tests/test_babel_xrefs.py | 36 +++++++++++++++++++------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py index 75f8f16..068a063 100644 --- a/src/babel_explorer/core/babel_xrefs.py +++ b/src/babel_explorer/core/babel_xrefs.py @@ -176,6 +176,15 @@ def _connect(self): os.makedirs(spill_dir, exist_ok=True) return duckdb.connect(config={"temp_directory": spill_dir}) + def clear_xref_cache(self) -> None: + """Discard the per-CURIE cross-reference cache. + + The cache is per instance and keyed by ``(curie, label_curies)``, so it is + normally left alone. Tests that want a query to actually hit Parquet — rather + than a result an earlier test in the same session put there — clear it first. + """ + self._xref_cache.clear() + def _require_nodenorm(self): if self.nodenorm is None: raise ValueError( diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py index 6938d7b..87497b1 100644 --- a/tests/test_babel_xrefs.py +++ b/tests/test_babel_xrefs.py @@ -204,6 +204,22 @@ def test_init_with_nodenorm(self, tmp_path): bx = BabelXRefs(dl, nn) assert bx.nodenorm is nn + def test_clear_xref_cache_empties_the_cache(self, tmp_path): + """The integration tests reset the cache between queries and must reach it. + + They used to call `get_curie_xref.cache_clear()`, left over from when the + method was decorated with lru_cache. It is a plain method over a per-instance + dict now, so that raised AttributeError — invisible because those tests skip + without a Babel release publishing the Parquet files. + """ + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + bx = BabelXRefs(dl) + bx._xref_cache[("A:1", False)] = [] + + bx.clear_xref_cache() + + assert bx._xref_cache == {} + def test_connect_spills_inside_the_cache_directory(self, tmp_path): """DuckDB's default temp_directory is `.tmp` in the *current* directory. @@ -411,7 +427,7 @@ def test_shared_adjacency_matches_a_freshly_built_one(self): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_curie_xref(babel_xrefs, curie): """get_curie_xref returns non-empty CrossReferences with the queried CURIE.""" - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() results = list(babel_xrefs.get_curie_xref(curie)) assert len(results) > 0, f"No cross-references found for {curie}" for xr in results: @@ -423,7 +439,7 @@ def test_get_curie_xref(babel_xrefs, curie): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_curie_xref_returns_known_xrefs(babel_xrefs, curie): """At least one cross-reference is found.""" - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() results = list(babel_xrefs.get_curie_xref(curie)) assert len(results) >= 1 @@ -432,7 +448,7 @@ def test_get_curie_xref_returns_known_xrefs(babel_xrefs, curie): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_curie_xrefs_single_no_expand(babel_xrefs, curie): """get_curie_xrefs without expansion returns sorted, non-empty results.""" - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() results = babel_xrefs.get_curie_xrefs([curie], recurse=False) assert len(results) > 0 assert results == sorted(results) @@ -442,9 +458,9 @@ def test_get_curie_xrefs_single_no_expand(babel_xrefs, curie): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_curie_xrefs_expansion_finds_more(babel_xrefs, curie): """Expanded results are at least as many as non-expanded.""" - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() non_expanded = babel_xrefs.get_curie_xrefs([curie], recurse=False) - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() expanded = babel_xrefs.get_curie_xrefs([curie], recurse=True) assert len(expanded) >= len(non_expanded) @@ -453,9 +469,9 @@ def test_get_curie_xrefs_expansion_finds_more(babel_xrefs, curie): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_curie_xrefs_expanded_includes_original(babel_xrefs, curie): """Non-expanded results are a subset of expanded results.""" - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() non_expanded = set(babel_xrefs.get_curie_xrefs([curie], recurse=False)) - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() expanded = set(babel_xrefs.get_curie_xrefs([curie], recurse=True)) assert non_expanded.issubset(expanded) @@ -464,7 +480,7 @@ def test_get_curie_xrefs_expanded_includes_original(babel_xrefs, curie): def test_get_curie_xref_caching(babel_xrefs): """Cached calls return the same object.""" curie = VALID_CURIES[0] - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() r1 = babel_xrefs.get_curie_xref(curie) r2 = babel_xrefs.get_curie_xref(curie) assert r1 is r2 @@ -474,7 +490,7 @@ def test_get_curie_xref_caching(babel_xrefs): @pytest.mark.parametrize("curie", VALID_CURIES) def test_get_curie_xref_with_labels(babel_xrefs_with_nodenorm, curie): """With labels, returns LabeledCrossReference objects.""" - babel_xrefs_with_nodenorm.get_curie_xref.cache_clear() + babel_xrefs_with_nodenorm.clear_xref_cache() results = list(babel_xrefs_with_nodenorm.get_curie_xref(curie, label_curies=True)) assert len(results) > 0 for xr in results: @@ -484,7 +500,7 @@ def test_get_curie_xref_with_labels(babel_xrefs_with_nodenorm, curie): @pytest.mark.integration def test_get_curie_xref_nonexistent_curie(babel_xrefs): """A made-up CURIE returns an empty list.""" - babel_xrefs.get_curie_xref.cache_clear() + babel_xrefs.clear_xref_cache() results = list(babel_xrefs.get_curie_xref("FAKE:9999999999")) assert results == [] From 1c3bc598e771c85b1649dc1f5b6f170e7e526b42 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 08:25:23 -0400 Subject: [PATCH 122/123] Derive the config template's expected settings from the CLI itself BABEL_ALLOW_VERSION_MISMATCH is read from the environment and documented in both CLAUDE.md and the changelog, but env.default omitted it, so copying the template did not in fact give you every supported setting. The omission survived because the test guarding that rule listed the settings by hand: `test_documents_every_setting_the_cli_reads` asserted against a hard-coded set, which passes just as happily when a new option is added to the CLI and forgotten in the template. It now reads the envvar= declarations off the Click commands, so the next setting cannot go undocumented the same way. Co-Authored-By: Claude Opus 5 --- env.default | 6 ++++++ tests/test_cli.py | 23 ++++++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/env.default b/env.default index 922c778..b60e934 100644 --- a/env.default +++ b/env.default @@ -26,6 +26,12 @@ BABEL_CHECK_DOWNLOAD=3h NODENORM_URL=https://nodenormalization-sri.renci.org/ +# Proceed even when NodeNorm reports a different Babel release than the one being +# queried. Off by default: labels and cliques from one release do not match another's +# cross-references, so the results would be wrong without looking wrong. Pinning +# BABEL_VERSION to the release NodeNorm was built from is the real fix. +BABEL_ALLOW_VERSION_MISMATCH=false + # NOTE: public Babel releases do not currently publish the DuckDB Parquet files # (duckdb/Concord.parquet, duckdb/Identifiers.parquet) that babel-explorer needs, so the # defaults above will report that the files are missing. Translator team members should diff --git a/tests/test_cli.py b/tests/test_cli.py index fe66820..b2fd53b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -694,15 +694,24 @@ def test_template_is_the_committed_one(self): assert self.TEMPLATE.is_file() assert not (self.TEMPLATE.parent / ".env.example").exists() + @staticmethod + def _envvars_the_cli_reads() -> set[str]: + """Every envvar= declared on any command's options, read off the CLI itself. + + Derived rather than listed: a hard-coded set passes just as happily when a new + setting is added to the CLI and forgotten here, which is how + BABEL_ALLOW_VERSION_MISMATCH went undocumented in the template. + """ + return { + param.envvar + for command in cli.commands.values() + for param in command.params + if getattr(param, "envvar", None) + } + def test_documents_every_setting_the_cli_reads(self): """A setting the CLI honours but the template omits is one nobody discovers.""" - assert set(self._settings()) == { - "BABEL_RELEASES_URL", - "BABEL_VERSION", - "BABEL_LOCAL_DIR", - "BABEL_CHECK_DOWNLOAD", - "NODENORM_URL", - } + assert set(self._settings()) == self._envvars_the_cli_reads() def test_defaults_match_the_cli_defaults(self): """A template that disagrees with the code silently changes what `cp` gives you.""" From ef2b6731aa9cf68679ad485b00b873543c19abe1 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Tue, 1 Sep 2026 08:25:23 -0400 Subject: [PATCH 123/123] Drop batched NodeNorm lookups from the future-work list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #12 asked for a `normalize_curies(curies)` batch method to collapse N serial round-trips into one. That shipped in this branch: NodeNorm batches at 100 CURIEs per request, and all three label paths pre-warm through it — `ids`, `xrefs --labels` and `test-concord`. Listing it as future work contradicted the code next to it. #13 is untouched and still open. Co-Authored-By: Claude Opus 5 --- FUTURE.md | 1 - 1 file changed, 1 deletion(-) diff --git a/FUTURE.md b/FUTURE.md index d6612f7..9b22363 100644 --- a/FUTURE.md +++ b/FUTURE.md @@ -2,5 +2,4 @@ These items are tracked as GitHub issues: -- [#12](https://github.com/TranslatorSRI/babel-explorer/issues/12) — Batch NodeNorm lookups to reduce N round-trips when `--labels` is set - [#13](https://github.com/TranslatorSRI/babel-explorer/issues/13) — Reuse a single DuckDB connection per `BabelXRefs` instance