diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c50f042 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,39 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + schedule: + - cron: "0 17 * * 2" # Tuesdays at 12pm EST (17:00 UTC); 1pm during EDT + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv sync --group dev + # `uv run` uses the ruff pinned in uv.lock, so CI lints with the same version + # developers have locally. --output-format github annotates the PR diff inline. + # Paths come from [tool.ruff] in pyproject.toml rather than being repeated here. + - run: uv run ruff check --output-format github + - run: uv run ruff format --check + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv sync --group dev + - run: uv run pytest -v -m "not integration" + + integration-test: + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv sync --group dev + - run: uv run pytest -v -m "integration and not slow" diff --git a/.gitignore b/.gitignore index b7faf40..62355d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# Ignore data files. +/data + +# Node dependencies, wherever they are installed (e.g. web/node_modules). +# Deliberately not /web, so frontend source under it is still tracked. +node_modules/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[codz] @@ -14,8 +21,9 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ +# Python distribution lib directories (not web/src/lib/) +/lib/ +/lib64/ parts/ sdist/ var/ @@ -135,7 +143,11 @@ celerybeat.pid *.sage.py # Environments +# .env.* as well as .env: a .env.backup or .env.local holding the Translator-specific +# releases URL is exactly what must never be committed, and a blanket `git add` would +# take it. env.default does not match this pattern and stays tracked. .env +.env.* .envrc .venv env/ @@ -173,7 +185,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Abstra # Abstra is an AI-powered process automation framework. diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..2c07333 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..77fe73a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog + +All notable changes to babel-explorer are documented here. This project follows +[semantic versioning](https://semver.org/). + +## 0.1.0 — 2026-09-01 + +First release. A Click CLI for querying Babel intermediate files through DuckDB, with optional +label enrichment from NodeNorm. + +### Added + +- `xrefs` — cross-references for one or more CURIEs, with `--recurse` for transitive expansion + (a single `WITH RECURSIVE` DuckDB query), `--paths` for the shortest paths connecting the given + CURIEs, and `--labels` for NodeNorm labels and Biolink types. +- `ids` — identifier records from `Identifiers.parquet`, with `--labels`. +- `test-concord` — compare a proposed concordance change against NodeNorm's current cliques. +- `--format json|tsv|csv` on `xrefs` and `ids` for machine-readable output. +- `BabelDownloader`: streaming downloads with ETag-based freshness checking, resumable retries, + and a cache that holds one Babel release at a time and refreshes itself when that release + changes. +- Configuration from `.env` or the environment — `BABEL_RELEASES_URL`, `BABEL_VERSION`, + `BABEL_LOCAL_DIR`, `BABEL_CHECK_DOWNLOAD`, `NODENORM_URL`, `BABEL_ALLOW_VERSION_MISMATCH` — + with `env.default` as the committed template. Precedence: flag > environment > `.env` > default. +- A version-skew check that refuses to mix labels from one Babel release with cross-references + from another, overridable with `--allow-version-mismatch`. + +### Known limitations + +- **The shipped defaults cannot query data yet.** Public Babel releases do not publish + `duckdb/Concord.parquet` or `duckdb/Identifiers.parquet`. Translator team members can set + `BABEL_RELEASES_URL` to an internal releases URL; everyone else gets a clear error rather than + results. Tracked in [#16](https://github.com/TranslatorSRI/babel-explorer/issues/16). +- **`--labels` fails against the public defaults.** NodeNorm dev reports Babel `2025sep1` while + public `latest` is `2025dec11`, so the skew check fires. Pin `BABEL_VERSION` to the release + NodeNorm was built from, or pass `--allow-version-mismatch`. Tracked in + [#17](https://github.com/TranslatorSRI/babel-explorer/issues/17). +- The integration tests skip without a Babel release publishing the Parquet files, so a default + run exercises only the unit suite. Tracked in + [#18](https://github.com/TranslatorSRI/babel-explorer/issues/18). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..3fedae1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,354 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +babel-explorer is a tool for querying and exploring Babel intermediate files. It allows users to discover why two biological/chemical identifiers are considered identical by the Babel system, which handles cross-references between different ontology and database identifiers (e.g., MONDO, HP, UMLS, HGNC). + +## Development Setup + +This project uses **uv** for package management: + +```bash +# Install dependencies +uv sync + +# Install with dev dependencies +uv sync --group dev + +# Configure the Babel and NodeNorm endpoints +cp env.default .env + +# Run the CLI +uv run babel-explorer --help +``` + +## Configuration + +`BABEL_RELEASES_URL`, `BABEL_VERSION`, `BABEL_LOCAL_DIR`, `BABEL_CHECK_DOWNLOAD`, `NODENORM_URL`, and +`BABEL_ALLOW_VERSION_MISMATCH` are read from `.env` (via `python-dotenv`, loaded in the `cli()` +group) or the environment. Each is also a command-line option, and precedence runs +**flag > environment variable > `.env` > built-in default**. + +The release actually queried — the **effective Babel URL** — is +`BABEL_RELEASES_URL.rstrip("/") + "/" + BABEL_VERSION + "/"`, composed by `resolve_babel_url()` +(`cli.py`) on top of the pure `compose_babel_url()` (`core/downloader.py`). `compose_babel_url` +lives in the downloader rather than the CLI because `tests/constants.py` needs the same +composition and must not import Click to get it. + +`--babel-url` overrides the composed pair with a complete URL, for a tree that does not follow the +releases-directory layout. It has **no `envvar=`, on purpose**: two variables already feed the +composed URL, and a third that silently outranked both would make "which release am I querying?" +unanswerable from the environment alone. Do not add one. `BABEL_URL` was the single pre-refactor +setting and is now inert; `cli()` warns if it is still set so it does not fail silently. + +`env.default` ships with the **public** Babel URL only. Public Babel releases do not currently +publish the DuckDB Parquet files this tool needs, so Translator team members must contact the +Babel developers for the Translator-specific releases URL and set `BABEL_RELEASES_URL` to it. +Never commit that URL to this repository. + +## Babel versions + +The Babel version behind the effective Babel URL is resolved by `resolve_babel_version()` +(`core/downloader.py`), which reads `VERSION.txt` (`Babel 2026jul22`) and falls back to the final +path segment for older trees that predate it. `latest/` resolves to whatever release it currently +points at. That fallback segment is now exactly `BABEL_VERSION`, so a pinned release still +resolves when `VERSION.txt` is unreachable, while `latest` yields `None` as before. + +The `.babel-version` marker records the release the server *resolved* to, not the one requested, so +`BABEL_VERSION=latest` and `BABEL_VERSION=2025dec11` share a cache while they name the same +release. That is deliberate — do not "fix" it into a spurious refresh. + +`BABEL_LOCAL_DIR` holds **one Babel release at a time**, recorded in a `.babel-version` marker. +When the release changes, `BabelDownloader.sync_cache_version()` clears `last_checked` from the +`.meta` sidecars in `/duckdb/` — never the Parquet files — so the existing ETag path +re-checks each cached file and re-downloads only what actually changed. The stored ETag is kept +deliberately: deleting the sidecar outright skips the HEAD and forces an unconditional +multi-gigabyte re-download. Partial `.tmp` downloads *are* deleted, so no prefix from the previous +release survives into the next one. This keeps `Concord.parquet` and `Identifiers.parquet` from +being read together across two different Babel releases. + +`sync_cache_version()` does **not** write the new release to `.babel-version` itself. The marker +claims "the local cache holds this release", which is only true once every cached file has been +re-validated against it, so it is written by `_write_version_marker_if_synced()` after a download +instead — once no `.meta` sidecar in `duckdb/` is still missing its `last_checked`. Stamping it up +front would leave a run interrupted between `Concord.parquet` and `Identifiers.parquet` with a +marker naming the new release over a half-old cache, and the next run would see a marker that +matches and skip the refresh entirely. A cached file nobody asks for holds the marker back +indefinitely, costing one HEAD per run; that is correct, not a bug — the file really is still from +the previous release. + +`--check-download never` (`freshness_seconds=inf`) suppresses re-checks *within* a release, not +across one. `_is_within_freshness()` tests for a missing `last_checked` **before** the `inf` +shortcut, so a sidecar the version change expired is never fresh. Reordering those two lines +re-opens the whole hole: `never` would hand back the previous release's Parquet with no network +call at all. + +A `.tmp` is deleted in two places, on purpose. The delete in `get_downloaded_file()` is the safety +guarantee (see [Partial downloads](#partial-downloads)); the sweep in `sync_cache_version()` is +housekeeping that reclaims gigabytes belonging to a release nobody will ask for again, including +for files that are never re-downloaded and so never reach `get_downloaded_file()`. Dropping the +sweep only wastes disk; dropping the other reintroduces silent Parquet corruption. + +If a HEAD request fails, `_remote_unchanged()` returns `None` — "could not check", distinct from +`True`/"confirmed unchanged". The cached file is still used, but `last_checked` is deliberately +**not** refreshed, so the next run checks again. Restamping it there would let one flaky HEAD pin +the previous release's Parquet as freshly validated for the whole freshness window, immediately +after `sync_cache_version()` cleared `last_checked` for a new release. + +`BabelExplorerGroup.invoke()` turns `requests.RequestException` into a `ClickException`, so an +unreachable NodeNorm reports an error rather than a traceback. In practice only NodeNorm reaches +it: the downloader handles its own network failures, while NodeNorm deliberately lets HTTP errors +propagate so a failed lookup is not cached. `get_babel_version()` swallows its own errors, so an +unreachable NodeNorm passes the version check below and only fails part-way through the query. + +`xrefs` fails when NodeNorm's `status` endpoint reports a different `babel_version` than the Babel +being queried, since labels and cliques would not match the cross-references. Pass +`--allow-version-mismatch` to proceed anyway. The check is skipped when NodeNorm is not consulted +(`xrefs` without `--labels`, including `--recurse`, which is served entirely by DuckDB; and `ids` +without `--labels`) or when either version is unavailable. + +## Partial downloads + +Downloads land in a sibling `.tmp` file and are promoted with `os.replace`. Three rules keep a +`.tmp` from becoming a corrupt Parquet that then passes every freshness check — a failure that is +permanent, because the file gets stamped with the *correct* ETag: + +- **A `.tmp` is never resumed across runs.** `get_downloaded_file()` deletes any it finds before + starting, and cleans up on `BaseException` so a Ctrl-C leaves nothing behind. Resume is by byte + offset, the only way to reach the download at all is that the remote bytes *changed*, and an + orphaned `.tmp` carries no record of which version its bytes came from. Restarting costs a + re-download; splicing costs silent data corruption. Do not "optimise" this back into a + cross-run resume without persisting the validator alongside the `.tmp`. +- **A resume requires a validator, and sends it as `If-Range`.** The validator is the ETag (else + Last-Modified) of the response the bytes on disk were written from, so a file rebuilt + mid-download restarts (HTTP 200) instead of splicing. A server that supplies neither leaves + nothing to make the resume conditional on, so `_download_with_retry()` discards the partial file + and restarts from zero rather than sending a bare `Range`. Do not relax that into "send `Range`, + add `If-Range` when we happen to have one": the case with no validator is exactly the one where + a splice cannot be detected afterwards. +- **Sizes are checked, twice.** A stream that ends short of `Content-Length` raises + `IncompleteDownloadError` and is retried, rather than being promoted as complete; and an HTTP + 416 is only treated as "already complete" once the local size matches the remote + `Content-Length`, since 416 also means the remote file *shrank* below the resume offset. A HEAD + that reports no `Content-Length` at all is "cannot confirm", not "complete", and restarts too — + which cannot loop, because the retry has nothing on disk and so sends no `Range`. + +`_save_meta()` records the length of the whole file, taken from `Content-Range` rather than a 206 +response's `Content-Length` (which is only the range's length). Storing the partial length would +make the Last-Modified fallback in `_remote_unchanged()` compare it against the full remote length +forever, re-downloading an unchanged multi-gigabyte file on every freshness expiry. + +## Commands + +### Running the Application + +```bash +# Get cross-references for one or more CURIEs +uv run babel-explorer xrefs MONDO:0004979 + +# Get cross-references with expansion (recursive lookup) +uv run babel-explorer xrefs MONDO:0004979 --recurse + +# Get cross-references with labels from NodeNorm +uv run babel-explorer xrefs MONDO:0004979 --labels + +# Get ID records for CURIEs +uv run babel-explorer ids MONDO:0004979 + +# Get ID records with labels from NodeNorm +uv run babel-explorer ids MONDO:0004979 --labels + +# Test concordance changes with NodeNorm +uv run babel-explorer test-concord MONDO:0004979 HP:0000001 + +# Use a custom Babel server or local directory (overrides .env) +uv run babel-explorer xrefs MONDO:0004979 --local-dir data --babel-version 2025dec11 + +# Override the composed URL entirely (command line only; there is no BABEL_URL env var) +uv run babel-explorer xrefs MONDO:0004979 --babel-url https://stars.renci.org/var/babel/latest/ +``` + +### Development Commands + +```bash +# Run all tests (includes large file downloads) +uv run pytest -v + +# Run unit tests only (fast, no network) +uv run pytest -v -m "not integration" + +# Run integration tests without the Identifiers.parquet download +uv run pytest -v -m "integration and not slow" + +# Run a single test file +uv run pytest -v tests/test_nodenorm.py +``` + +### Linting + +**Run both of these before committing or pushing.** CI checks them on every PR, and a push +that skips them turns the PR red for reasons unrelated to the change under review. + +```bash +uv run ruff check # Python lint +uv run ruff check --fix # Python auto-fix +uv run ruff format --check # Python format check +uv run ruff format # Python auto-format +``` + +Run them over the whole repository, not just the files you touched — `[tool.ruff]` in +`pyproject.toml` sets the scope. If `ruff format` reports files you did not edit, the repository +had drifted; commit that reformatting separately from your change so review stays readable, and +do not silently revert it. + +Rules are `E`, `F`, `I` (import sorting) and `UP` (pyupgrade), with `E501` left to the formatter. +Line length is ruff's default of 88. `*.md` is excluded, because ruff 0.16+ reformats Python +inside Markdown code blocks and this repository's snippets are illustrative fragments. + +## Console Output Format Conventions + +### Label display + +When a human-readable label is shown alongside a CURIE in console output, it always appears **immediately after the CURIE, in double quotes**: + +``` +MONDO:0004979 "asthma" skos:exactMatch EFO:0000270 "asthma" +``` + +This applies everywhere labels appear: `xrefs --labels`, `xrefs --paths --labels`, `ids --labels`, and `test-concord`. + +`--paths` is console-only; combining it with `--format json`/`tsv`/`csv` is rejected up front rather +than silently emitting the full cross-reference list. It also needs at least two CURIEs, and that +is checked in the same place, before `make_downloader()` — `--paths` implies `--recurse`, so +finding out inside `_print_paths()` would cost a multi-gigabyte download and a full recursive query +before rejecting the run. + +**When a label is absent, omit it entirely** — do not substitute a placeholder like `-` or `""`. A CURIE with no label renders as just the bare CURIE. + +**Escaping:** embedded backslashes are escaped as `\\` and embedded double quotes as `\"`. Downstream tools can parse labels with the regex `"([^"\\]|\\.)*"`. + +**Do not** use parentheses `(label)` or any other delimiter — double quotes are the sole convention. + +## Architecture + +### Core Components + +1. **BabelDownloader** (`src/babel_explorer/core/downloader.py`): + - Downloads Babel intermediate files from a remote HTTP(S) server using Python's `requests` library (streaming downloads) + - Caches files locally in a configurable directory (default: `data/`), one Babel release at a time + - Caching is on disk, keyed by ETag and the `.meta` sidecars — not in memory. Only + `babel_version` is memoised, via `functools.cached_property` + - Resolves the Babel version (`resolve_babel_version`) and refreshes the cache when it changes (`sync_cache_version`) + - Raises `MissingBabelFileError` on a 404 for a `duckdb/` file, since public releases do not publish them + - **Important**: Requires network access but no external tools like `wget` + +2. **BabelXRefs** (`src/babel_explorer/core/babel_xrefs.py`): + - Main query engine for cross-references + - Uses DuckDB to query Parquet files (`Concord.parquet`, `Identifiers.parquet`) + - Supports recursive expansion of cross-references via a single `WITH RECURSIVE` query + - Uses ephemeral in-memory DuckDB connections, opened by `BabelXRefs._connect()`. No + database is persisted, but larger-than-memory queries spill to disk: `_connect()` sets + `temp_directory` to `/duckdb-spill/`, because DuckDB's default is + `.tmp` in the *current working directory* and a `--recurse` run materialises the whole + multi-gigabyte Concord relation + +3. **NodeNorm** (`src/babel_explorer/core/nodenorm.py`): + - Integration with NodeNormalization API (https://nodenormalization-sri.renci.org/) + - Fetches labels, biolink types, and equivalent identifiers for CURIEs + - Caches normalisation results, identifiers and cliques in per-instance dicts; a new + `NodeNorm` object is the way to get uncached results + - `get_babel_version()` reads the `status` endpoint to report which Babel release it was built from + - Optional component for label enrichment + +4. **CLI** (`src/babel_explorer/cli.py`): + - Click-based command-line interface + - Three main commands: `xrefs`, `ids`, `test-concord` + +### Data Flow + +1. User provides CURIEs via CLI; `BABEL_RELEASES_URL` + `BABEL_VERSION` / `NODENORM_URL` come from `.env` or the environment, and are composed into the effective Babel URL +2. BabelDownloader resolves the Babel version, refreshes the cache if it changed, and ensures required Parquet files are downloaded +3. BabelXRefs queries files using DuckDB +4. If `--labels` is set, NodeNorm is queried for additional metadata (`--recurse` alone does not consult NodeNorm — the recursive expansion is a single DuckDB query) +5. Results are printed to stdout + +### Key Design Patterns + +- **Lazy downloading**: Files are only downloaded when first accessed +- **Caching**: Downloads are cached on disk (ETag + `.meta` sidecar); NodeNorm results are cached + in per-instance dicts. Neither uses `functools.lru_cache` +- **Recursive expansion**: The `--recurse` flag recursively follows all cross-references to build complete graphs +- **DuckDB for querying**: In-memory SQL queries against Parquet files for fast lookups, spilling + to `/duckdb-spill/` rather than the working directory + +## Testing + +### Test Structure + +Tests live in `tests/` and are split into fast **unit tests** (mocked, no network) and slower **integration tests** (real downloads and API calls). Pytest markers control which tests run: + +- **`@pytest.mark.integration`** — requires network access (downloads Parquet files or calls NodeNorm API) +- **`@pytest.mark.slow`** — downloads `Identifiers.parquet`, the largest file Babel publishes + +Note that `not slow` is *not* the same as "small". `Concord.parquet` is itself multi-gigabyte in +current releases (4.6 GB in `2026jul22`) and its tests are not marked slow, because excluding them +would leave the non-slow integration set covering nothing that touches real data. Budget for that +before pointing CI at a Babel that publishes the Parquet files (see issue #18). + +Do not record per-file test counts here — they drift silently and then mislead. Get them on demand: + +```bash +uv run pytest --collect-only -q -m "not integration" # unit test count +uv run pytest --collect-only -q # full count +``` + +**Integration tests skip when the composed Babel URL points at a release that does not publish +`duckdb/Concord.parquet`**, which is the case for every public release right now. A run reporting +a couple of dozen skips is the expected result without a Translator `BABEL_RELEASES_URL` in `.env`, not a +broken test environment. + +### Test Infrastructure + +- **`tests/conftest.py`** — Session-scoped fixtures that download Parquet files once and share them across all integration tests. The `shared_downloader` fixture HEADs `duckdb/Concord.parquet` first and skips the session on 404. Teardown removes the `data/test/` directory so the next run starts fresh. +- **`tests/constants.py`** — Shared constants (URLs, file paths) and `load_curies()` helper. +- **`tests/data/valid_curies.txt`** — One CURIE per line (`#` comments allowed). Integration tests are parametrized over this list — adding a new line automatically expands test coverage. + +### Key Dataclasses + +- **`Identifier`** — Frozen dataclass for a normalized NodeNorm entry (curie, label, biolink_type, taxa, description). Returned by `NodeNorm.get_identifier()` and `get_clique_identifiers()`. +- **`CrossReference`** — Frozen dataclass for Concord.parquet rows (filename, subj, pred, obj) +- **`LabeledCrossReference`** — Extends CrossReference with labels and biolink types from NodeNorm +- **`IdentifierRecord`** — Frozen dataclass for Identifiers.parquet rows (curie + dynamic extra fields, plus `nodenorm_label` under `--labels`). Returned by `BabelXRefs.get_curie_ids()`. The NodeNorm label is *not* called `label`: Identifiers.parquet has its own `label` column, which lands in `extra_fields` and would collide with it once the record is flattened for json/tsv/csv. + +## Repository history was rewritten on 2026-09-01 + +Every commit was rewritten to remove an internal Babel URL that had been the hardcoded default +since the initial commit. Consequences a future contributor will trip over: + +- **A clone taken before that date has divergent history.** Every SHA changed except `gh-pages`. + Re-clone; do not try to merge or rebase the old history back together. +- **PRs #1, #4, #6, #7 and #11 are dead.** GitHub refuses to reopen a PR whose original head + commits no longer exist, so they were recreated as #20-#24. Old PR links and commit SHAs in + issue comments point at nothing. +- **`.env.*` is gitignored, `env.default` is not.** The URL leaked in the first place because it + was a default in source rather than configuration. `TestCommittedConfigTemplate` + (`tests/test_cli.py`) now fails if a non-public host appears in `env.default`; that test is the + enforcement, so do not weaken it to accommodate a convenient default. + +## Important Notes + +- **Data directory**: The `data/` directory is gitignored and contains downloaded Parquet files and generated DuckDB databases +- **Babel versions**: The Babel release comes from `BABEL_RELEASES_URL` + `BABEL_VERSION`, or from `--babel-url` when given; see [Babel versions](#babel-versions) above +- **`.env`**: gitignored. Only `env.default` is committed, and it must never contain the Translator-specific Babel URL + +## File Locations + +- Source code: `src/babel_explorer/` +- Tests: `tests/` +- Test CURIEs: `tests/data/valid_curies.txt` +- Downloaded Babel files: `/duckdb/*.parquet` (default `data/duckdb/`) +- DuckDB query spill: `/duckdb-spill/` (default `data/duckdb-spill/`) +- Endpoint configuration: `.env` (gitignored), template in `env.default` +- Entry point: `src/babel_explorer/cli.py` diff --git a/FUTURE.md b/FUTURE.md new file mode 100644 index 0000000..9b22363 --- /dev/null +++ b/FUTURE.md @@ -0,0 +1,5 @@ +# Future Work + +These items are tracked as GitHub issues: + +- [#13](https://github.com/TranslatorSRI/babel-explorer/issues/13) — Reuse a single DuckDB connection per `BabelXRefs` instance diff --git a/README.md b/README.md index 077ce44..d769b8a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,115 @@ # Babel Explorer -Software for querying and exporting Babel intermediate files +Software for querying and exploring Babel intermediate files. + +babel-explorer allows you to discover why two biological/chemical identifiers are considered identical by the [Babel](https://github.com/TranslatorSRI/Babel) system, which handles cross-references between different ontology and database identifiers (e.g., MONDO, HP, UMLS, HGNC). + +## Setup + +This project uses [uv](https://docs.astral.sh/uv/) for package management: + +```bash +uv sync --group dev +cp env.default .env +``` + +## Configuration + +`.env` holds the endpoints babel-explorer talks to: + +| Variable | Default | Purpose | +|---|---|---| +| `BABEL_RELEASES_URL` | `https://stars.renci.org/var/babel/` | Directory holding one subdirectory per Babel release | +| `BABEL_VERSION` | `latest` | Which release subdirectory to query | +| `BABEL_LOCAL_DIR` | `data` | Where downloaded Babel files are cached | +| `BABEL_CHECK_DOWNLOAD` | `3h` | How often to re-check downloads | +| `NODENORM_URL` | `https://nodenormalization-sri.renci.org/` | NodeNorm instance for labels and cliques | + +Each has a matching command-line option, and precedence runs **flag > environment variable > +`.env` > default**. The release actually queried — the *effective Babel URL* — is +`BABEL_RELEASES_URL` + `BABEL_VERSION` + `/`. + +`--babel-url` is the one exception. It takes a complete URL and overrides the composed pair, for a +tree that does not follow the releases-directory layout. It is **command-line only**: there is no +`BABEL_URL` environment variable, so the environment can never disagree with itself about which +release is in effect. + +> **Public releases cannot serve data yet.** Public Babel releases do not currently publish the +> DuckDB Parquet files (`duckdb/Concord.parquet`, `duckdb/Identifiers.parquet`) that babel-explorer +> needs, so the shipped defaults will report that the files are missing. Translator team members +> should contact the Babel developers for the Translator-specific releases URL and set +> `BABEL_RELEASES_URL` to it in their `.env`, or pass `--babel-url ` for a single +> run. Tracked in [#16](https://github.com/TranslatorSRI/babel-explorer/issues/16). + +### Babel versions + +`BABEL_LOCAL_DIR` holds one Babel release at a time. When the effective Babel URL starts pointing +at a different release, babel-explorer notices and re-downloads the files that changed — you do not +need to clear the cache by hand. The cache marker records the release the server *resolved* to, so +`BABEL_VERSION=latest` and `BABEL_VERSION=2025dec11` share a cache while they name the same +release. + +`xrefs` refuses to run when NodeNorm was built from a different Babel release than the one being +queried, since the labels and cliques would not match the cross-references. Either pin +`BABEL_VERSION` to the release NodeNorm reports, point `--nodenorm-url` at a matching NodeNorm, or +pass `--allow-version-mismatch` to override. + +## Usage + +```bash +# Get cross-references for one or more CURIEs +uv run babel-explorer xrefs MONDO:0004979 + +# Get cross-references with expansion (recursive lookup) +uv run babel-explorer xrefs MONDO:0004979 --recurse + +# Get cross-references with labels from NodeNorm +uv run babel-explorer xrefs MONDO:0004979 --labels +# Labels appear in double quotes immediately after the CURIE: +# MONDO:0004979 "asthma" skos:exactMatch EFO:0000270 "asthma" + +# Get ID records for CURIEs +uv run babel-explorer ids MONDO:0004979 + +# Get ID records with labels from NodeNorm +uv run babel-explorer ids MONDO:0004979 --labels + +# Test concordance changes with NodeNorm +uv run babel-explorer test-concord MONDO:0004979 HP:0000001 +``` + +## Testing + +Tests are split into fast **unit tests** (mocked, no network) and slower **integration tests** (real file downloads and API calls), controlled by pytest markers. + +Integration tests run against whatever `BABEL_RELEASES_URL` and `BABEL_VERSION` compose to, and +skip when that release does not publish the DuckDB Parquet files. + +```bash +# Unit tests only — fast, no network required +uv run pytest -v -m "not integration" + +# Integration tests without the Identifiers.parquet download +uv run pytest -v -m "integration and not slow" + +# Full suite including large file downloads +uv run pytest -v +``` + +## Linting + +Run both checks before committing; CI enforces them on every pull request: + +```bash +uv run ruff check --fix # lint, with auto-fix +uv run ruff format # format +``` + +### Adding Test CURIEs + +Integration tests are parametrized over the CURIEs listed in `tests/data/valid_curies.txt`. Add a new CURIE on its own line to automatically expand test coverage: + +``` +# tests/data/valid_curies.txt +MONDO:0004979 +HP:0000001 +``` diff --git a/env.default b/env.default new file mode 100644 index 0000000..b60e934 --- /dev/null +++ b/env.default @@ -0,0 +1,39 @@ +# Copy this file to .env and edit as needed. Every value here can also be set as an +# environment variable, or overridden per-run by the matching command-line option. +# Precedence: command-line flag > environment variable > .env > built-in default. + +# Directory holding one subdirectory per Babel release. The release actually queried is +# BABEL_RELEASES_URL + BABEL_VERSION + "/". +BABEL_RELEASES_URL=https://stars.renci.org/var/babel/ + +# Which release to use: a subdirectory name such as 2025dec11, or "latest" to follow +# whatever the server currently publishes. "latest" is resolved through VERSION.txt, so +# when the release behind it changes, babel-explorer refreshes the cached files +# automatically. +BABEL_VERSION=latest + +# There is deliberately no BABEL_URL variable. To query a complete URL that does not fit +# the releases-directory + version layout, pass --babel-url on the command line; it +# overrides both settings above for that run. + +# Where downloaded Babel files are cached. This holds one Babel release at a time. +# Point it somewhere per-release if you need to keep several around. +BABEL_LOCAL_DIR=data + +# How often to re-check downloads: e.g. 3h, 30m, 1d, 0, never. +# "never" always uses cached files; "0" forces a re-check every time. +BABEL_CHECK_DOWNLOAD=3h + +NODENORM_URL=https://nodenormalization-sri.renci.org/ + +# Proceed even when NodeNorm reports a different Babel release than the one being +# queried. Off by default: labels and cliques from one release do not match another's +# cross-references, so the results would be wrong without looking wrong. Pinning +# BABEL_VERSION to the release NodeNorm was built from is the real fix. +BABEL_ALLOW_VERSION_MISMATCH=false + +# NOTE: public Babel releases do not currently publish the DuckDB Parquet files +# (duckdb/Concord.parquet, duckdb/Identifiers.parquet) that babel-explorer needs, so the +# defaults above will report that the files are missing. Translator team members should +# contact the Babel developers for the Translator-specific releases URL and set +# BABEL_RELEASES_URL to it here. Never commit that URL to this repository. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5d12da4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[project] +name = "babel-explorer" +version = "0.1.0" +description = "Tool for querying and exploring Babel APIs and intermediate files" +readme = "README.md" +license = "MIT" +license-files = ["LICENSE"] +authors = [{ name = "Gaurav Vaidya", email = "gaurav@ggvaidya.com" }] +requires-python = ">=3.11" +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] +dependencies = [ + "click>=8.3.1", + "duckdb>=1.4.2", + "python-dotenv>=1.0", + "requests>=2.32.5", + "rich>=13", + "tqdm>=4.67.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "filelock>=3.16", + "pytest>=8.3.5", + "pytest-xdist[psutil]>=3.6", + "ruff>=0.11.0", +] + +[project.scripts] +babel-explorer = "babel_explorer.cli:cli" + +[tool.ruff] +# ruff 0.16 began formatting Python inside Markdown code blocks. Our snippets are +# illustrative fragments rather than runnable modules, so keep ruff out of them. +# Line length is left at ruff's default of 88. +extend-exclude = ["*.md"] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort (import sorting) + "UP", # pyupgrade +] +ignore = [ + "E501", # let the formatter handle wrapping consistently +] + +[tool.pytest.ini_options] +addopts = "-n auto" +markers = [ + "integration: tests requiring network access (deselect with '-m \"not integration\"')", + "slow: tests downloading Identifiers.parquet, the largest file Babel publishes (deselect with '-m \"not slow\"')", +] diff --git a/src/babel_explorer/__init__.py b/src/babel_explorer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/babel_explorer/cli.py b/src/babel_explorer/cli.py new file mode 100644 index 0000000..7c712c1 --- /dev/null +++ b/src/babel_explorer/cli.py @@ -0,0 +1,577 @@ +"""Command-line interface for babel-explorer.""" + +import logging +import os +import re +from itertools import combinations + +import click +import requests +from dotenv import load_dotenv +from rich.markup import escape + +from babel_explorer.core.babel_xrefs import ( + BabelXRefs, + LabeledCrossReference, + build_adjacency, + build_depth_map, + find_shortest_path, +) +from babel_explorer.core.downloader import ( + BabelDownloader, + MissingBabelFileError, + compose_babel_url, +) +from babel_explorer.core.nodenorm import NodeNorm +from babel_explorer.formatting import ( + curie_with_label, + format_identifier_record, + hl_curie, + make_console, + record_to_dict, + write_records, +) + + +def _validate_babel_version(ctx, param, value): + """Reject a --babel-version that is really a URL or a path traversal. + + Anyone with muscle memory from the old BABEL_URL variable will eventually put a + complete URL here, which would compose into nonsense with no useful diagnostic. + """ + if value is None: + return value + if "://" in value: + raise click.BadParameter( + f"{value!r} looks like a complete URL. Pass it to --babel-url instead, " + f"or give --babel-version just the release name (e.g. '2025dec11')." + ) + if ".." in value: + raise click.BadParameter(f"{value!r} may not contain '..'.") + return value + + +def babel_options(f): + """Decorator adding the Babel source options: --local-dir, --babel-releases-url, + --babel-version, --babel-url, --check-download and --allow-version-mismatch.""" + f = click.option( + "--allow-version-mismatch", + is_flag=True, + envvar="BABEL_ALLOW_VERSION_MISMATCH", + help="Proceed even if NodeNorm was built from a different Babel release than " + "the one being queried", + )(f) + f = click.option( + "--check-download", + type=str, + default="3h", + show_default=True, + envvar="BABEL_CHECK_DOWNLOAD", + help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never'). " + "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.", + )(f) + f = click.option( + "--babel-url", + type=str, + default=None, + # Deliberately NO envvar=. BABEL_RELEASES_URL + BABEL_VERSION is the only + # environment-driven path to a Babel URL, so there is never a question of which + # variable wins. This is a per-run escape hatch, not configuration. Do not add one. + help="Complete URL of one Babel release, overriding --babel-releases-url and " + "--babel-version. Command line only: there is no BABEL_URL environment variable. " + "[default: --babel-releases-url + --babel-version]", + )(f) + f = click.option( + "--babel-version", + type=str, + default="latest", + show_default=True, + show_envvar=True, + envvar="BABEL_VERSION", + callback=_validate_babel_version, + help="Babel release to use: the name of a subdirectory under --babel-releases-url " + "(e.g. '2025dec11'). 'latest' follows whatever the server currently publishes.", + )(f) + f = click.option( + "--babel-releases-url", + type=str, + default="https://stars.renci.org/var/babel/", + show_default=True, + show_envvar=True, + envvar="BABEL_RELEASES_URL", + help="URL of a directory holding one subdirectory per Babel release.", + )(f) + f = click.option( + "--local-dir", + type=str, + default="data", + show_default=True, + show_envvar=True, + envvar="BABEL_LOCAL_DIR", + help="Local location to save Babel download files to. Holds one Babel release at " + "a time; cached files are refreshed automatically when the effective Babel URL " + "points at a new one.", + )(f) + return f + + +def nodenorm_options(f): + """Decorator adding --nodenorm-url to a command.""" + return click.option( + "--nodenorm-url", + type=str, + default="https://nodenormalization-sri.renci.org/", + show_default=True, + envvar="NODENORM_URL", + help="NodeNorm base URL used for node normalization and label enrichment", + )(f) + + +def resolve_babel_url( + babel_url: str | None, babel_releases_url: str, babel_version: str +) -> str: + """The effective Babel URL: the one release this run will query. + + ``--babel-url`` is a complete URL and wins outright; otherwise the release is + composed from the releases directory and the version. ``--babel-url`` has no + matching environment variable on purpose — with two variables already feeding the + composed URL, a third that silently outranked both would make "which release am I + actually querying?" unanswerable from the environment alone. + """ + if babel_url: + # Warn only when --babel-version was actually typed. A developer with + # BABEL_VERSION permanently in .env would otherwise be warned on every + # --babel-url run, which just teaches them to ignore warnings. + ctx = click.get_current_context(silent=True) + if ctx is not None and ctx.get_parameter_source("babel_version") == ( + click.core.ParameterSource.COMMANDLINE + ): + click.echo( + f"Warning: --babel-url overrides --babel-version, so " + f"{babel_version!r} is ignored.", + err=True, + ) + return babel_url.strip().rstrip("/") + "/" + return compose_babel_url(babel_releases_url, babel_version) + + +def make_downloader( + babel_url: str | None, + babel_releases_url: str, + babel_version: str, + local_dir: str, + check_download: str, +): + """Build a BabelDownloader and point its cache at the effective Babel release. + + Composition happens here rather than at each call site so a future command cannot + take the options and forget to resolve them. + """ + downloader = BabelDownloader( + resolve_babel_url(babel_url, babel_releases_url, babel_version), + local_path=local_dir, + freshness_seconds=parse_duration(check_download), + ) + downloader.sync_cache_version() + return downloader + + +def check_babel_versions( + downloader: BabelDownloader, nodenorm: NodeNorm, allow_version_mismatch: bool +): + """Fail if NodeNorm was built from a different Babel release than the one being queried. + + Cross-release results are silently wrong rather than obviously wrong: labels and + cliques would come from one Babel while the cross-references come from another. + Skipped when either version is unavailable. + """ + # Named for the release the *server* reports, to keep it distinct from the + # babel_version parameter the commands take, which is the release the user asked for. + downloader_version = downloader.babel_version + nodenorm_version = nodenorm.get_babel_version() + if ( + downloader_version + and nodenorm_version + and downloader_version != nodenorm_version + and not allow_version_mismatch + ): + raise click.ClickException( + f"NodeNorm at {nodenorm.nodenorm_url} was built from Babel {nodenorm_version}, " + f"but {downloader.url_base} is Babel {downloader_version}. Labels and cliques would " + f"not match the cross-references. Point --nodenorm-url at a matching NodeNorm, " + f"pin --babel-version to the release NodeNorm was built from, or pass " + f"--allow-version-mismatch to proceed anyway." + ) + + +def format_option(f): + """Decorator adding --format and --json-indent options to a command.""" + f = click.option( + "--format", + "fmt", + default="console", + type=click.Choice(["console", "json", "tsv", "csv"]), + show_default=True, + help="Output format", + )(f) + f = click.option( + "--json-indent", + default=2, + show_default=True, + help="Indentation depth for JSON output", + )(f) + return f + + +#: Duration suffixes accepted by --check-download; no suffix means seconds. +_DURATION_UNITS = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} + + +def parse_duration(value: str) -> int | float: + """Parse a duration string like '3h', '30m', '1d', '7200', or 'never' → seconds.""" + lower = (value or "").strip().lower() + if lower == "never": + return float("inf") + # The pattern rejects empty, negative and non-integer values in one go, so there + # is a single wording of the error rather than one per rejected shape. + match = re.fullmatch(r"(\d+)([smhd]?)", lower) + if not match: + raise click.BadParameter( + f"Invalid duration {value!r}: expected a non-negative integer number of " + "seconds, optionally followed by 's', 'm', 'h', or 'd', or 'never'." + ) + return int(match[1]) * _DURATION_UNITS[match[2]] + + +def _depth_of(curie: str, query_set: set, depth: int | None) -> int | None: + """Depth to render a CURIE at: query CURIEs are always depth 0.""" + return 0 if curie in query_set else depth + + +def _print_paths(console, curies, xrefs_list, labels: bool) -> None: + """Print the shortest path between every pair of query CURIEs. + + The caller checks the "at least two CURIEs" requirement up front — see ``xrefs`` — + so that a run that cannot produce a path never downloads Concord.parquet to find + that out. With fewer than two, ``combinations`` simply yields no pairs. + """ + curie_list = list(curies) + query_set = set(curie_list) + # One neighbour map for every pair: rebuilding it per pair re-walks the whole + # recursive xref list C(n,2) times. + adj = build_adjacency(xrefs_list) + + for from_c, to_c in combinations(curie_list, 2): + path = find_shortest_path(from_c, to_c, xrefs_list, adj) + header_from = hl_curie(from_c, 0) + header_to = hl_curie(to_c, 0) + + if path is None: + console.print( + f"[bold]Path:[/bold] {header_from} [dim]→[/dim] {header_to}" + f" [red]no path found[/red]" + ) + console.print() + continue + + if len(path) == 0: + console.print( + f"[bold]Path:[/bold] {header_from} [dim]=[/dim] {header_to}" + f" [dim](same node)[/dim]" + ) + console.print() + continue + + # Reconstruct ordered node list from the edge sequence. + nodes = [from_c] + for edge in path: + prev = nodes[-1] + nodes.append(edge.obj if edge.subj == prev else edge.subj) + + # Header: node1 → node2 → … → nodeN. Position along the path is the depth + # from from_c, except for query CURIEs, which always render as depth 0. + node_strs = [ + hl_curie(node, _depth_of(node, query_set, i)) + for i, node in enumerate(nodes) + ] + n_steps = len(path) + step_word = "step" if n_steps == 1 else "steps" + console.print( + f"[bold]Path ({n_steps} {step_word}):[/bold] " + + " [dim]→[/dim] ".join(node_strs) + ) + + # Edge details, indented, oriented in traversal direction. + for i, edge in enumerate(path): + from_node = nodes[i] + subj_node = edge.subj if edge.subj == from_node else edge.obj + obj_node = edge.obj if edge.subj == from_node else edge.subj + + subj_label = obj_label = None + if labels and isinstance(edge, LabeledCrossReference): + if edge.subj == subj_node: + subj_label = edge.subj_label + obj_label = edge.obj_label + else: + subj_label = edge.obj_label + obj_label = edge.subj_label + + subj_str = curie_with_label( + subj_node, _depth_of(subj_node, query_set, i), subj_label + ) + obj_str = curie_with_label( + obj_node, _depth_of(obj_node, query_set, i + 1), obj_label + ) + + console.print( + f" - {subj_str} [dim]{escape(edge.pred)}[/dim] " + f"{obj_str} [dim italic]{escape(edge.filename)}[/dim italic]" + ) + + console.print() + + +class BabelExplorerGroup(click.Group): + """Group that reports service failures as a plain error rather than a traceback.""" + + def invoke(self, ctx): + try: + return super().invoke(ctx) + except MissingBabelFileError as e: + raise click.ClickException(str(e)) from e + except requests.RequestException as e: + # In practice this is always NodeNorm: the downloader handles its own + # network failures (a failed HEAD falls back to the cached file, and + # _download_with_retry re-raises as RuntimeError after its last attempt), + # while NodeNorm deliberately lets HTTP errors propagate so a failed + # lookup is not cached. Note that get_babel_version() swallows its own + # errors, so an unreachable NodeNorm passes the version check and only + # fails here, part-way through a query. + raise click.ClickException( + f"NodeNorm request failed: {e}. Check that --nodenorm-url is " + f"reachable; `xrefs` and `ids` can also be run without --labels, " + f"which does not consult NodeNorm at all." + ) from e + + +@click.group(cls=BabelExplorerGroup) +def cli(): + """babel-explorer: query and explore Babel intermediate files.""" + logging.basicConfig(level=logging.INFO) + # Runs before subcommand parameters are parsed, so .env feeds the envvar= defaults. + load_dotenv() + # BABEL_URL was the single Babel setting before BABEL_RELEASES_URL + BABEL_VERSION. + # It is now inert, and silently ignoring it would send someone to the wrong release + # with no clue why. Checked after load_dotenv() so a stale .env is caught too. + if os.environ.get("BABEL_URL"): + click.echo( + "Warning: BABEL_URL is no longer used. Set BABEL_RELEASES_URL and " + "BABEL_VERSION instead, or pass --babel-url for a single run.", + err=True, + ) + + +@cli.command("xrefs") +@click.argument("curies", type=str, required=True, nargs=-1) +@babel_options +@nodenorm_options +@click.option("--recurse", is_flag=True, help="Recursively query returned xrefs") +@click.option("--labels", is_flag=True, help="Include labels for CURIEs") +@click.option( + "--paths", + is_flag=True, + help="Show shortest path(s) connecting the given CURIEs (implies --recurse)", +) +@format_option +def xrefs( + curies: list[str], + babel_url: str | None, + babel_releases_url: str, + babel_version: str, + nodenorm_url: str, + local_dir: str, + recurse: bool, + labels: bool, + paths: bool, + check_download: str, + allow_version_mismatch: bool, + fmt: str, + json_indent: int, +): + """ + Fetches and prints the cross-references (xrefs) for the given CURIEs. + + \f + + :param curies: A list of CURIEs (Compact URI) for which cross-references need + to be retrieved. + :type curies: list[str] + :param babel_url: Complete URL of one Babel release, overriding the two below. + ``None`` unless ``--babel-url`` was passed. + :type babel_url: str | None + :param babel_releases_url: URL of a directory holding one subdirectory per release. + :type babel_releases_url: str + :param babel_version: Which release subdirectory to query, or ``latest``. + :type babel_version: str + + :return: None + """ + if paths: + # Both checks happen before anything is downloaded. --paths implies --recurse, + # so getting one of them wrong otherwise costs a multi-gigabyte Concord.parquet + # download and a full recursive query before the run is rejected. + # + # Only the console renderer knows how to lay out paths; the other formats would + # silently emit the full recursive xref list instead, which looks like a + # successful --paths run but is not one. + if fmt != "console": + raise click.UsageError( + f"--paths is only supported with --format console, not --format {fmt}. " + f"Drop --paths to emit the full cross-reference list as {fmt}." + ) + if len(curies) < 2: + raise click.UsageError( + "--paths needs at least two CURIEs to find a path between. " + "Drop --paths to list the cross-references of a single CURIE." + ) + recurse = True + + downloader = make_downloader( + babel_url, babel_releases_url, babel_version, local_dir, check_download + ) + nodenorm = NodeNorm(nodenorm_url) + # NodeNorm is only consulted for labels; --recurse is served entirely by the + # recursive DuckDB query, so its results cannot disagree with NodeNorm's release. + if labels: + check_babel_versions(downloader, nodenorm, allow_version_mismatch) + + bxref = BabelXRefs(downloader, nodenorm) + xref_list = bxref.get_curie_xrefs(curies, recurse, label_curies=labels) + + if fmt == "console": + console = make_console() + if paths: + _print_paths(console, curies, xref_list, labels) + else: + query_set = set(curies) + # Without --recurse every result is one hop from a query CURIE, so there + # is no depth to show: only the query CURIEs themselves are highlighted. + depth_map = build_depth_map(list(curies), xref_list) if recurse else {} + for xref in xref_list: + labeled = isinstance(xref, LabeledCrossReference) + subj_str = curie_with_label( + xref.subj, + _depth_of(xref.subj, query_set, depth_map.get(xref.subj)), + xref.subj_label if labeled else None, + ) + obj_str = curie_with_label( + xref.obj, + _depth_of(xref.obj, query_set, depth_map.get(xref.obj)), + xref.obj_label if labeled else None, + ) + console.print( + f"{subj_str} [dim]{escape(xref.pred)}[/dim] " + f"{obj_str} [dim italic]{escape(xref.filename)}[/dim italic]" + ) + else: + write_records(xref_list, fmt=fmt, indent=json_indent) + + +@cli.command("ids") +@click.argument("curies", type=str, required=True, nargs=-1) +@babel_options +@nodenorm_options +@click.option("--labels", is_flag=True, help="Include labels for CURIEs") +@format_option +def ids( + curies: list[str], + babel_url: str | None, + babel_releases_url: str, + babel_version: str, + nodenorm_url: str, + local_dir: str, + labels: bool, + check_download: str, + allow_version_mismatch: bool, + fmt: str, + json_indent: int, +): + """ + Fetches and prints the ID records for the given CURIEs, along with Biolink type if provided. + + \f + + :param curies: A list of CURIEs (Compact URI) for which cross-references need + to be retrieved. + :type curies: list[str] + :param babel_url: Complete URL of one Babel release, overriding the two below. + ``None`` unless ``--babel-url`` was passed. + :type babel_url: str | None + :param babel_releases_url: URL of a directory holding one subdirectory per release. + :type babel_releases_url: str + :param babel_version: Which release subdirectory to query, or ``latest``. + :type babel_version: str + + :return: None + """ + downloader = make_downloader( + babel_url, babel_releases_url, babel_version, local_dir, check_download + ) + nodenorm = NodeNorm(nodenorm_url) + # NodeNorm is only consulted for labels, so only then can its Babel release differ. + if labels: + check_babel_versions(downloader, nodenorm, allow_version_mismatch) + + bxref = BabelXRefs(downloader, nodenorm) + xrefs = bxref.get_curie_ids(curies, label_curies=labels) + + if fmt == "console": + console = make_console() + for record in xrefs: + console.print(format_identifier_record(record)) + else: + write_records(xrefs, fmt=fmt, indent=json_indent) + + +@cli.command("test-concord") +@click.argument("curies", type=str, required=True, nargs=-1) +@nodenorm_options +@format_option +def test_concord(curies, nodenorm_url, fmt, json_indent): + """For each CURIE, print the current NodeNorm clique (all equivalent identifiers, labels, and Biolink types). + + Useful for inspecting how a potential Babel concordance change would affect NodeNorm: + run before and after a Babel rebuild to see how cliques would shift. + """ + nodenorm = NodeNorm(nodenorm_url) + nodenorm.normalize_curies(curies) + + # Resolved once, before the format branch, so console and JSON report the same rows. + query_set = set(curies) + cliques = [ + (curie, ident) + for curie in curies + for ident in nodenorm.get_clique_identifiers(curie) + ] + + if fmt == "console": + console = make_console() + for curie, ident in cliques: + member = curie_with_label( + ident.curie, _depth_of(ident.curie, query_set, None), ident.label + ) + biolink = escape(", ".join(ident.biolink_type)) + console.print(f"{hl_curie(curie, 0)} {member} [dim]{biolink}[/dim]") + else: + write_records( + [ + {"query_curie": curie, **record_to_dict(ident)} + for curie, ident in cliques + ], + fmt=fmt, + indent=json_indent, + ) + + +if __name__ == "__main__": + cli() diff --git a/src/babel_explorer/core/__init__.py b/src/babel_explorer/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/babel_explorer/core/babel_xrefs.py b/src/babel_explorer/core/babel_xrefs.py new file mode 100644 index 0000000..068a063 --- /dev/null +++ b/src/babel_explorer/core/babel_xrefs.py @@ -0,0 +1,382 @@ +"""Query engine for Babel cross-reference intermediate files. + +Provides access to Concord.parquet and Identifiers.parquet via DuckDB, +allowing callers to discover why two biological/chemical identifiers are +considered identical in a Babel build. +""" + +import dataclasses +import logging +import os +from collections import deque + +import duckdb + +from babel_explorer.core.downloader import BabelDownloader +from babel_explorer.core.nodenorm import NodeNorm + + +@dataclasses.dataclass(frozen=True) +class CrossReference: + """A single cross-reference edge read from Concord.parquet.""" + + filename: str + subj: str + pred: str + obj: str + + @staticmethod + def from_tuple(row: tuple[str, str, str, str]): + """Construct from a ``(filename, subj, pred, obj)`` database row tuple.""" + return CrossReference(filename=row[0], subj=row[1], pred=row[2], obj=row[3]) + + @property + def curies(self): + """The frozenset of both CURIEs in this edge (subject and object).""" + return frozenset([self.subj, self.obj]) + + def __lt__(self, other): + return (self.filename, self.subj, self.obj, self.pred) < ( + other.filename, + other.subj, + other.obj, + other.pred, + ) + + +@dataclasses.dataclass(frozen=True) +class LabeledCrossReference(CrossReference): + """A CrossReference enriched with human-readable labels and Biolink types from NodeNorm.""" + + subj_label: str + subj_biolink_type: tuple[str, ...] + obj_label: str + obj_biolink_type: tuple[str, ...] + + +@dataclasses.dataclass(frozen=True) +class IdentifierRecord: + """A record from the Identifiers.parquet file.""" + + curie: str + extra_fields: tuple = () + # Not "label": Identifiers.parquet has its own label column, which lands in + # extra_fields and would otherwise collide with (and silently win over) this one + # once record_to_dict() flattens the record for json/tsv/csv output. + nodenorm_label: str = "" + + @staticmethod + def from_row(row: tuple, column_names: list[str]): + """Create an IdentifierRecord from a DuckDB result row and its column names.""" + curie_idx = column_names.index("curie") + extra = tuple( + (col, row[i]) for i, col in enumerate(column_names) if i != curie_idx + ) + return IdentifierRecord(curie=row[curie_idx], extra_fields=extra) + + +def build_adjacency(xrefs: list) -> dict[str, list]: + """Build the undirected neighbour map ``{curie: [(neighbor, xref), ...]}``. + + Cross-references are stored in one direction but traversed in both, so each + edge contributes an entry to both of its endpoints. Build this once and share + it: it is O(len(xrefs)) and the recursive expansion can return 10^5+ edges. + """ + adj: dict[str, list] = {} + for xref in xrefs: + adj.setdefault(xref.subj, []).append((xref.obj, xref)) + adj.setdefault(xref.obj, []).append((xref.subj, xref)) + return adj + + +def build_depth_map( + query_curies: list[str], xrefs: list, adj: dict[str, list] | None = None +) -> dict[str, int]: + """BFS from query_curies over xref edges; returns {curie: depth_from_nearest_query}. + + Pass *adj* from ``build_adjacency`` to reuse an already-built neighbour map. + """ + if adj is None: + adj = build_adjacency(xrefs) + + depths: dict[str, int] = {c: 0 for c in query_curies} + frontier = list(query_curies) + while frontier: + next_frontier = [] + for node in frontier: + for neighbor, _ in adj.get(node, []): + if neighbor not in depths: + depths[neighbor] = depths[node] + 1 + next_frontier.append(neighbor) + frontier = next_frontier + return depths + + +def find_shortest_path( + from_curie: str, to_curie: str, xrefs: list, adj: dict[str, list] | None = None +) -> list | None: + """Return the shortest list of CrossReference edges from from_curie to to_curie. + + Returns ``[]`` if from_curie == to_curie, or ``None`` if no path exists. + The returned edges may be stored in either direction; callers should check + ``edge.subj`` / ``edge.obj`` against the expected traversal direction. + + Pass *adj* from ``build_adjacency`` to reuse an already-built neighbour map; + callers resolving many pairs over one graph should always do so. + """ + if from_curie == to_curie: + return [] + + if adj is None: + adj = build_adjacency(xrefs) + + visited = {from_curie} + queue: deque = deque([(from_curie, [])]) + while queue: + current, path = queue.popleft() + for neighbor, xref in adj.get(current, []): + if neighbor == to_curie: + return path + [xref] + if neighbor not in visited: + visited.add(neighbor) + queue.append((neighbor, path + [xref])) + return None + + +class BabelXRefs: + """Query engine for Babel cross-reference and identifier Parquet files. + + Uses DuckDB for in-memory SQL queries against Concord.parquet and + Identifiers.parquet. NodeNorm is optional and only required when + ``label_curies=True`` is passed to enrichment-aware methods. + """ + + def __init__(self, downloader: BabelDownloader, nodenorm: NodeNorm = None): + """ + :param downloader: A configured ``BabelDownloader`` that provides local paths + to the required Parquet files, downloading them on first access. + :param nodenorm: Optional ``NodeNorm`` client. Required only when callers pass + ``label_curies=True``; may be ``None`` for label-free queries. + """ + self.downloader = downloader + self.nodenorm = nodenorm + self._xref_cache: dict = {} + + def _connect(self): + """Open an ephemeral in-memory DuckDB connection that spills into the cache dir. + + Nothing is persisted, but "in-memory" is not the same as "touches no disk": + DuckDB's default ``temp_directory`` is ``.tmp`` in the *current working + directory*, and the recursive expansion materialises a multi-gigabyte Concord + relation. Left at the default, a ``--recurse`` run drops gigabytes of spill files + wherever the user happened to be standing. Point it at the directory the Parquet + files already live in, which the user chose knowing it holds bulk data. + """ + spill_dir = os.path.join(self.downloader.local_path, "duckdb-spill") + os.makedirs(spill_dir, exist_ok=True) + return duckdb.connect(config={"temp_directory": spill_dir}) + + def clear_xref_cache(self) -> None: + """Discard the per-CURIE cross-reference cache. + + The cache is per instance and keyed by ``(curie, label_curies)``, so it is + normally left alone. Tests that want a query to actually hit Parquet — rather + than a result an earlier test in the same session put there — clear it first. + """ + self._xref_cache.clear() + + def _require_nodenorm(self): + if self.nodenorm is None: + raise ValueError( + "label_curies=True requires a configured NodeNorm instance (nodenorm was None)." + ) + + def get_curie_ids( + self, curies: list[str], label_curies: bool = False + ) -> list[IdentifierRecord]: + """ + Search for all identifiers in the /ids/ files for a particular CURIE. + + :param curies: A list of CURIEs to search for. + :param label_curies: If ``True``, annotate each record with its NodeNorm label. + Requires a NodeNorm instance to have been passed to ``__init__``. + :raises ValueError: If ``label_curies=True`` but no NodeNorm instance is available. + :return: A list of IdentifierRecords containing those CURIEs. + """ + if label_curies: + self._require_nodenorm() + + identifier_parquet = self.downloader.get_downloaded_file( + "duckdb/Identifiers.parquet" + ) + + # Query the Parquet files using DuckDB (in-memory; nothing is persisted). + with self._connect() as db: + result = db.execute( + "SELECT * FROM read_parquet($1) WHERE curie IN (SELECT unnest($2::VARCHAR[]))", + [identifier_parquet, list(curies)], + ) + column_names = [desc[0] for desc in result.description] + rows = result.fetchall() + + records = [IdentifierRecord.from_row(row, column_names) for row in rows] + if label_curies: + self.nodenorm.normalize_curies({r.curie for r in records}) + records = [ + dataclasses.replace( + r, nodenorm_label=self.nodenorm.get_identifier(r.curie).label + ) + for r in records + ] + return records + + def get_curie_xref(self, curie: str, label_curies: bool = False): + """Return all cross-references in Concord.parquet where *curie* is the subject or object. + + Results are cached per ``(curie, label_curies)`` pair on this instance. + + :param curie: The CURIE to look up. + :param label_curies: If ``True``, annotate each result with NodeNorm labels and + Biolink types. Requires a NodeNorm instance to have been passed to ``__init__``. + :raises ValueError: If ``label_curies=True`` but no NodeNorm instance is available. + :return: A list of ``CrossReference`` (or ``LabeledCrossReference``) objects. + """ + cache_key = (curie, label_curies) + if cache_key not in self._xref_cache: + self._query_xrefs([curie], label_curies) + return self._xref_cache[cache_key] + + def _query_xrefs(self, curies: list[str], label_curies: bool = False) -> list: + """Fetch the direct cross-references for *curies* in a single Parquet scan. + + Concord.parquet is multi-gigabyte, so one scan matching every CURIE at once + costs a fraction of one scan per CURIE. Results are bucketed back into the + per-CURIE cache that ``get_curie_xref`` reads. + """ + if label_curies: + self._require_nodenorm() + if not curies: + return [] + + concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") + + with self._connect() as db: + xref_tuples = db.execute( + """ + SELECT filename, subj, pred, obj FROM read_parquet($1) + WHERE subj IN (SELECT unnest($2::VARCHAR[])) + OR obj IN (SELECT unnest($2::VARCHAR[])) + """, + [concord_parquet, list(curies)], + ).fetchall() + + xrefs = [CrossReference.from_tuple(rec) for rec in xref_tuples] + if label_curies: + xrefs = self._to_labeled_xrefs(xrefs) + + # Bucket per query CURIE so get_curie_xref's cache stays exact: a CURIE with + # no cross-references must cache an empty list, not stay absent. + wanted = set(curies) + buckets: dict[str, list] = {curie: [] for curie in wanted} + for xref in xrefs: + for curie in xref.curies & wanted: + buckets[curie].append(xref) + for curie, bucket in buckets.items(): + self._xref_cache[(curie, label_curies)] = bucket + return xrefs + + def _to_labeled_xrefs(self, xrefs: list) -> list[LabeledCrossReference]: + """Annotate cross-references with NodeNorm labels and Biolink types. + + Every CURIE in the batch is normalised up front in a handful of requests; + the per-edge ``get_identifier`` calls below are then served from cache. + """ + self.nodenorm.normalize_curies({c for xref in xrefs for c in xref.curies}) + + labeled = [] + for xref in xrefs: + subj = self.nodenorm.get_identifier(xref.subj) + obj = self.nodenorm.get_identifier(xref.obj) + labeled.append( + LabeledCrossReference( + subj=xref.subj, + obj=xref.obj, + filename=xref.filename, + pred=xref.pred, + subj_label=subj.label, + subj_biolink_type=subj.biolink_type, + obj_label=obj.label, + obj_biolink_type=obj.biolink_type, + ) + ) + return labeled + + def _get_curie_xrefs_recursive(self, curies: list[str], label_curies: bool = False): + """Traverse the cross-reference graph in one DuckDB WITH RECURSIVE query.""" + if label_curies: + self._require_nodenorm() + if not curies: + return [] + + concord_parquet = self.downloader.get_downloaded_file("duckdb/Concord.parquet") + + with self._connect() as db: + rows = db.execute( + """ + WITH RECURSIVE + concord AS MATERIALIZED ( + SELECT filename, subj, pred, obj FROM read_parquet($1) + ), + edges(a, b) AS ( + SELECT subj, obj FROM concord + UNION ALL + SELECT obj, subj FROM concord + ), + frontier(curie) AS ( + SELECT unnest($2::VARCHAR[]) + UNION + SELECT e.b + FROM edges e + INNER JOIN frontier f ON e.a = f.curie + ) + SELECT DISTINCT c.filename, c.subj, c.pred, c.obj + FROM concord c + WHERE c.subj IN (SELECT curie FROM frontier) + OR c.obj IN (SELECT curie FROM frontier) + ORDER BY c.filename, c.subj, c.obj, c.pred + """, + [concord_parquet, curies], + ).fetchall() + + xrefs = [CrossReference.from_tuple(row) for row in rows] + + if label_curies: + xrefs = self._to_labeled_xrefs(xrefs) + + return xrefs + + def get_curie_xrefs( + self, curies: list[str], recurse: bool = False, label_curies: bool = False + ): + """ + Search for all identifiers that are cross-referenced to the given CURIE. + + :param curies: A list of CURIEs to search for. + :param recurse: Whether to expand the cross-references (i.e. recursively follow all identifiers). + :param label_curies: Whether to annotate results with labels from NodeNorm. + :return: A list of cross-references containing those CURIEs. + """ + + if recurse: + return self._get_curie_xrefs_recursive(curies, label_curies) + + logging.info(f"Searching for cross-references for {', '.join(curies)}") + + # One Parquet scan covers every CURIE not already cached; the scan returns the + # union of their cross-references, so only cached CURIEs need adding separately. + uncached = [c for c in curies if (c, label_curies) not in self._xref_cache] + xrefs = set(self._query_xrefs(uncached, label_curies)) + for curie in set(curies) - set(uncached): + xrefs.update(self._xref_cache[(curie, label_curies)]) + + return sorted(xrefs) diff --git a/src/babel_explorer/core/downloader.py b/src/babel_explorer/core/downloader.py new file mode 100644 index 0000000..0897852 --- /dev/null +++ b/src/babel_explorer/core/downloader.py @@ -0,0 +1,693 @@ +"""HTTP downloader for Babel Parquet files with ETag-based freshness checking.""" + +import functools +import glob +import json +import logging +import os +import re +import tempfile +import time +from datetime import UTC, datetime + +import requests +from tqdm import tqdm + +#: Name of the file recording which Babel release the local cache holds. +VERSION_MARKER = ".babel-version" + + +class MissingBabelFileError(RuntimeError): + """Raised when a Babel release does not publish a file this tool needs.""" + + +class IncompleteDownloadError(RuntimeError): + """Raised when a download ends before the whole advertised file arrived. + + A stream that stops early without raising (a proxy or CDN closing the + connection cleanly, say) would otherwise be promoted to the final path and + stamped with the correct ETag, leaving a truncated Parquet that passes every + later freshness check. Raising instead lets the retry loop resume it. + """ + + +def compose_babel_url(releases_url: str, version: str) -> str: + """Join a releases-directory URL and a release name into one Babel base URL. + + Normalised the way ``BabelDownloader`` wants it — exactly one trailing slash — so + callers can join relative paths straight onto the result. A releases URL with no + trailing slash would otherwise compose ``.../babellatest/`` and 404 everything. + + Lives here rather than in ``cli.py`` because ``tests/constants.py`` needs the same + composition and should not import Click to get it. + """ + return releases_url.strip().rstrip("/") + "/" + version.strip().strip("/") + "/" + + +def resolve_babel_version(url_base: str, timeout: int = 30) -> str | None: + """ + Resolve the Babel version behind a Babel base URL. + + Reads ``VERSION.txt`` (present on all full Babel releases, e.g. ``Babel 2026jul22``), + falling back to the final path segment for older trees that predate it. Under the + ``BABEL_RELEASES_URL`` + ``BABEL_VERSION`` scheme that final segment *is* the + requested version, so a pinned release survives a missing ``VERSION.txt`` while + ``latest`` still resolves to ``None``. + + :return: The version string, or ``None`` if it cannot be determined. + """ + try: + response = requests.get(url_base + "VERSION.txt", timeout=timeout) + response.raise_for_status() + match = re.search(r"Babel\s+(\S+)", response.text) + if match: + return match.group(1) + except requests.RequestException: + pass + + # Legacy trees (e.g. the 2025nov19 development directory) have no VERSION.txt. + segment = url_base.rstrip("/").rsplit("/", 1)[-1] + return None if segment == "latest" else segment + + +class BabelDownloader: + """ + Class for downloading Babel cross-reference files to a local directory as needed. + """ + + def __init__( + self, + url_base, + local_path=None, + retries=10, + freshness_seconds=3 * 3600, + timeout: int = 30, + ): + """ + :param url_base: Base URL of the Babel server (must end with ``/``). + :param local_path: Directory for cached downloads. Defaults to + ``tempfile.gettempdir()`` if ``None``; created automatically if it + does not exist. + :param retries: Maximum number of download retry attempts on failure. + :param freshness_seconds: How long a local file is considered fresh without + re-checking the server. Use ``float('inf')`` to never re-check, or ``0`` + to always issue a HEAD request. Defaults to 3 hours. + :param timeout: HTTP request timeout in seconds. + """ + if not url_base.endswith("/"): + url_base += "/" + self.url_base = url_base + self.retries = retries + self.freshness_seconds = freshness_seconds + self.timeout = timeout + self.logger = logging.getLogger(BabelDownloader.__name__) + # Release whose marker is waiting for the cache to catch up with it; see + # _write_version_marker_if_synced. + self._pending_version: str | None = None + + if local_path is None: + local_path = tempfile.gettempdir() + + if not os.path.exists(local_path): + os.makedirs(local_path, exist_ok=True) + self.local_path = local_path + elif os.path.isdir(local_path): + self.local_path = local_path + else: + raise ValueError( + f"Invalid local_path (must be an existing directory): '{local_path}'" + ) + + @functools.cached_property + def babel_version(self) -> str | None: + """The Babel release behind ``url_base``, resolved once and cached. + + ``cached_property`` caches a ``None`` result too, so a tree without a + readable version is not re-fetched on every access. + """ + return resolve_babel_version(self.url_base, self.timeout) + + def sync_cache_version(self): + """ + Point the local cache at the Babel release behind ``url_base``. + + The cache holds one release at a time. When the release changes, ``last_checked`` + is cleared from every ``.meta`` sidecar so the existing ETag path re-checks each + cached file on next use — whatever actually changed is re-downloaded, and files + that are unchanged cost one HEAD instead of a fresh multi-gigabyte download. The + ETag itself is deliberately kept: deleting the sidecar outright would skip the + HEAD entirely and force an unconditional re-download of every file. + + Editing the sidecars rather than the Parquet files also means nothing large is + destroyed if the version cannot be trusted, and an interrupted refresh self-heals: + a ``.meta`` file is only written after a successful download. + + Partial ``.tmp`` downloads are removed, because the bytes already on disk belong + to the previous release. This is deliberately the *second* of two deletes, and the + redundancy is the point: + + * ``get_downloaded_file`` discarding a leftover ``.tmp`` before each download is + the one that makes resume safe. It is the only one that covers a content change + within a single release, or a ``local_path`` this method never looked at. + * This sweep is housekeeping. It reclaims gigabytes belonging to a release nobody + will ask for again, including for files that are never re-downloaded and so + never reach ``get_downloaded_file``. + + Neither is redundant with the other for the case it owns. Removing this sweep only + wastes disk; removing the one in ``get_downloaded_file`` reintroduces silent + Parquet corruption — see the comment there before touching either. + + The new release is *not* written to the version marker here — it is handed to + ``_write_version_marker_if_synced``, which stamps it only once the cache actually + holds it. See that method for why writing it up front is unsafe. + """ + version = self.babel_version + if version is None: + self.logger.warning( + f"Could not determine the Babel version at {self.url_base}; " + f"using cached files in {self.local_path} as-is" + ) + return + + marker_path = os.path.join(self.local_path, VERSION_MARKER) + try: + with open(marker_path) as f: + cached_version = f.read().strip() + except OSError: + cached_version = None + + if cached_version and cached_version != version: + self.logger.warning( + f"Babel version changed: {cached_version} → {version}; " + f"refreshing cached files in {self.local_path}" + ) + # Only ever touch files this downloader wrote. Not recursive: local_path + # may be a directory the user pointed us at (or one holding other Babel + # releases in sibling subdirectories), and must not be cleared wholesale. + duckdb_dir = os.path.join(self.local_path, "duckdb") + for meta_path in glob.glob(os.path.join(duckdb_dir, "*.meta")): + meta = self._load_meta(meta_path.removesuffix(".meta")) + if meta is None: + os.remove(meta_path) + continue + meta.pop("last_checked", None) + with open(meta_path, "w") as f: + json.dump(meta, f, indent=2) + for tmp_path in glob.glob(os.path.join(duckdb_dir, "*.tmp")): + os.remove(tmp_path) + + if cached_version != version: + self._pending_version = version + # Usually a no-op here (the sidecars were just expired), but it covers the + # cases with nothing to catch up on: a first run with an empty cache, and a + # cache written before this tool kept a marker at all. + self._write_version_marker_if_synced() + + def _write_version_marker_if_synced(self): + """Record the pending release in ``.babel-version`` once the cache matches it. + + The marker claims "the local cache holds this release", and that only becomes + true once every cached file has been re-validated against it — so it is written + here, after downloads, rather than by ``sync_cache_version`` the moment the change + is noticed. Stamping it up front leaves a run that is interrupted after + ``Concord.parquet`` is refreshed but before ``Identifiers.parquet`` is with a + marker naming the new release over a half-old cache; the next run then sees a + marker that matches, does no version-driven refresh, and reads the two Parquet + files together across two different Babel releases. + + "Re-validated" is exactly "the sidecar has a ``last_checked`` again": + ``sync_cache_version`` cleared it from all of them, and only a confirmed-unchanged + HEAD or a completed download puts it back. A cached file that nobody asks for + therefore holds the marker back indefinitely, at a cost of one HEAD per run. That + is the honest answer rather than a bug — that file really is still from the + previous release. + """ + version = self._pending_version + if version is None: + return + + duckdb_dir = os.path.join(self.local_path, "duckdb") + for meta_path in glob.glob(os.path.join(duckdb_dir, "*.meta")): + meta = self._load_meta(meta_path.removesuffix(".meta")) + if meta is None or "last_checked" not in meta: + return + + with open(os.path.join(self.local_path, VERSION_MARKER), "w") as f: + f.write(version + "\n") + self._pending_version = None + + def _get_meta_path(self, local_path): + """Return the sidecar metadata file path for a given local file.""" + return local_path + ".meta" + + def _load_meta(self, local_path): + """Load sidecar metadata JSON, or return None if not found/invalid.""" + meta_path = self._get_meta_path(local_path) + if not os.path.exists(meta_path): + return None + try: + with open(meta_path) as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return None + + def _write_meta(self, local_path, meta): + """Write the sidecar .meta JSON file for local_path, stamping last_checked as now.""" + meta = meta | {"last_checked": datetime.now(UTC).isoformat()} + with open(self._get_meta_path(local_path), "w") as f: + json.dump(meta, f, indent=2) + + @staticmethod + def _full_content_length(headers, local_path): + """ + Return the length of the *whole* remote file, or None if it is not known. + + A partial (HTTP 206) response's ``Content-Length`` is the length of the + returned range, not of the file. Recording that as the file's length would + make the Last-Modified + Content-Length fallback in ``_remote_unchanged`` + compare a partial length against the full remote one forever, re-downloading + a multi-gigabyte file on every freshness expiry. ``Content-Range`` carries + the total (``bytes 100-999/1000``); when it is present but the total is + unknown (``/*``), the file on disk is the better answer. + """ + content_range = headers.get("Content-Range") + if content_range: + total = content_range.rsplit("/", 1)[-1].strip() + if total.isdigit(): + return int(total) + try: + return os.path.getsize(local_path) + except OSError: + return None + if "Content-Length" in headers: + return int(headers["Content-Length"]) + return None + + def _save_meta(self, local_path, headers): + """ + Write a sidecar .meta JSON file next to local_path from response headers. + + Args: + local_path: Path to the downloaded file + headers: Response headers dict (or requests.structures.CaseInsensitiveDict) + """ + meta = {} + if "ETag" in headers: + meta["etag"] = headers["ETag"] + if "Last-Modified" in headers: + meta["last_modified"] = headers["Last-Modified"] + content_length = self._full_content_length(headers, local_path) + if content_length is not None: + meta["content_length"] = content_length + + self._write_meta(local_path, meta) + + def _is_within_freshness(self, meta, freshness_seconds): + """ + Return True if last_checked is within freshness_seconds of now. + + A file that has never been validated against the current release is never + fresh, whatever the window. That case is tested *before* the ``float('inf')`` + shortcut, and the order matters: ``sync_cache_version`` clears ``last_checked`` + from every sidecar when the Babel release changes, and if ``inf`` short-circuited + ahead of that, ``--check-download never`` would hand back the previous release's + Parquet without a single network call — while the version marker went on to name + the new release, so the mismatch would never be noticed again. ``never`` means + "do not re-check for changes *within* a release", not "ignore a release change". + + Args: + meta: dict loaded from .meta file + freshness_seconds: Number of seconds; float('inf') means always fresh + + Returns: + bool + """ + last_checked_str = meta.get("last_checked") + if not last_checked_str: + return False + if freshness_seconds == float("inf"): + return True + try: + last_checked = datetime.fromisoformat(last_checked_str) + age = (datetime.now(UTC) - last_checked).total_seconds() + return age < freshness_seconds + except (ValueError, TypeError): + return False + + def _remote_unchanged(self, url, meta): + """ + Do a HEAD request and check if the ETag (or Last-Modified + Content-Length) + matches the stored metadata. + + Does not write to disk — the caller is responsible for updating last_checked + when this returns ``True``. + + Args: + url: URL to HEAD + meta: dict loaded from .meta file (may have etag, last_modified, content_length) + + Returns: + True if the remote file is confirmed to match the local metadata, + False if it is confirmed to have changed, and ``None`` if the check + could not be made (the HEAD request failed). ``None`` is *not* the + same as ``True``: the cached file is still usable, but the caller must + not refresh ``last_checked`` on the strength of a check that never + happened. Doing so would let one flaky HEAD pin the previous release's + Parquet as "freshly validated" for the whole freshness window, right + after ``sync_cache_version`` cleared ``last_checked`` for a new release + — exactly the cross-release mixing the version marker exists to prevent. + """ + try: + response = requests.head(url, timeout=self.timeout) + response.raise_for_status() + except requests.RequestException as e: + self.logger.warning( + f"HEAD request failed for {url}: {e}; using the cached file, " + f"but it will be re-checked on next use" + ) + return None + + remote_headers = response.headers + + # Primary check: ETag + local_etag = meta.get("etag") + remote_etag = remote_headers.get("ETag") + if local_etag and remote_etag: + if local_etag == remote_etag: + self.logger.info(f"ETag matches ({remote_etag}), file is current") + return True + else: + self.logger.info( + f"ETag changed: {local_etag!r} → {remote_etag!r}, re-downloading" + ) + return False + + # Fallback: Last-Modified + Content-Length + local_lm = meta.get("last_modified") + remote_lm = remote_headers.get("Last-Modified") + local_cl = meta.get("content_length") + remote_cl = remote_headers.get("Content-Length") + + if local_lm and remote_lm and local_lm == remote_lm: + if local_cl is None or remote_cl is None or int(remote_cl) == local_cl: + self.logger.info( + f"Last-Modified matches ({remote_lm}), file is current" + ) + return True + + self.logger.info( + "Cannot confirm file is current (no matching ETag or Last-Modified), will re-download" + ) + return False + + def _stream_download(self, response, local_path, resume_byte_pos, chunk_size): + """ + Stream download from response to file with progress bar. + + Args: + response: requests.Response object with stream=True + local_path: Local file path to write to + resume_byte_pos: Starting byte position (for resume) + chunk_size: Size of chunks to read/write + + Raises: + IncompleteDownloadError: If fewer bytes arrived than Content-Length + advertised. + """ + content_length = response.headers.get("Content-Length") + if content_length: + total_size = int(content_length) + resume_byte_pos + else: + total_size = None + + mode = "ab" if resume_byte_pos > 0 else "wb" + + with open(local_path, mode) as f: + with tqdm( + total=total_size, + initial=resume_byte_pos, + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=os.path.basename(local_path), + ) as progress_bar: + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + progress_bar.update(len(chunk)) + + # A stream can end early without raising. Comparing against Content-Length + # is only meaningful for an identity-coded body: with Content-Encoding set, + # iter_content hands back decoded bytes whose count is unrelated to it. + if total_size is not None and not response.headers.get("Content-Encoding"): + written = os.path.getsize(local_path) + if written != total_size: + raise IncompleteDownloadError( + f"{local_path}: expected {total_size} bytes, received {written}" + ) + + def _download_with_retry(self, url, local_path, chunk_size): + """ + Download a file with retry logic and resume capability. + + Args: + url: URL to download from + local_path: Local file path to save to + chunk_size: Size of chunks to read/write + + Returns: + requests.structures.CaseInsensitiveDict: Response headers from the final request + + Raises: + RuntimeError: If all retry attempts fail + """ + # Validator (ETag, else Last-Modified) of the response we started writing + # from. Sent back as If-Range on a resume so a file that changed mid-download + # restarts from scratch instead of having the new version's tail appended to + # the old version's prefix — a splice that would pass every later ETag check. + # A leftover .tmp from an earlier run carries no validator and is never + # resumed; get_downloaded_file removes it before we are called. + # + # Having one is a *precondition* for resuming at all: see the restart below. + validator = None + + for attempt in range(1, self.retries + 1): + try: + resume_byte_pos = 0 + if os.path.exists(local_path): + resume_byte_pos = os.path.getsize(local_path) + + if resume_byte_pos > 0 and not validator: + # Nothing to make the resume conditional on. A server that sent + # neither an ETag nor a Last-Modified leaves no way to ask for + # "the rest of *this* file", so a bare Range against a file + # rebuilt between attempts splices the new version's tail onto + # the old version's prefix — the very corruption If-Range exists + # to prevent, and just as permanent, since what lands gets + # stamped with the new validator and passes every later check. + # Such a server costs a restart instead. + self.logger.warning( + f"Restarting {local_path} from the beginning: the response it " + f"was written from carried no ETag or Last-Modified, so the " + f"resume cannot be made conditional" + ) + os.remove(local_path) + resume_byte_pos = 0 + + headers = {} + if resume_byte_pos > 0: + headers["Range"] = f"bytes={resume_byte_pos}-" + headers["If-Range"] = validator + self.logger.info(f"Resuming download from byte {resume_byte_pos}") + + # timeout is per-read (seconds without receiving bytes), not a total time limit. + with requests.get( + url, headers=headers, stream=True, timeout=self.timeout + ) as response: + if response.status_code == 416: + # 416 also comes back when the remote file *shrank* below our + # resume offset, so "the range is past the end" does not by + # itself mean the local file is the remote one. Only a remote + # length that matches the local one proves that, so a missing + # Content-Length is treated as "cannot confirm" and restarts + # too: promoting an unverified file leaves a wrong-length + # Parquet stamped with valid-looking metadata. The restart is + # safe from looping, since the retry sends no Range at all. + head = requests.head(url, timeout=self.timeout) + head.raise_for_status() + remote_length = head.headers.get("Content-Length") + if ( + remote_length is None + or int(remote_length) != resume_byte_pos + ): + self.logger.warning( + f"Local file is {resume_byte_pos} bytes but the remote " + f"length is {remote_length or 'unknown'}; discarding it " + f"and downloading afresh" + ) + if os.path.exists(local_path): + os.remove(local_path) + continue + self.logger.info(f"File already complete: {local_path}") + # The 416 headers describe the error body, not the file; saving + # them as this file's metadata would record a bogus + # content_length and force a full re-download on the next check. + return head.headers + elif response.status_code == 206: + self.logger.info("Resuming download (HTTP 206)") + elif response.status_code == 200: + if resume_byte_pos > 0: + self.logger.warning( + "Server doesn't support resume, restarting from beginning" + ) + resume_byte_pos = 0 + if os.path.exists(local_path): + os.remove(local_path) + elif response.status_code == 404: + # Not worth retrying, and worth explaining: public Babel releases + # do not currently publish the DuckDB Parquet files. + raise MissingBabelFileError( + f"This Babel release ({self.babel_version or self.url_base}) does not " + f"publish {url[len(self.url_base) :]}. Translator team members " + f"should contact the Babel developers for the Translator-specific " + f"releases URL and set BABEL_RELEASES_URL in .env, or pass " + f"--babel-url with a complete release URL for a one-off run." + ) + else: + response.raise_for_status() + + validator = response.headers.get("ETag") or response.headers.get( + "Last-Modified" + ) + self._stream_download( + response, local_path, resume_byte_pos, chunk_size + ) + return response.headers + + except (OSError, requests.RequestException, IncompleteDownloadError) as e: + self.logger.warning( + f"Download attempt {attempt}/{self.retries} failed: {e}" + ) + + if attempt < self.retries: + wait_time = min(2**attempt, 60) + self.logger.info(f"Retrying in {wait_time} seconds...") + time.sleep(wait_time) + else: + raise RuntimeError( + f"Failed to download {url} after {self.retries} attempts: {e}" + ) + + # Only reachable if the last attempt was a 416 that restarted the download + # (`continue`) with no attempts left. Falling through would return None and + # leave the caller replacing a .tmp that is no longer there. + raise RuntimeError(f"Failed to download {url} after {self.retries} attempts") + + def get_downloaded_file(self, dirpath: str, chunk_size: int = 1024 * 1024): + """ + Download a file from the Babel server to local storage with ETag-based caching. + + Three-tier freshness logic: + 1. If .meta exists and last_checked is within freshness window → return immediately + 2. If .meta exists but stale → HEAD request to compare ETag; return if unchanged + (or if the HEAD failed, in which case last_checked is left alone so the + check is retried on next use) + 3. If ETag changed or no .meta → full re-download + + Args: + dirpath: Relative path from url_base to the file + chunk_size: Size of chunks to download (default 1MB) + + Returns: + str: Local path to the downloaded file + """ + local_path = self._fetch_file(dirpath, chunk_size) + # This file may have been the last one still holding the cache back from a + # release change sync_cache_version spotted. If so, this is where the version + # marker finally gets written. + self._write_version_marker_if_synced() + return local_path + + def _fetch_file(self, dirpath: str, chunk_size: int): + """Do the actual fetching for ``get_downloaded_file``, which see.""" + local_path_to_download_to = os.path.join(self.local_path, dirpath) + os.makedirs(os.path.dirname(local_path_to_download_to), exist_ok=True) + + url_to_download = self.url_base + dirpath + + if os.path.exists(local_path_to_download_to): + meta = self._load_meta(local_path_to_download_to) + if meta is not None: + # Tier 1: within freshness window — skip all network calls + if self._is_within_freshness(meta, self.freshness_seconds): + self.logger.info( + f"File within freshness window ({self.freshness_seconds} seconds), skipping check: {local_path_to_download_to}" + ) + return local_path_to_download_to + + # Tier 2: stale but maybe unchanged — HEAD request + unchanged = self._remote_unchanged(url_to_download, meta) + if unchanged is True: + self._write_meta(local_path_to_download_to, meta) + self.logger.info( + f"ETag matches, using existing file: {local_path_to_download_to}" + ) + return local_path_to_download_to + if unchanged is None: + # Could not reach the server. Use the cached file, but leave + # last_checked stale so the next run checks again rather than + # treating an unverified file as fresh for hours. + self.logger.warning( + f"Could not check whether {url_to_download} changed; " + f"using the cached file: {local_path_to_download_to}" + ) + return local_path_to_download_to + + # Tier 3: ETag changed — re-download + self.logger.warning( + f"Remote file changed, re-downloading: {local_path_to_download_to}" + ) + + self.logger.info( + f"Downloading {url_to_download} to {local_path_to_download_to}" + ) + + # Download to a sibling .tmp file, then atomically replace the final destination. + # This ensures the final file is never partially written. + tmp_path = local_path_to_download_to + ".tmp" + + # Discard any .tmp left behind by an earlier run (killed process, Ctrl-C). + # _download_with_retry resumes by byte offset, and we have no way to tell + # which version of the remote file those bytes came from — while the only + # way to reach this point with a cached file present is that the remote + # bytes *changed*. Resuming would splice the new file's tail onto the old + # file's prefix and then stamp the result with the new ETag, making the + # corruption permanent. Restarting costs a re-download; splicing costs + # silent, undetectable data corruption. + if os.path.exists(tmp_path): + self.logger.warning( + f"Discarding partial download from an earlier run: {tmp_path}" + ) + os.remove(tmp_path) + + try: + response_headers = self._download_with_retry( + url_to_download, tmp_path, chunk_size + ) + os.replace(tmp_path, local_path_to_download_to) + except BaseException: + # BaseException, not Exception: a Ctrl-C mid-download must clean up too, + # since the partial file cannot be safely resumed later. + if os.path.exists(tmp_path): + os.remove(tmp_path) + raise + + # Save sidecar metadata + if response_headers is not None: + self._save_meta(local_path_to_download_to, response_headers) + + bytes_downloaded = os.path.getsize(local_path_to_download_to) + self.logger.info( + f"Downloaded {url_to_download} to {local_path_to_download_to}: {bytes_downloaded} bytes" + ) + return local_path_to_download_to diff --git a/src/babel_explorer/core/nodenorm.py b/src/babel_explorer/core/nodenorm.py new file mode 100644 index 0000000..31592b5 --- /dev/null +++ b/src/babel_explorer/core/nodenorm.py @@ -0,0 +1,202 @@ +"""NodeNorm API client for identifier normalisation and label enrichment.""" + +import dataclasses +import functools +import logging + +import requests + +#: Maximum CURIEs per get_normalized_nodes request. Keeps the query string well +#: inside the usual 8 KB server limit while collapsing hundreds of lookups into +#: a handful of round-trips. +NORMALIZE_BATCH_SIZE = 100 + + +@dataclasses.dataclass(frozen=True) +class Identifier: + """Normalised identifier record returned by the NodeNorm API.""" + + curie: str + label: str = "" + biolink_type: tuple[str, ...] = () + taxa: tuple[str, ...] = () + description: tuple[str, ...] = () + + def __lt__(self, other): + return self.curie < other.curie + + @staticmethod + def from_dict(d: dict) -> "Identifier": + def _to_tuple(val) -> tuple[str, ...]: + """Coerce a string or list to a tuple — guards against iterating string chars.""" + if not val: + return () + return (val,) if isinstance(val, str) else tuple(val) + + return Identifier( + curie=d["identifier"], + label=d.get("label", ""), + biolink_type=_to_tuple(d.get("type")), + taxa=_to_tuple(d.get("taxa")), + description=_to_tuple(d.get("description")), + ) + + +class NodeNorm: + """Client for the NodeNormalization API (https://nodenormalization-sri.renci.org/). + + Results are cached per instance. To get uncached results, instantiate a new + NodeNorm object. + """ + + def __init__(self, nodenorm_url: str = "", timeout: int = 30): + """ + :param nodenorm_url: Base URL of the NodeNorm service. Pass an empty string (default) + to skip all network calls and have every lookup return a bare ``Identifier``. + :param timeout: HTTP request timeout in seconds. + """ + self.nodenorm_url = nodenorm_url + self.timeout = timeout + if self.nodenorm_url and not self.nodenorm_url.endswith("/"): + self.nodenorm_url += "/" + self._normalize_cache: dict[str, dict | None] = {} + self._identifier_cache: dict[str, Identifier] = {} + self._clique_cache: dict[str, list[Identifier]] = {} + + def get_babel_version(self) -> str | None: + """Return the Babel release this NodeNorm instance was built from. + + :return: The version reported by the ``status`` endpoint, or ``None`` in offline + mode or if the endpoint cannot be reached or does not report one. + + The result is cached per instance — including ``None``, so an unreachable + status endpoint is not re-queried on every lookup. + """ + return self._babel_version + + @functools.cached_property + def _babel_version(self) -> str | None: + if not self.nodenorm_url: + return None + try: + response = requests.get(f"{self.nodenorm_url}status", timeout=self.timeout) + response.raise_for_status() + return response.json().get("babel_version") + except (requests.RequestException, ValueError) as e: + logging.warning(f"Could not read the Babel version from NodeNorm: {e}") + return None + + def get_identifier(self, curie: str) -> "Identifier": + """Return the ``Identifier`` for *curie* by looking it up in its NodeNorm clique. + + Searches ``equivalent_identifiers`` for an entry whose ``identifier`` field matches + *curie* exactly. Falls back to a bare ``Identifier(curie=curie)`` (empty label and + type) if NodeNorm does not recognise the CURIE or it is not listed in the clique. + + Results are cached per instance. + """ + if curie in self._identifier_cache: + return self._identifier_cache[curie] + + result = self.normalize_curie(curie) + logging.debug(f"Normalizing {curie} with NodeNorm to result: {result}") + if not result: + ident = Identifier(curie=curie) + else: + for identifier in result.get("equivalent_identifiers", []): + if identifier["identifier"] == curie: + logging.debug(f"Found exact match for {curie}: {identifier}") + ident = Identifier.from_dict(identifier) + break + else: + logging.debug( + f"No exact match for {curie!r} in equivalent_identifiers; returning bare Identifier" + ) + ident = Identifier(curie=curie) + + self._identifier_cache[curie] = ident + return ident + + def _fetch_normalized(self, curies: list[str]) -> None: + """Fetch *curies* from ``get_normalized_nodes`` and populate the cache. + + :raises requests.HTTPError: If the API returns a non-2xx status code. + Nothing is cached in that case, so the lookup is retried next time. + """ + response = requests.get( + f"{self.nodenorm_url}get_normalized_nodes", + params={ + "curie": curies, + "conflate": True, + "drug_chemical_conflate": True, + "description": True, + "individual_types": True, + "include_taxa": True, + }, + timeout=self.timeout, + ) + response.raise_for_status() + result = response.json() + + for curie in curies: + if curie not in result: + logging.debug( + f"NodeNorm response did not contain CURIE {curie!r}; caching None" + ) + # NodeNorm reports an unrecognised CURIE as a null value, not a missing key. + self._normalize_cache[curie] = result.get(curie) + + def normalize_curies(self, curies) -> None: + """Populate the normalisation cache for *curies* in as few requests as possible. + + Callers that are about to look up many CURIEs should call this first: the + per-CURIE accessors then hit the cache instead of issuing one HTTPS + round-trip each, which dominates runtime on a large clique. + + :raises requests.HTTPError: If the API returns a non-2xx status code. + """ + missing = sorted({c for c in curies if c not in self._normalize_cache}) + if not missing: + return + + if not self.nodenorm_url: + self._normalize_cache.update(dict.fromkeys(missing)) + return + + for i in range(0, len(missing), NORMALIZE_BATCH_SIZE): + self._fetch_normalized(missing[i : i + NORMALIZE_BATCH_SIZE]) + + def normalize_curie(self, curie: str): + """Call ``get_normalized_nodes`` and return the per-CURIE result dict. + + :return: The normalisation dict for *curie* (contains ``id``, ``equivalent_identifiers``, + ``type``, etc.), or ``None`` if the CURIE is not recognised by NodeNorm. + :raises requests.HTTPError: If the API returns a non-2xx status code. + + Results are cached per instance. HTTP errors are not cached. + """ + if curie not in self._normalize_cache: + self.normalize_curies([curie]) + return self._normalize_cache[curie] + + def get_clique_identifiers(self, curie: str) -> list[Identifier]: + """Return all ``Identifier`` objects in the NodeNorm clique for *curie*. + + :return: A list of ``Identifier`` objects (one per entry in ``equivalent_identifiers``), + or an empty list if the CURIE is unknown or has no equivalents. + + Results are cached per instance. + """ + if curie in self._clique_cache: + return self._clique_cache[curie] + + result = self.normalize_curie(curie) + if not result or "equivalent_identifiers" not in result: + identifiers = [] + else: + identifiers = [ + Identifier.from_dict(x) for x in result["equivalent_identifiers"] + ] + + self._clique_cache[curie] = identifiers + return identifiers diff --git a/src/babel_explorer/formatting.py b/src/babel_explorer/formatting.py new file mode 100644 index 0000000..b658a0b --- /dev/null +++ b/src/babel_explorer/formatting.py @@ -0,0 +1,160 @@ +"""Output formatting for babel-explorer CLI commands. + +Provides: +- write_records() for machine-readable output (json, tsv, csv) +- make_console(), hl_curie() and curie_with_label() for rich console output +""" + +import csv +import dataclasses +import json +import sys +from typing import Any + +from rich.console import Console +from rich.markup import escape + +#: Label fields populated from NodeNorm, which are omitted from machine-readable +#: output when empty. IdentifierRecord spells it nodenorm_label; +#: LabeledCrossReference spells it subj_label/obj_label. +_NODENORM_LABEL_FIELDS = ("nodenorm_label", "subj_label", "obj_label") + + +def record_to_dict(record) -> dict[str, Any]: + """Convert a dataclass (or plain dict) to a flat dict. + + Handles IdentifierRecord's extra_fields, which asdict() returns as a + list of [col, val] pairs rather than a nested dict. + """ + if isinstance(record, dict): + return record + d = dataclasses.asdict(record) + if "extra_fields" in d: + for col, val in d.pop("extra_fields"): + d[col] = val + # An absent label is omitted rather than emitted as "", matching the console + # convention and keeping TSV/CSV columns stable when labels were not requested. + # Listed explicitly rather than matched on a "label" suffix: extra_fields has + # already been flattened in above, and Identifiers.parquet has its own `label` + # column (Babel may add more) whose empty values are real data, not an absent + # NodeNorm lookup. Dropping those would make `label` present on some rows of a + # json/tsv/csv run and missing on others. + for key in [k for k in _NODENORM_LABEL_FIELDS if k in d and not d[k]]: + del d[key] + return d + + +def _flatten_for_tabular(row: dict) -> dict: + """Convert list/tuple fields to pipe-joined strings for TSV/CSV output.""" + return { + k: "|".join(v) if isinstance(v, (list, tuple)) else v for k, v in row.items() + } + + +def make_console(file=None) -> Console: + """Create a rich Console with babel-explorer defaults. + + Auto-detects TTY and NO_COLOR; strips markup when output is piped. + highlight=False prevents rich from auto-highlighting numbers and strings. + """ + return Console(file=file, highlight=False) + + +# Styles indexed by BFS depth from the nearest query CURIE. +# Depth 0 = the query term itself; higher = further away. +_DEPTH_STYLES = [ + "bold cyan", # 0: query CURIE + "bold yellow", # 1: one hop away + "yellow", # 2: two hops + "green", # 3: three hops + "dim", # 4+: further +] + + +def hl_curie(curie: str, depth: int | None) -> str: + """Return rich markup for a CURIE colored by its BFS depth from the nearest query CURIE. + + Depth 0 is a query CURIE itself. Pass ``depth=None`` for CURIEs whose depth is + unknown or irrelevant (rendered unstyled). + """ + escaped = escape(curie) + if depth is None: + return escaped + style = _DEPTH_STYLES[min(depth, len(_DEPTH_STYLES) - 1)] + return f"[{style}]{escaped}[/{style}]" + + +def escape_label(label: str) -> str: + """Escape a label for display inside double quotes: backslashes first, then quotes. + + Downstream tools can parse the result with the regex ``"([^"\\\\]|\\\\.)*"``. + """ + return label.replace("\\", "\\\\").replace('"', '\\"') + + +def curie_with_label(curie: str, depth: int | None, label: str | None = None) -> str: + """Render a CURIE as rich markup, followed by its label in double quotes. + + The sole implementation of the console label convention: the label sits + immediately after the CURIE in double quotes, and is omitted entirely when + absent rather than rendered as a placeholder. + """ + markup = hl_curie(curie, depth) + if label: + markup += f' "{escape(escape_label(label))}"' + return markup + + +def format_identifier_record(record) -> str: + """Render an IdentifierRecord as a ``key=value`` line of rich markup. + + The label sits immediately after the CURIE in double quotes and is omitted + entirely when absent, per the console convention. + """ + parts = [f"curie={record.curie!r}"] + if record.nodenorm_label: + parts.append(f'nodenorm_label="{escape_label(record.nodenorm_label)}"') + parts.extend(f"{name}={value!r}" for name, value in record.extra_fields) + # Parquet values are arbitrary text; escape so they are not read as markup. + return escape(f"IdentifierRecord({', '.join(parts)})") + + +def write_records(records, fmt: str, indent: int = 2, file=None): + """Write an iterable of dataclass records (or dicts) in the requested format. + + :param records: Iterable of dataclass instances or plain dicts. + :param fmt: One of "json", "tsv", "csv". (Console output is handled by + make_console/hl_curie in the CLI layer.) + :param indent: JSON indentation depth (ignored for other formats). + :param file: Output file-like object; defaults to sys.stdout. + :raises ValueError: If fmt is not a recognised format. + """ + if file is None: + file = sys.stdout + records = list(records) + + if fmt == "json": + rows = [record_to_dict(r) for r in records] + json.dump(rows, file, indent=indent, default=str) + print(file=file) # trailing newline + + elif fmt in ("tsv", "csv"): + if not records: + return + rows = [_flatten_for_tabular(record_to_dict(r)) for r in records] + # Union of keys, in first-seen order: records that omit an absent label have + # fewer keys than their neighbours, and DictWriter rejects any key not declared. + fieldnames = list(dict.fromkeys(k for row in rows for k in row)) + delimiter = "\t" if fmt == "tsv" else "," + writer = csv.DictWriter( + file, + fieldnames=fieldnames, + restval="", + delimiter=delimiter, + lineterminator="\n", + ) + writer.writeheader() + writer.writerows(rows) + + else: + raise ValueError(f"Unknown format: {fmt!r}") diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f39ae03 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,145 @@ +""" +Shared fixtures for babel-explorer tests. + +Session-scoped fixtures download Babel files once and share them across all test modules. +Teardown removes the test data directory so the next run starts fresh. +""" + +import os +import shutil + +import pytest +import requests +from filelock import FileLock + +from babel_explorer.core.babel_xrefs import BabelXRefs +from babel_explorer.core.downloader import BabelDownloader +from babel_explorer.core.nodenorm import NodeNorm +from tests.constants import ( + BABEL_URL, + CONCORD_FILE, + IDENTIFIERS_FILE, + METADATA_FILE, + NODENORM_URL, + TEST_DATA_DIR, + load_curies, +) + +# --------------------------------------------------------------------------- +# Session-scoped fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def valid_curies() -> list[str]: + """Load test CURIEs from tests/data/valid_curies.txt.""" + curies = load_curies() + assert len(curies) > 0, "No CURIEs found in valid_curies.txt" + return curies + + +def pytest_sessionfinish(session, exitstatus): + """Remove the shared test data directory once every worker has finished. + + This runs in the xdist controller, which finishes only after all workers do. + Workers are identified by having a ``workerinput`` attribute; a plain + non-parallel run has none either, so cleanup happens there too. + + Cleaning up from a session fixture's teardown instead does not work: with + ``-n auto`` in ``addopts`` every run is parallel, so each worker would tear + down at an unpredictable time and gw0 could delete Concord.parquet while gw5 + is still reading it. Guarding that teardown on the worker id, as this used to, + meant the directory was simply never removed. + """ + if hasattr(session.config, "workerinput"): + return + if os.path.exists(TEST_DATA_DIR): + shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + + +@pytest.fixture(scope="session") +def test_data_dir(): + """Provide a test data directory for the entire session. + + Removed by ``pytest_sessionfinish`` once all workers are done, so the next + run starts fresh. + """ + os.makedirs(TEST_DATA_DIR, exist_ok=True) + return TEST_DATA_DIR + + +@pytest.fixture(scope="session") +def shared_downloader(test_data_dir) -> BabelDownloader: + """A BabelDownloader pointed at the test data directory. + + Skips the whole session when the composed Babel URL points at a release that does + not publish the DuckDB Parquet files (as the public releases currently do not). + """ + probe_url = BABEL_URL + CONCORD_FILE + try: + response = requests.head(probe_url, timeout=30) + except requests.RequestException as e: + pytest.skip(f"Babel server unreachable at {probe_url}: {e}") + if response.status_code == 404: + pytest.skip(f"{BABEL_URL} does not publish {CONCORD_FILE}") + return BabelDownloader(url_base=BABEL_URL, local_path=test_data_dir) + + +@pytest.fixture(scope="session") +def downloaded_concord(shared_downloader, test_data_dir) -> str: + """Download duckdb/Concord.parquet. Returns the local path. + + Multi-gigabyte in current releases and growing; do not record a figure here, + it drifts silently and then misleads. + """ + lock_path = os.path.join(test_data_dir, "concord.lock") + with FileLock(lock_path): + return shared_downloader.get_downloaded_file(CONCORD_FILE) + + +@pytest.fixture(scope="session") +def downloaded_metadata(shared_downloader, test_data_dir) -> str: + """Download duckdb/Metadata.parquet (small). Returns the local path.""" + lock_path = os.path.join(test_data_dir, "metadata.lock") + with FileLock(lock_path): + return shared_downloader.get_downloaded_file(METADATA_FILE) + + +@pytest.fixture(scope="session") +def downloaded_parquet_files(downloaded_concord, downloaded_metadata) -> dict[str, str]: + """Dict of {relative_name: local_path} for Concord and Metadata files.""" + return { + CONCORD_FILE: downloaded_concord, + METADATA_FILE: downloaded_metadata, + } + + +@pytest.fixture(scope="session") +def downloaded_identifiers(shared_downloader, test_data_dir) -> str: + """Download duckdb/Identifiers.parquet, the largest file Babel publishes. + + Every test that reaches this is marked ``slow``. + """ + lock_path = os.path.join(test_data_dir, "identifiers.lock") + with FileLock(lock_path): + return shared_downloader.get_downloaded_file(IDENTIFIERS_FILE) + + +@pytest.fixture(scope="session") +def nodenorm() -> NodeNorm: + """A NodeNorm client pointed at the public API.""" + return NodeNorm(nodenorm_url=NODENORM_URL) + + +@pytest.fixture(scope="session") +def babel_xrefs(shared_downloader, downloaded_parquet_files) -> BabelXRefs: + """A BabelXRefs instance (no NodeNorm) with Concord + Metadata already downloaded.""" + return BabelXRefs(shared_downloader) + + +@pytest.fixture(scope="session") +def babel_xrefs_with_nodenorm( + shared_downloader, nodenorm, downloaded_parquet_files +) -> BabelXRefs: + """A BabelXRefs instance with NodeNorm, Concord + Metadata already downloaded.""" + return BabelXRefs(shared_downloader, nodenorm) diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 0000000..025a289 --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,46 @@ +"""Shared constants for babel-explorer tests.""" + +import os +import pathlib + +from dotenv import load_dotenv + +from babel_explorer.core.downloader import compose_babel_url + +# Integration tests run against whatever BABEL_RELEASES_URL and BABEL_VERSION compose to, +# so a Translator developer with a .env exercises them while public contributors and CI +# fall back to the public release (which does not yet publish the DuckDB Parquet files, +# so those tests skip). +load_dotenv() + +BABEL_RELEASES_URL = os.environ.get( + "BABEL_RELEASES_URL", "https://stars.renci.org/var/babel/" +) +BABEL_VERSION = os.environ.get("BABEL_VERSION", "latest") + +# Composed exactly the way the CLI composes it, so tests that join paths onto it directly +# agree with the downloader instead of quietly requesting ".../latestduckdb/". +BABEL_URL = compose_babel_url(BABEL_RELEASES_URL, BABEL_VERSION) +NODENORM_URL = os.environ.get( + "NODENORM_URL", "https://nodenormalization-sri.renci.org/" +) +TEST_DATA_DIR = "data/test" + +# Parquet file paths (relative to the Babel server / local data dir) +CONCORD_FILE = "duckdb/Concord.parquet" +METADATA_FILE = "duckdb/Metadata.parquet" +IDENTIFIERS_FILE = "duckdb/Identifiers.parquet" + +# Path to the valid CURIEs file +VALID_CURIES_PATH = pathlib.Path(__file__).parent / "data" / "valid_curies.txt" + + +def load_curies(path: pathlib.Path = VALID_CURIES_PATH) -> list[str]: + """Read CURIEs from a text file, skipping comments and blank lines.""" + curies = [] + with open(path) as f: + for line in f: + stripped = line.strip() + if stripped and not stripped.startswith("#"): + curies.append(stripped) + return curies diff --git a/tests/data/valid_curies.txt b/tests/data/valid_curies.txt new file mode 100644 index 0000000..89a53b3 --- /dev/null +++ b/tests/data/valid_curies.txt @@ -0,0 +1,5 @@ +# Valid CURIEs for integration tests. +# Add new CURIEs here to expand test coverage — tests are parametrized over this list. +MONDO:0004979 +MONDO:0005044 +NCIT:C55060 diff --git a/tests/test_babel_xrefs.py b/tests/test_babel_xrefs.py new file mode 100644 index 0000000..87497b1 --- /dev/null +++ b/tests/test_babel_xrefs.py @@ -0,0 +1,517 @@ +""" +Tests for BabelXRefs, CrossReference, LabeledCrossReference, and IdentifierRecord. + +Unit tests use mocks; integration tests query real Parquet files via DuckDB. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from babel_explorer.core.babel_xrefs import ( + BabelXRefs, + CrossReference, + IdentifierRecord, + LabeledCrossReference, + build_adjacency, + build_depth_map, + find_shortest_path, +) +from babel_explorer.core.downloader import BabelDownloader +from babel_explorer.core.nodenorm import NodeNorm +from tests.constants import load_curies + +VALID_CURIES = load_curies() + + +# ========================================================================== +# Unit Tests — CrossReference +# ========================================================================== + + +class TestCrossReference: + """Tests for the CrossReference frozen dataclass.""" + + def test_creation(self): + xr = CrossReference( + filename="f.txt", subj="A:1", pred="skos:exactMatch", obj="B:2" + ) + assert xr.filename == "f.txt" + assert xr.subj == "A:1" + assert xr.pred == "skos:exactMatch" + assert xr.obj == "B:2" + + def test_from_tuple(self): + t = ("file.tsv", "MONDO:1", "owl:sameAs", "HP:2") + xr = CrossReference.from_tuple(t) + assert xr.filename == "file.tsv" + assert xr.subj == "MONDO:1" + assert xr.pred == "owl:sameAs" + assert xr.obj == "HP:2" + + def test_curies_property(self): + xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + assert xr.curies == frozenset({"A:1", "B:2"}) + + def test_frozen_immutability(self): + xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + with pytest.raises(AttributeError): + xr.subj = "changed" + + def test_equality(self): + a = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + b = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + assert a == b + + def test_hashability(self): + a = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + b = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + assert hash(a) == hash(b) + assert len({a, b}) == 1 + + def test_lt_ordering(self): + a = CrossReference(filename="a.tsv", subj="A:1", pred="p", obj="B:2") + b = CrossReference(filename="b.tsv", subj="A:1", pred="p", obj="B:2") + assert a < b + + def test_sorting(self): + items = [ + CrossReference(filename="c", subj="C:1", pred="p", obj="D:1"), + CrossReference(filename="a", subj="A:1", pred="p", obj="B:1"), + CrossReference(filename="b", subj="B:1", pred="p", obj="C:1"), + ] + result = sorted(items) + assert [x.filename for x in result] == ["a", "b", "c"] + + +# ========================================================================== +# Unit Tests — LabeledCrossReference +# ========================================================================== + + +class TestLabeledCrossReference: + """Tests for the LabeledCrossReference frozen dataclass.""" + + def test_creation(self): + lxr = LabeledCrossReference( + subj="A:1", + pred="p", + obj="B:2", + filename="f", + subj_label="Alpha", + subj_biolink_type=("biolink:Disease",), + obj_label="Beta", + obj_biolink_type=("biolink:Gene",), + ) + assert lxr.subj == "A:1" + assert lxr.subj_label == "Alpha" + assert lxr.obj_biolink_type == ("biolink:Gene",) + + def test_inherits_from_cross_reference(self): + lxr = LabeledCrossReference( + subj="A:1", + pred="p", + obj="B:2", + filename="f", + subj_label="", + subj_biolink_type=(), + obj_label="", + obj_biolink_type=(), + ) + assert isinstance(lxr, CrossReference) + + def test_curies_property(self): + lxr = LabeledCrossReference( + subj="A:1", + pred="p", + obj="B:2", + filename="f", + subj_label="", + subj_biolink_type=(), + obj_label="", + obj_biolink_type=(), + ) + assert lxr.curies == frozenset({"A:1", "B:2"}) + + def test_str(self): + lxr = LabeledCrossReference( + subj="A:1", + pred="p", + obj="B:2", + filename="f", + subj_label="Alpha", + subj_biolink_type=("biolink:Disease",), + obj_label="Beta", + obj_biolink_type=("biolink:Gene",), + ) + s = str(lxr) + assert "A:1" in s + assert "B:2" in s + assert "Alpha" in s + + +# ========================================================================== +# Unit Tests — IdentifierRecord +# ========================================================================== + + +class TestIdentifierRecord: + """Tests for the IdentifierRecord frozen dataclass.""" + + def test_creation(self): + rec = IdentifierRecord(curie="MONDO:0004979") + assert rec.curie == "MONDO:0004979" + assert rec.extra_fields == () + + def test_from_row(self): + row = ("MONDO:0004979", "Disease", "asthma") + cols = ["curie", "category", "label"] + rec = IdentifierRecord.from_row(row, cols) + assert rec.curie == "MONDO:0004979" + assert ("category", "Disease") in rec.extra_fields + assert ("label", "asthma") in rec.extra_fields + + def test_frozen(self): + rec = IdentifierRecord(curie="X:1") + with pytest.raises(AttributeError): + rec.curie = "changed" + + def test_str(self): + rec = IdentifierRecord(curie="X:1", extra_fields=(("type", "Gene"),)) + s = str(rec) + assert "X:1" in s + assert "type" in s + assert "Gene" in s + + +# ========================================================================== +# Unit Tests — BabelXRefs (mocked) +# ========================================================================== + + +class TestBabelXRefsInit: + """Tests for BabelXRefs constructor.""" + + def test_init_without_nodenorm(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + bx = BabelXRefs(dl) + assert bx.downloader is dl + assert bx.nodenorm is None + + def test_init_with_nodenorm(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + nn = NodeNorm("https://example.com/") + bx = BabelXRefs(dl, nn) + assert bx.nodenorm is nn + + def test_clear_xref_cache_empties_the_cache(self, tmp_path): + """The integration tests reset the cache between queries and must reach it. + + They used to call `get_curie_xref.cache_clear()`, left over from when the + method was decorated with lru_cache. It is a plain method over a per-instance + dict now, so that raised AttributeError — invisible because those tests skip + without a Babel release publishing the Parquet files. + """ + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + bx = BabelXRefs(dl) + bx._xref_cache[("A:1", False)] = [] + + bx.clear_xref_cache() + + assert bx._xref_cache == {} + + def test_connect_spills_inside_the_cache_directory(self, tmp_path): + """DuckDB's default temp_directory is `.tmp` in the *current* directory. + + The recursive expansion materialises a multi-gigabyte Concord relation, so + leaving that at the default drops gigabytes of spill wherever the user ran the + command. + """ + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + with BabelXRefs(dl)._connect() as db: + setting = db.execute("SELECT current_setting('temp_directory')").fetchone() + + assert setting[0] == str(tmp_path / "duckdb-spill") + assert (tmp_path / "duckdb-spill").is_dir() + + +class TestBabelXRefsMocked: + """Mocked query tests — no DuckDB or Parquet files needed.""" + + def _make_bx(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + return BabelXRefs(dl) + + def test_get_curie_xref_calls_downloader(self, tmp_path): + bx = self._make_bx(tmp_path) + mock_result = MagicMock() + mock_result.fetchall.return_value = [ + ("concord.tsv", "A:1", "skos:exactMatch", "B:2"), + ] + mock_db = MagicMock() + mock_db.__enter__.return_value = mock_db + mock_db.read_parquet.return_value = "table" + mock_db.execute.return_value = mock_result + + with patch.object( + bx.downloader, "get_downloaded_file", return_value="/fake/path" + ) as mock_dl: + with patch( + "babel_explorer.core.babel_xrefs.duckdb.connect", + return_value=mock_db, + ): + result = bx.get_curie_xref("A:1") + # Downloader should be called for Concord only (Metadata unused here) + assert mock_dl.call_count == 1 + result_list = list(result) + assert len(result_list) == 1 + assert isinstance(result_list[0], CrossReference) + + def test_get_curie_xrefs_no_expand(self, tmp_path): + bx = self._make_bx(tmp_path) + xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + with patch.object(bx, "_query_xrefs", return_value=[xr]): + result = bx.get_curie_xrefs(["A:1"], recurse=False) + assert len(result) == 1 + assert result[0] == xr + + def test_get_curie_xrefs_with_expand(self, tmp_path): + bx = self._make_bx(tmp_path) + xr1 = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + xr2 = CrossReference(filename="f", subj="B:2", pred="p", obj="C:3") + + with patch.object( + bx, "_get_curie_xrefs_recursive", return_value=[xr1, xr2] + ) as mock_rec: + result = bx.get_curie_xrefs(["A:1"], recurse=True) + mock_rec.assert_called_once_with(["A:1"], False) + assert xr1 in result + assert xr2 in result + + def test_get_curie_xrefs_recursive_sql_traversal(self, tmp_path): + """_get_curie_xrefs_recursive uses SQL graph traversal, not Python recursion.""" + import duckdb as real_duckdb + + bx = self._make_bx(tmp_path) + + # Write a tiny Parquet file: graph A-B, B-C, D-E (disconnected from A-B-C) + parquet_path = str(tmp_path / "test_concord.parquet") + setup_db = real_duckdb.connect() + setup_db.execute(f""" + COPY ( + SELECT * FROM (VALUES + ('f1.tsv', 'A:1', 'skos:exactMatch', 'B:2'), + ('f1.tsv', 'B:2', 'skos:exactMatch', 'C:3'), + ('f2.tsv', 'D:4', 'skos:exactMatch', 'E:5') + ) AS t(filename, subj, pred, obj) + ) TO '{parquet_path}' (FORMAT PARQUET) + """) + setup_db.close() + + with patch.object( + bx.downloader, "get_downloaded_file", return_value=parquet_path + ): + # Starting from A:1 should reach B:2 and C:3 but not the D-E component + result = bx._get_curie_xrefs_recursive(["A:1"]) + pairs = {(xr.subj, xr.obj) for xr in result} + assert ("A:1", "B:2") in pairs + assert ("B:2", "C:3") in pairs + assert ("D:4", "E:5") not in pairs + + # Starting from D:4 should only reach E:5 + result = bx._get_curie_xrefs_recursive(["D:4"]) + pairs = {(xr.subj, xr.obj) for xr in result} + assert ("D:4", "E:5") in pairs + assert ("A:1", "B:2") not in pairs + + # Empty input returns empty list + result = bx._get_curie_xrefs_recursive([]) + assert result == [] + + def test_results_are_sorted(self, tmp_path): + bx = self._make_bx(tmp_path) + xr_b = CrossReference(filename="b", subj="B:1", pred="p", obj="C:1") + xr_a = CrossReference(filename="a", subj="A:1", pred="p", obj="B:1") + + with patch.object(bx, "_query_xrefs", return_value=[xr_b, xr_a]): + result = bx.get_curie_xrefs(["X:1"], recurse=False) + assert result == [xr_a, xr_b] + + +class TestQueryXrefsBatching: + """Multi-CURIE lookups must cost one Parquet scan, not one per CURIE.""" + + @staticmethod + def _mock_db(rows): + result = MagicMock() + result.fetchall.return_value = rows + db = MagicMock() + db.__enter__.return_value = db + db.execute.return_value = result + return db + + def _run(self, tmp_path, curies, rows): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + bx = BabelXRefs(dl) + db = self._mock_db(rows) + with patch.object(bx.downloader, "get_downloaded_file", return_value="/fake"): + with patch( + "babel_explorer.core.babel_xrefs.duckdb.connect", return_value=db + ): + return bx, bx.get_curie_xrefs(curies), db + + def test_one_scan_for_many_curies(self, tmp_path): + rows = [ + ("f", "A:1", "p", "B:2"), + ("f", "C:3", "p", "D:4"), + ] + _, result, db = self._run(tmp_path, ["A:1", "C:3"], rows) + assert db.execute.call_count == 1 + assert len(result) == 2 + + def test_results_are_bucketed_per_curie(self, tmp_path): + rows = [ + ("f", "A:1", "p", "B:2"), + ("f", "C:3", "p", "D:4"), + ] + bx, _, _ = self._run(tmp_path, ["A:1", "C:3"], rows) + assert [x.obj for x in bx._xref_cache[("A:1", False)]] == ["B:2"] + assert [x.obj for x in bx._xref_cache[("C:3", False)]] == ["D:4"] + + def test_curie_with_no_xrefs_caches_an_empty_list(self, tmp_path): + """An absent bucket would silently re-scan the whole Parquet file.""" + bx, result, _ = self._run( + tmp_path, ["A:1", "LONELY:9"], [("f", "A:1", "p", "B:2")] + ) + assert bx._xref_cache[("LONELY:9", False)] == [] + assert len(result) == 1 + + def test_cached_curies_are_not_rescanned(self, tmp_path): + bx, _, db = self._run(tmp_path, ["A:1"], [("f", "A:1", "p", "B:2")]) + with patch.object(bx.downloader, "get_downloaded_file", return_value="/fake"): + with patch( + "babel_explorer.core.babel_xrefs.duckdb.connect", return_value=db + ) as mock_connect: + again = bx.get_curie_xrefs(["A:1"]) + mock_connect.assert_not_called() + assert [x.obj for x in again] == ["B:2"] + + +class TestBuildAdjacency: + """The neighbour map is shared by the depth BFS and the path search.""" + + def test_edges_are_traversable_in_both_directions(self): + xr = CrossReference(filename="f", subj="A:1", pred="p", obj="B:2") + adj = build_adjacency([xr]) + assert adj["A:1"] == [("B:2", xr)] + assert adj["B:2"] == [("A:1", xr)] + + def test_shared_adjacency_matches_a_freshly_built_one(self): + xrefs = [ + CrossReference(filename="f", subj="A:1", pred="p", obj="B:2"), + CrossReference(filename="f", subj="B:2", pred="p", obj="C:3"), + ] + adj = build_adjacency(xrefs) + assert find_shortest_path("A:1", "C:3", xrefs, adj) == find_shortest_path( + "A:1", "C:3", xrefs + ) + assert build_depth_map(["A:1"], xrefs, adj) == build_depth_map(["A:1"], xrefs) + + +# ========================================================================== +# Integration Tests — require downloaded Parquet files +# ========================================================================== + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xref(babel_xrefs, curie): + """get_curie_xref returns non-empty CrossReferences with the queried CURIE.""" + babel_xrefs.clear_xref_cache() + results = list(babel_xrefs.get_curie_xref(curie)) + assert len(results) > 0, f"No cross-references found for {curie}" + for xr in results: + assert isinstance(xr, CrossReference) + assert curie in (xr.subj, xr.obj) + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xref_returns_known_xrefs(babel_xrefs, curie): + """At least one cross-reference is found.""" + babel_xrefs.clear_xref_cache() + results = list(babel_xrefs.get_curie_xref(curie)) + assert len(results) >= 1 + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xrefs_single_no_expand(babel_xrefs, curie): + """get_curie_xrefs without expansion returns sorted, non-empty results.""" + babel_xrefs.clear_xref_cache() + results = babel_xrefs.get_curie_xrefs([curie], recurse=False) + assert len(results) > 0 + assert results == sorted(results) + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xrefs_expansion_finds_more(babel_xrefs, curie): + """Expanded results are at least as many as non-expanded.""" + babel_xrefs.clear_xref_cache() + non_expanded = babel_xrefs.get_curie_xrefs([curie], recurse=False) + babel_xrefs.clear_xref_cache() + expanded = babel_xrefs.get_curie_xrefs([curie], recurse=True) + assert len(expanded) >= len(non_expanded) + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xrefs_expanded_includes_original(babel_xrefs, curie): + """Non-expanded results are a subset of expanded results.""" + babel_xrefs.clear_xref_cache() + non_expanded = set(babel_xrefs.get_curie_xrefs([curie], recurse=False)) + babel_xrefs.clear_xref_cache() + expanded = set(babel_xrefs.get_curie_xrefs([curie], recurse=True)) + assert non_expanded.issubset(expanded) + + +@pytest.mark.integration +def test_get_curie_xref_caching(babel_xrefs): + """Cached calls return the same object.""" + curie = VALID_CURIES[0] + babel_xrefs.clear_xref_cache() + r1 = babel_xrefs.get_curie_xref(curie) + r2 = babel_xrefs.get_curie_xref(curie) + assert r1 is r2 + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_xref_with_labels(babel_xrefs_with_nodenorm, curie): + """With labels, returns LabeledCrossReference objects.""" + babel_xrefs_with_nodenorm.clear_xref_cache() + results = list(babel_xrefs_with_nodenorm.get_curie_xref(curie, label_curies=True)) + assert len(results) > 0 + for xr in results: + assert isinstance(xr, LabeledCrossReference) + + +@pytest.mark.integration +def test_get_curie_xref_nonexistent_curie(babel_xrefs): + """A made-up CURIE returns an empty list.""" + babel_xrefs.clear_xref_cache() + results = list(babel_xrefs.get_curie_xref("FAKE:9999999999")) + assert results == [] + + +@pytest.mark.integration +@pytest.mark.slow +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_curie_ids(babel_xrefs, downloaded_identifiers, curie): + """get_curie_ids returns non-empty IdentifierRecord objects.""" + results = babel_xrefs.get_curie_ids([curie]) + assert len(results) > 0 + for rec in results: + assert isinstance(rec, IdentifierRecord) + assert rec.curie == curie diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..b2fd53b --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,897 @@ +""" +Tests for CLI helper functions. + +Unit tests — no network required. +""" + +import json +import pathlib +import re +from unittest.mock import MagicMock, patch + +import click +import pytest +import requests +from click.testing import CliRunner + +from babel_explorer.cli import cli, parse_duration +from babel_explorer.core.babel_xrefs import CrossReference, IdentifierRecord +from babel_explorer.core.downloader import ( + MissingBabelFileError, + compose_babel_url, +) +from babel_explorer.core.nodenorm import Identifier + +# ========================================================================== +# Unit Tests — no network required +# ========================================================================== + + +class TestParseDuration: + """Tests for parse_duration().""" + + @pytest.mark.parametrize( + "value, expected", + [ + ("never", float("inf")), + ("NEVER", float("inf")), + ("3h", 10800), + ("3H", 10800), + ("30m", 1800), + ("1d", 86400), + ("7200s", 7200), + ("7200", 7200), + ("0", 0), + (" 3h ", 10800), + ], + ) + def test_valid_inputs(self, value, expected): + assert parse_duration(value) == expected + + @pytest.mark.parametrize( + "value", + [ + "", + None, + "abc", + "3.5h", + "1.5", + "3x", + "-5", + "-5h", + ], + ) + def test_invalid_inputs_raise_bad_parameter(self, value): + with pytest.raises(click.BadParameter): + parse_duration(value) + + +class TestCliCommands: + """Tests for CLI commands using CliRunner — no network required.""" + + def test_xrefs_happy_path(self): + runner = CliRunner() + mock_xref = MagicMock() + mock_xref.__str__ = lambda self: "A:1 skos:exactMatch B:2" + mock_xref.subj = "A:1" + mock_xref.obj = "B:2" + mock_xref.pred = "skos:exactMatch" + mock_xref.filename = "test.parquet" + + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [mock_xref] + result = runner.invoke(cli, ["xrefs", "MONDO:0004979"]) + + assert result.exit_code == 0 + mock_bx.return_value.get_curie_xrefs.assert_called_once_with( + ("MONDO:0004979",), False, label_curies=False + ) + + def test_xrefs_recurse_and_labels_flags(self): + runner = CliRunner() + mock_xref = MagicMock() + mock_xref.subj = "A:1" + mock_xref.obj = "B:2" + mock_xref.pred = "skos:exactMatch" + mock_xref.filename = "test.parquet" + + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = "2026jul22" + mock_nn.return_value.get_babel_version.return_value = "2026jul22" + mock_bx.return_value.get_curie_xrefs.return_value = [mock_xref] + result = runner.invoke( + cli, ["xrefs", "MONDO:0004979", "--recurse", "--labels"] + ) + + assert result.exit_code == 0 + mock_bx.return_value.get_curie_xrefs.assert_called_once_with( + ("MONDO:0004979",), True, label_curies=True + ) + + def test_recurse_alone_does_not_consult_nodenorm_for_its_version(self): + """--recurse is served entirely by DuckDB, so a NodeNorm skew is irrelevant.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = "2026jul22" + mock_nn.return_value.get_babel_version.return_value = "2025sep1" + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke(cli, ["xrefs", "MONDO:0004979", "--recurse"]) + + assert result.exit_code == 0, result.output + mock_nn.return_value.get_babel_version.assert_not_called() + + def test_xrefs_check_download_option(self): + runner = CliRunner() + + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke( + cli, ["xrefs", "MONDO:0004979", "--check-download", "1h"] + ) + + assert result.exit_code == 0 + _, kwargs = mock_dl.call_args + assert kwargs.get("freshness_seconds") == 3600 + + def test_ids_happy_path(self): + runner = CliRunner() + mock_id = MagicMock() + mock_id.__str__ = lambda self: "MONDO:0004979 record" + + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + ): + mock_bx.return_value.get_curie_ids.return_value = [mock_id] + result = runner.invoke(cli, ["ids", "MONDO:0004979"]) + + assert result.exit_code == 0 + mock_bx.return_value.get_curie_ids.assert_called_once_with( + ("MONDO:0004979",), label_curies=False + ) + + def test_test_concord_happy_path(self): + runner = CliRunner() + mock_ident = MagicMock() + mock_ident.curie = "MONDO:0004979" + mock_ident.label = "asthma" + mock_ident.biolink_type = ["biolink:Disease"] + + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] + result = runner.invoke(cli, ["test-concord", "MONDO:0004979"]) + + assert result.exit_code == 0 + assert "asthma" in result.output + mock_nn.return_value.get_clique_identifiers.assert_called_once_with( + "MONDO:0004979" + ) + + def test_test_concord_no_label(self): + runner = CliRunner() + mock_ident = MagicMock() + mock_ident.curie = "MONDO:0004979" + mock_ident.label = None + mock_ident.biolink_type = ["biolink:Disease"] + + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] + result = runner.invoke(cli, ["test-concord", "MONDO:0004979"]) + + assert result.exit_code == 0 + assert "MONDO:0004979" in result.output + assert "biolink:Disease" in result.output + + def test_test_concord_unknown_curie_produces_no_output(self): + """When get_clique_identifiers returns [], no output is produced and exit code is 0.""" + runner = CliRunner() + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [] + result = runner.invoke(cli, ["test-concord", "UNKNOWN:9999"]) + assert result.exit_code == 0 + assert result.output.strip() == "" + + def test_test_concord_multiple_curies(self): + """Each CURIE is looked up independently.""" + runner = CliRunner() + mock_a = MagicMock() + mock_a.curie = "A:1" + mock_a.label = "Alpha" + mock_a.biolink_type = ["biolink:Disease"] + mock_b = MagicMock() + mock_b.curie = "B:2" + mock_b.label = "Beta" + mock_b.biolink_type = ["biolink:Gene"] + + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.side_effect = [ + [mock_a], + [mock_b], + ] + result = runner.invoke(cli, ["test-concord", "A:1", "B:2"]) + + assert result.exit_code == 0 + assert mock_nn.return_value.get_clique_identifiers.call_count == 2 + assert "Alpha" in result.output + assert "Beta" in result.output + + +class TestOutputFormats: + """Tests for --format option on all commands.""" + + # Shared real dataclass instances (no mocking needed for formatting logic) + _xref = CrossReference( + filename="Concord.parquet", subj="A:1", pred="skos:exactMatch", obj="B:2" + ) + _id_record = IdentifierRecord( + curie="A:1", extra_fields=(("type", "gene"), ("label", "Alpha")) + ) + _identifier = Identifier( + curie="MONDO:0004979", + label="asthma", + biolink_type=("biolink:Disease",), + taxa=(), + description=(), + ) + + # -- console format (default) -- + + def test_xrefs_default_format_is_console(self): + """Default format is console — output contains the CURIEs as plain text (no TTY in runner).""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1"]) + + assert result.exit_code == 0 + # Rich strips markup on non-TTY; plain CURIEs and predicate appear + assert "A:1" in result.output + assert "B:2" in result.output + assert "skos:exactMatch" in result.output + + def test_xrefs_console_shows_query_curie(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "console"]) + + assert result.exit_code == 0 + assert "A:1" in result.output + + def test_test_concord_console_format(self): + runner = CliRunner() + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [ + self._identifier + ] + result = runner.invoke( + cli, ["test-concord", "MONDO:0004979", "--format", "console"] + ) + + assert result.exit_code == 0 + assert "MONDO:0004979" in result.output + assert "asthma" in result.output + assert "biolink:Disease" in result.output + + def test_test_concord_console_no_label_omits_label(self): + """Identifiers with no label omit the label field entirely in console format.""" + runner = CliRunner() + mock_ident = MagicMock() + mock_ident.curie = "MONDO:0004979" + mock_ident.label = None + mock_ident.biolink_type = ["biolink:Disease"] + + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [mock_ident] + result = runner.invoke( + cli, ["test-concord", "MONDO:0004979", "--format", "console"] + ) + + assert result.exit_code == 0 + assert '"' not in result.output + + # -- json format -- + + def test_xrefs_format_json(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "json"]) + + assert result.exit_code == 0 + data = json.loads(result.output) + assert isinstance(data, list) + assert data[0]["subj"] == "A:1" + assert data[0]["obj"] == "B:2" + + def test_xrefs_format_tsv(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "tsv"]) + + assert result.exit_code == 0 + lines = result.output.splitlines() + assert lines[0] == "filename\tsubj\tpred\tobj" + assert "A:1" in lines[1] + + def test_xrefs_format_csv(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [self._xref] + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "csv"]) + + assert result.exit_code == 0 + lines = result.output.splitlines() + assert lines[0] == "filename,subj,pred,obj" + assert "A:1" in lines[1] + + # -- ids -- + + def test_ids_format_json_expands_extra_fields(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + ): + mock_bx.return_value.get_curie_ids.return_value = [self._id_record] + result = runner.invoke(cli, ["ids", "A:1", "--format", "json"]) + + assert result.exit_code == 0 + data = json.loads(result.output) + assert data[0]["curie"] == "A:1" + assert data[0]["type"] == "gene" + assert data[0]["label"] == "Alpha" + assert "extra_fields" not in data[0] + + def test_ids_format_tsv_expands_extra_fields(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + ): + mock_bx.return_value.get_curie_ids.return_value = [self._id_record] + result = runner.invoke(cli, ["ids", "A:1", "--format", "tsv"]) + + assert result.exit_code == 0 + lines = result.output.splitlines() + assert "type" in lines[0] + assert "label" in lines[0] + assert "gene" in lines[1] + + # -- test-concord structured formats -- + + def test_test_concord_format_json_includes_query_curie(self): + runner = CliRunner() + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [ + self._identifier + ] + result = runner.invoke( + cli, ["test-concord", "MONDO:0004979", "--format", "json"] + ) + + assert result.exit_code == 0 + data = json.loads(result.output) + assert data[0]["query_curie"] == "MONDO:0004979" + assert data[0]["curie"] == "MONDO:0004979" + assert data[0]["label"] == "asthma" + assert data[0]["biolink_type"] == ["biolink:Disease"] + + def test_test_concord_format_tsv(self): + runner = CliRunner() + with patch("babel_explorer.cli.NodeNorm") as mock_nn: + mock_nn.return_value.get_clique_identifiers.return_value = [ + self._identifier + ] + result = runner.invoke( + cli, ["test-concord", "MONDO:0004979", "--format", "tsv"] + ) + + assert result.exit_code == 0 + lines = result.output.splitlines() + assert "query_curie" in lines[0] + assert "MONDO:0004979" in lines[1] + + # -- format validation -- + + def test_invalid_format_rejected_by_click(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs"), + patch("babel_explorer.cli.NodeNorm"), + ): + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "xml"]) + + assert result.exit_code != 0 + + def test_text_format_rejected_by_click(self): + """'text' was removed; it is no longer a valid choice.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs"), + patch("babel_explorer.cli.NodeNorm"), + ): + result = runner.invoke(cli, ["xrefs", "A:1", "--format", "text"]) + + assert result.exit_code != 0 + + +class TestVersionChecking: + """The Babel release behind --babel-url must match the one NodeNorm was built from.""" + + @staticmethod + def _run(args, babel_version="2026jul22", nodenorm_version="2026jul22", env=None): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = babel_version + mock_nn.return_value.get_babel_version.return_value = nodenorm_version + mock_bx.return_value.get_curie_xrefs.return_value = [] + mock_bx.return_value.get_curie_ids.return_value = [] + result = runner.invoke(cli, args, env=env) + return result, mock_dl, mock_nn + + def test_mismatch_fails(self): + result, _, _ = self._run( + ["xrefs", "A:1", "--labels"], nodenorm_version="2025sep1" + ) + assert result.exit_code != 0 + assert "2025sep1" in result.output and "2026jul22" in result.output + + def test_mismatch_allowed_with_flag(self): + result, _, _ = self._run( + ["xrefs", "A:1", "--labels", "--allow-version-mismatch"], + nodenorm_version="2025sep1", + ) + assert result.exit_code == 0 + + def test_mismatch_allowed_via_env(self): + result, _, _ = self._run( + ["xrefs", "A:1", "--labels"], + nodenorm_version="2025sep1", + env={"BABEL_ALLOW_VERSION_MISMATCH": "1"}, + ) + assert result.exit_code == 0 + + def test_unknown_version_skips_check(self): + """Nothing to compare means nothing to complain about.""" + result, _, _ = self._run(["xrefs", "A:1", "--labels"], babel_version=None) + assert result.exit_code == 0 + + def test_plain_xrefs_skips_check(self): + """Plain xrefs builds a NodeNorm but never queries it, so skew is irrelevant.""" + result, _, mock_nn = self._run(["xrefs", "A:1"], nodenorm_version="2025sep1") + assert result.exit_code == 0 + mock_nn.return_value.get_babel_version.assert_not_called() + + def test_ids_skips_check(self): + """ids uses no NodeNorm at all.""" + result, _, mock_nn = self._run(["ids", "A:1"], nodenorm_version="2025sep1") + assert result.exit_code == 0 + mock_nn.return_value.get_babel_version.assert_not_called() + + def test_cache_is_synced_to_the_babel_release(self): + _, mock_dl, _ = self._run(["ids", "A:1"]) + mock_dl.return_value.sync_cache_version.assert_called_once() + + +class TestUrlConfiguration: + """URLs come from the environment (and hence .env), overridable per-run.""" + + @staticmethod + def _invoke(args, env): + runner = CliRunner() + with ( + # load_dotenv() runs inside cli(), i.e. after CliRunner(env=...) has cleared a + # variable and before Click reads envvars — so a real .env would leak into + # these assertions. Every Translator developer is about to have + # BABEL_RELEASES_URL in theirs, so neutralise it rather than hope. + patch("babel_explorer.cli.load_dotenv"), + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke(cli, args, env=env) + assert result.exit_code == 0, result.output + return mock_dl, mock_nn + + def test_defaults_are_public(self): + mock_dl, mock_nn = self._invoke( + ["xrefs", "A:1"], + env={ + "BABEL_RELEASES_URL": None, + "BABEL_VERSION": None, + "NODENORM_URL": None, + }, + ) + assert mock_dl.call_args[0][0] == "https://stars.renci.org/var/babel/latest/" + assert mock_nn.call_args[0][0] == "https://nodenormalization-sri.renci.org/" + + def test_env_overrides_defaults(self): + mock_dl, mock_nn = self._invoke( + ["xrefs", "A:1"], + env={ + "BABEL_RELEASES_URL": "https://example.com/babel/", + "BABEL_VERSION": "2025dec11", + "BABEL_LOCAL_DIR": "/tmp/babel-cache", + "NODENORM_URL": "https://example.com/nn/", + }, + ) + assert mock_dl.call_args[0][0] == "https://example.com/babel/2025dec11/" + assert mock_dl.call_args.kwargs["local_path"] == "/tmp/babel-cache" + assert mock_nn.call_args[0][0] == "https://example.com/nn/" + + def test_releases_url_without_trailing_slash_still_composes(self): + mock_dl, _ = self._invoke( + ["xrefs", "A:1"], + env={ + "BABEL_RELEASES_URL": "https://example.com/babel", + "BABEL_VERSION": "2025dec11", + }, + ) + assert mock_dl.call_args[0][0] == "https://example.com/babel/2025dec11/" + + def test_flag_beats_env(self): + mock_dl, _ = self._invoke( + ["xrefs", "A:1", "--babel-url", "https://flag.example.com/"], + env={ + "BABEL_RELEASES_URL": "https://env.example.com/", + "BABEL_VERSION": "2025dec11", + }, + ) + assert mock_dl.call_args[0][0] == "https://flag.example.com/" + + def test_babel_version_flag_beats_env(self): + mock_dl, _ = self._invoke( + ["xrefs", "A:1", "--babel-version", "2026jul22"], + env={ + "BABEL_RELEASES_URL": "https://example.com/babel/", + "BABEL_VERSION": "2025dec11", + }, + ) + assert mock_dl.call_args[0][0] == "https://example.com/babel/2026jul22/" + + def test_babel_url_has_no_environment_variable(self): + """The design decision, pinned: BABEL_URL in the environment does nothing. + + Two variables already feed the composed URL; a third that silently outranked + both would make the effective release unreadable from the environment alone. + """ + mock_dl, _ = self._invoke( + ["xrefs", "A:1"], + env={ + "BABEL_URL": "https://ignored.example.com/", + "BABEL_RELEASES_URL": None, + "BABEL_VERSION": None, + }, + ) + assert mock_dl.call_args[0][0] == "https://stars.renci.org/var/babel/latest/" + + def test_stale_babel_url_warns(self): + """Ignoring it silently would send someone to the wrong release with no clue.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.load_dotenv"), + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke( + cli, ["xrefs", "A:1"], env={"BABEL_URL": "https://stale.example.com/"} + ) + assert result.exit_code == 0, result.output + assert "BABEL_URL is no longer used" in result.output + + def test_babel_url_with_typed_version_warns(self): + mock_dl, _ = self._invoke( + [ + "xrefs", + "A:1", + "--babel-url", + "https://flag.example.com/", + "--babel-version", + "2026jul22", + ], + env={}, + ) + assert mock_dl.call_args[0][0] == "https://flag.example.com/" + + def test_babel_url_with_env_version_is_silent(self): + """Warning on an env-supplied version would fire on every --babel-url run.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.load_dotenv"), + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke( + cli, + ["xrefs", "A:1", "--babel-url", "https://flag.example.com/"], + env={"BABEL_VERSION": "2026jul22"}, + ) + assert result.exit_code == 0, result.output + assert "is ignored" not in result.output + + @pytest.mark.parametrize( + "bad", ["https://example.com/babel/latest/", "../../etc/passwd"] + ) + def test_babel_version_rejects_urls_and_traversal(self, bad): + runner = CliRunner() + with patch("babel_explorer.cli.load_dotenv"): + result = runner.invoke(cli, ["xrefs", "A:1", "--babel-version", bad]) + assert result.exit_code != 0 + assert "--babel-version" in result.output or "may not contain" in result.output + + +class TestCommittedConfigTemplate: + """env.default is the only config file that ships, so it is the one that can leak. + + CLAUDE.md and README.md both say the Translator-specific URL must never be committed. + Until now nothing enforced it, and the URL did in fact sit in this repository's git + history from the initial commit until it was rewritten out on 2026-09-01. A rule with + no test is a rule that comes back. + """ + + TEMPLATE = pathlib.Path(__file__).resolve().parent.parent / "env.default" + + def _settings(self) -> dict[str, str]: + settings = {} + for line in self.TEMPLATE.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, _, value = line.partition("=") + settings[key.strip()] = value.strip().strip("\"'") + return settings + + def test_template_is_the_committed_one(self): + """.env.example was renamed; nothing should resurrect it alongside env.default.""" + assert self.TEMPLATE.is_file() + assert not (self.TEMPLATE.parent / ".env.example").exists() + + @staticmethod + def _envvars_the_cli_reads() -> set[str]: + """Every envvar= declared on any command's options, read off the CLI itself. + + Derived rather than listed: a hard-coded set passes just as happily when a new + setting is added to the CLI and forgotten here, which is how + BABEL_ALLOW_VERSION_MISMATCH went undocumented in the template. + """ + return { + param.envvar + for command in cli.commands.values() + for param in command.params + if getattr(param, "envvar", None) + } + + def test_documents_every_setting_the_cli_reads(self): + """A setting the CLI honours but the template omits is one nobody discovers.""" + assert set(self._settings()) == self._envvars_the_cli_reads() + + def test_defaults_match_the_cli_defaults(self): + """A template that disagrees with the code silently changes what `cp` gives you.""" + settings = self._settings() + assert settings["BABEL_RELEASES_URL"] == "https://stars.renci.org/var/babel/" + assert settings["BABEL_VERSION"] == "latest" + assert ( + compose_babel_url(settings["BABEL_RELEASES_URL"], settings["BABEL_VERSION"]) + == "https://stars.renci.org/var/babel/latest/" + ) + + def test_defines_no_babel_url(self): + """BABEL_URL is inert. Shipping it would send people to a setting that does nothing.""" + assert "BABEL_URL" not in self._settings() + + def test_carries_no_non_public_url(self): + """The guard that matters: only public hosts, and never the internal outputs tree.""" + text = self.TEMPLATE.read_text() + for host in re.findall(r"https?://([^/\s\"']+)", text): + assert host in { + "stars.renci.org", + "nodenormalization-sri.renci.org", + }, f"{host} is not a public endpoint" + for path in re.findall(r"https?://\S+", text): + assert "/var/babel/" in path or "nodenormalization" in path, ( + f"{path} is not the public Babel or NodeNorm endpoint" + ) + + +class TestMissingBabelFileReporting: + """A missing Parquet file should read as an error, not a traceback.""" + + def test_reported_without_traceback(self): + runner = CliRunner() + message = ( + "This Babel release (2025dec11) does not publish duckdb/Concord.parquet." + ) + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm"), + ): + mock_bx.return_value.get_curie_xrefs.side_effect = MissingBabelFileError( + message + ) + result = runner.invoke(cli, ["xrefs", "A:1"]) + + assert result.exit_code == 1 + assert message in result.output + assert "Traceback" not in result.output + assert isinstance(result.exception, SystemExit) + + def test_also_wrapped_for_ids(self): + """The conversion lives on the group, so every command inherits it.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader"), + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + ): + mock_bx.return_value.get_curie_ids.side_effect = MissingBabelFileError( + "nope" + ) + result = runner.invoke(cli, ["ids", "A:1"]) + + assert result.exit_code == 1 + assert "nope" in result.output + assert "Traceback" not in result.output + + +class TestNodeNormFailureIsNotATraceback: + """An unreachable NodeNorm must not end the run in a Python stack trace.""" + + def test_connection_error_becomes_a_click_error(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = "2026jul22" + # get_babel_version() swallows its own errors, so the version check passes + # and the failure only surfaces once the query is under way. + mock_nn.return_value.get_babel_version.return_value = None + mock_bx.return_value.get_curie_xrefs.side_effect = requests.ConnectionError( + "connection refused" + ) + result = runner.invoke(cli, ["xrefs", "A:1", "--labels"]) + + assert result.exit_code == 1 + assert "NodeNorm request failed" in result.output + assert "connection refused" in result.output + assert "Traceback" not in result.output + + +class TestPathsFormatGuard: + """--paths only has a renderer for the console format.""" + + @pytest.mark.parametrize("fmt", ["json", "tsv", "csv"]) + def test_rejected_for_non_console_formats(self, fmt): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs"), + patch("babel_explorer.cli.NodeNorm"), + ): + result = runner.invoke( + cli, ["xrefs", "A:1", "B:2", "--paths", "--format", fmt] + ) + + assert result.exit_code != 0 + assert "--paths is only supported with --format console" in result.output + # Rejected before anything is downloaded. + mock_dl.assert_not_called() + + def test_single_curie_rejected_before_downloading(self): + """--paths implies --recurse, so finding out late costs a 4.6 GB download.""" + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs"), + patch("babel_explorer.cli.NodeNorm"), + ): + result = runner.invoke(cli, ["xrefs", "A:1", "--paths"]) + + assert result.exit_code != 0 + assert "--paths needs at least two CURIEs" in result.output + mock_dl.assert_not_called() + + def test_allowed_for_console(self): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = "2026jul22" + mock_nn.return_value.get_babel_version.return_value = "2026jul22" + mock_bx.return_value.get_curie_xrefs.return_value = [] + result = runner.invoke(cli, ["xrefs", "A:1", "B:2", "--paths"]) + + assert result.exit_code == 0 + + +class TestIdsLabels: + """`ids --labels` enriches records via NodeNorm.""" + + @staticmethod + def _run(args, babel_version="2026jul22", nodenorm_version="2026jul22"): + runner = CliRunner() + with ( + patch("babel_explorer.cli.BabelDownloader") as mock_dl, + patch("babel_explorer.cli.BabelXRefs") as mock_bx, + patch("babel_explorer.cli.NodeNorm") as mock_nn, + ): + mock_dl.return_value.babel_version = babel_version + mock_nn.return_value.get_babel_version.return_value = nodenorm_version + mock_bx.return_value.get_curie_ids.return_value = [ + IdentifierRecord(curie="MONDO:0004979", nodenorm_label="asthma") + ] + result = runner.invoke(cli, args) + return result, mock_bx, mock_nn + + def test_labels_flag_is_passed_through(self): + result, mock_bx, _ = self._run(["ids", "MONDO:0004979", "--labels"]) + assert result.exit_code == 0 + mock_bx.return_value.get_curie_ids.assert_called_once_with( + ("MONDO:0004979",), label_curies=True + ) + + def test_label_rendered_in_double_quotes(self): + result, _, _ = self._run(["ids", "MONDO:0004979", "--labels"]) + assert '"asthma"' in result.output + + def test_version_checked_only_with_labels(self): + _, _, mock_nn = self._run(["ids", "MONDO:0004979"], nodenorm_version="2025sep1") + mock_nn.return_value.get_babel_version.assert_not_called() + + def test_version_mismatch_fails_with_labels(self): + result, _, _ = self._run( + ["ids", "MONDO:0004979", "--labels"], nodenorm_version="2025sep1" + ) + assert result.exit_code != 0 + assert "2025sep1" in result.output diff --git a/tests/test_downloader.py b/tests/test_downloader.py new file mode 100644 index 0000000..eb2aee5 --- /dev/null +++ b/tests/test_downloader.py @@ -0,0 +1,1285 @@ +""" +Tests for the BabelDownloader class. + +Unit tests use mocks and run without network access. +Integration tests download real files from the Babel server. +""" + +import json +import os +import tempfile +from datetime import UTC, datetime, timedelta +from unittest.mock import MagicMock, Mock, patch + +import pytest +import requests + +from babel_explorer.core.downloader import ( + VERSION_MARKER, + BabelDownloader, + IncompleteDownloadError, + MissingBabelFileError, + compose_babel_url, + resolve_babel_version, +) +from tests.constants import BABEL_URL, CONCORD_FILE + + +def test_babel_url_is_normalised_for_direct_path_joins(): + """The env-driven URL must end in "/" too, or the skip probe requests + ".../latestduckdb/Concord.parquet", 404s, and silently skips every integration test.""" + assert BABEL_URL.endswith("/") + + +class TestComposeBabelUrl: + """Composition has to absorb the slashes users will and will not type.""" + + @pytest.mark.parametrize( + "releases, version, expected", + [ + ("https://ex.com/babel/", "latest", "https://ex.com/babel/latest/"), + ("https://ex.com/babel", "latest", "https://ex.com/babel/latest/"), + ("https://ex.com/babel//", "2025dec11", "https://ex.com/babel/2025dec11/"), + ("https://ex.com/babel", "/2025dec11/", "https://ex.com/babel/2025dec11/"), + (" https://ex.com/babel ", "latest", "https://ex.com/babel/latest/"), + ( + "https://ex.com/babel", + " 2025dec11 ", + "https://ex.com/babel/2025dec11/", + ), + ], + ) + def test_normalisation(self, releases, version, expected): + assert compose_babel_url(releases, version) == expected + + def test_public_default_composes_to_the_historical_url(self): + """The default pair must reproduce the single URL this option pair replaced.""" + assert ( + compose_babel_url("https://stars.renci.org/var/babel/", "latest") + == "https://stars.renci.org/var/babel/latest/" + ) + + +def _version_response(text): + """A mock requests response serving *text* as the body of VERSION.txt.""" + response = Mock() + response.text = text + response.raise_for_status = Mock() + return response + + +class TestResolveBabelVersion: + """Unit tests for resolve_babel_version().""" + + def test_reads_version_txt(self): + with patch( + "babel_explorer.core.downloader.requests.get", + return_value=_version_response( + "Babel 2026jul22\nhttps://github.com/NCATSTranslator/Babel\n" + ), + ) as mock_get: + assert ( + resolve_babel_version("https://example.com/babel/latest/") + == "2026jul22" + ) + assert ( + mock_get.call_args[0][0] == "https://example.com/babel/latest/VERSION.txt" + ) + + def test_version_txt_wins_over_path_segment(self): + """VERSION.txt is authoritative even when the URL names a version.""" + with patch( + "babel_explorer.core.downloader.requests.get", + return_value=_version_response("Babel 2026jul22\n"), + ): + assert ( + resolve_babel_version("https://example.com/babel/2025nov19/") + == "2026jul22" + ) + + def test_falls_back_to_path_segment(self): + """Trees predating VERSION.txt fall back to the final path segment.""" + with patch( + "babel_explorer.core.downloader.requests.get", + side_effect=requests.HTTPError("404"), + ): + assert ( + resolve_babel_version("https://example.com/babel/2025nov19/") + == "2025nov19" + ) + + def test_unresolvable_latest_returns_none(self): + """'latest' is not a version, so an unreachable VERSION.txt means unknown.""" + with patch( + "babel_explorer.core.downloader.requests.get", + side_effect=requests.ConnectionError("boom"), + ): + assert resolve_babel_version("https://example.com/babel/latest/") is None + + def test_unparseable_version_txt_falls_back(self): + with patch( + "babel_explorer.core.downloader.requests.get", + return_value=_version_response("something else entirely"), + ): + assert ( + resolve_babel_version("https://example.com/babel/2025nov19/") + == "2025nov19" + ) + + +class TestSyncCacheVersion: + """Unit tests for BabelDownloader.sync_cache_version().""" + + @staticmethod + def _downloader(tmp_path, version): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + dl.babel_version = version + return dl + + @staticmethod + def _seed_cache(tmp_path): + """Create a cached parquet file with its .meta sidecar.""" + duckdb_dir = tmp_path / "duckdb" + duckdb_dir.mkdir() + parquet = duckdb_dir / "Concord.parquet" + parquet.write_text("data") + meta = duckdb_dir / "Concord.parquet.meta" + meta.write_text( + json.dumps({"etag": '"abc"', "last_checked": "2026-07-22T00:00:00+00:00"}) + ) + return parquet, meta + + def test_writes_marker_when_absent(self, tmp_path): + self._downloader(tmp_path, "2026jul22").sync_cache_version() + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2026jul22" + + def test_matching_version_keeps_meta(self, tmp_path): + _, meta = self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2026jul22\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert meta.exists() + assert "last_checked" in json.loads(meta.read_text()) + + def test_changed_version_expires_meta_but_keeps_etag_and_parquet(self, tmp_path): + """The ETag must survive so the refresh costs a HEAD, not a full re-download.""" + parquet, meta = self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + remaining = json.loads(meta.read_text()) + assert "last_checked" not in remaining, "sidecar should no longer look fresh" + assert remaining["etag"] == '"abc"', ( + "dropping the ETag would force an unconditional multi-gigabyte re-download" + ) + assert parquet.exists(), "the Parquet file itself must never be deleted" + + def test_changed_version_leaves_marker_until_the_cache_catches_up(self, tmp_path): + """A marker written up front makes an interrupted refresh look complete. + + If the marker named the new release straight away and the run died after + Concord was refreshed but before Identifiers was, the next run would see a + matching marker, skip the version-driven refresh entirely, and read the two + Parquet files together across two Babel releases. + """ + self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2025nov19" + + def test_marker_written_once_every_sidecar_is_revalidated(self, tmp_path): + _, meta = self._seed_cache(tmp_path) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + dl = self._downloader(tmp_path, "2026jul22") + dl.sync_cache_version() + + # What a confirmed-unchanged HEAD or a completed download leaves behind. + dl._write_meta(str(meta).removesuffix(".meta"), json.loads(meta.read_text())) + dl._write_version_marker_if_synced() + + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2026jul22" + + def test_marker_withheld_while_one_cached_file_is_still_stale(self, tmp_path): + """Every cached file must be re-validated, not just the one that was asked for.""" + _, meta = self._seed_cache(tmp_path) + other = tmp_path / "duckdb" / "Identifiers.parquet.meta" + other.write_text( + json.dumps({"etag": '"def"', "last_checked": "2026-07-22T00:00:00+00:00"}) + ) + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + dl = self._downloader(tmp_path, "2026jul22") + dl.sync_cache_version() + + dl._write_meta(str(meta).removesuffix(".meta"), json.loads(meta.read_text())) + dl._write_version_marker_if_synced() + + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2025nov19" + + def test_changed_version_removes_partial_downloads(self, tmp_path): + """A .tmp from the previous release must not be resumed against the new one.""" + self._seed_cache(tmp_path) + partial = tmp_path / "duckdb" / "Concord.parquet.tmp" + partial.write_text("half of the previous release") + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert not partial.exists() + + def test_changed_version_drops_unreadable_meta(self, tmp_path): + _, meta = self._seed_cache(tmp_path) + meta.write_text("not json") + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert not meta.exists() + + def test_refresh_does_not_reach_into_sibling_directories(self, tmp_path): + """local_path may hold other Babel releases; only our own duckdb/ is cleared.""" + self._seed_cache(tmp_path) + sibling = tmp_path / "2025nov19" / "duckdb" + sibling.mkdir(parents=True) + sibling_meta = sibling / "Concord.parquet.meta" + sibling_meta.write_text("{}") + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, "2026jul22").sync_cache_version() + + assert sibling_meta.exists(), ( + "a nested release directory must not be swept up in the refresh" + ) + + def test_unknown_version_leaves_cache_untouched(self, tmp_path): + """An unresolvable version must not trigger a multi-gigabyte re-download.""" + _, meta = self._seed_cache(tmp_path) + before = meta.read_text() + (tmp_path / VERSION_MARKER).write_text("2025nov19\n") + + self._downloader(tmp_path, None).sync_cache_version() + + assert meta.read_text() == before + assert (tmp_path / VERSION_MARKER).read_text().strip() == "2025nov19" + + +class TestMissingBabelFile: + """A 404 should explain itself and not be retried.""" + + def test_404_raises_immediately(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=10 + ) + dl.babel_version = "2025dec11" + + response = MagicMock() + response.status_code = 404 + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + + with patch( + "babel_explorer.core.downloader.requests.get", return_value=response + ) as mock_get: + with pytest.raises(MissingBabelFileError, match="2025dec11"): + dl.get_downloaded_file(CONCORD_FILE) + + assert mock_get.call_count == 1, "a 404 must not be retried" + + def test_404_message_names_the_current_setting(self, tmp_path): + """This message is where most people learn the config scheme exists. + + It named BABEL_URL for as long as that variable did; nothing caught the + wording when the variable was replaced. Pin it to the setting that works. + """ + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + # Preset so the patched requests.get is not also asked to resolve VERSION.txt. + dl.babel_version = "2025dec11" + response = MagicMock(status_code=404) + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + + with patch( + "babel_explorer.core.downloader.requests.get", return_value=response + ): + with pytest.raises(MissingBabelFileError) as excinfo: + dl.get_downloaded_file(CONCORD_FILE) + + message = str(excinfo.value) + assert "BABEL_RELEASES_URL" in message + assert "--babel-url" in message + assert "set BABEL_URL" not in message + + +# ========================================================================== +# Unit Tests — no network required +# ========================================================================== + + +class TestBabelDownloaderInit: + """Tests for BabelDownloader constructor.""" + + def test_constructor_stores_url_and_path(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + assert dl.url_base == "https://example.com/" + assert dl.local_path == str(tmp_path) + + def test_creates_directory_if_missing(self, tmp_path): + new_dir = str(tmp_path / "nested" / "dir") + dl = BabelDownloader(url_base="https://example.com/", local_path=new_dir) + assert os.path.isdir(new_dir) + assert dl.local_path == new_dir + + def test_custom_retries(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + assert dl.retries == 3 + + def test_default_retries(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + assert dl.retries == 10 + + def test_default_freshness_seconds(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + assert dl.freshness_seconds == 3 * 3600 + + def test_custom_freshness_seconds(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", + local_path=str(tmp_path), + freshness_seconds=0, + ) + assert dl.freshness_seconds == 0 + + def test_url_base_trailing_slash_added(self, tmp_path): + """url_base without trailing slash gets one appended automatically.""" + dl = BabelDownloader( + url_base="https://example.com/path", local_path=str(tmp_path) + ) + assert dl.url_base == "https://example.com/path/" + + def test_url_base_with_trailing_slash_unchanged(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/path/", local_path=str(tmp_path) + ) + assert dl.url_base == "https://example.com/path/" + + def test_invalid_path_raises_value_error(self): + """Using a file path (not a directory) should raise ValueError.""" + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(b"not a directory") + f.flush() + try: + with pytest.raises(ValueError, match="Invalid local_path"): + BabelDownloader(url_base="https://example.com/", local_path=f.name) + finally: + os.unlink(f.name) + + +class TestSaveMeta: + """Tests for _save_meta.""" + + def _make_dl(self, tmp_path): + return BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path) + ) + + def test_writes_all_fields(self, tmp_path): + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "test.parquet") + # Create the file so the path is valid + open(file_path, "wb").close() + + headers = { + "ETag": '"abc123"', + "Last-Modified": "Wed, 03 Dec 2025 15:54:19 GMT", + "Content-Length": "12345", + } + dl._save_meta(file_path, headers) + + meta_path = file_path + ".meta" + assert os.path.exists(meta_path) + with open(meta_path) as f: + meta = json.load(f) + + assert meta["etag"] == '"abc123"' + assert meta["last_modified"] == "Wed, 03 Dec 2025 15:54:19 GMT" + assert meta["content_length"] == 12345 + assert "last_checked" in meta + + def test_last_checked_is_recent_utc(self, tmp_path): + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "f.parquet") + open(file_path, "wb").close() + + dl._save_meta(file_path, {"ETag": '"x"'}) + + with open(file_path + ".meta") as f: + meta = json.load(f) + + last_checked = datetime.fromisoformat(meta["last_checked"]) + age = (datetime.now(UTC) - last_checked).total_seconds() + assert age < 5 # written less than 5 seconds ago + + def test_missing_headers_not_written(self, tmp_path): + """Headers not present in the response should not appear in .meta.""" + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "sparse.parquet") + open(file_path, "wb").close() + + dl._save_meta(file_path, {}) + + with open(file_path + ".meta") as f: + meta = json.load(f) + + assert "etag" not in meta + assert "last_modified" not in meta + assert "content_length" not in meta + assert "last_checked" in meta + + +class TestLoadMeta: + """Tests for _load_meta.""" + + def _make_dl(self, tmp_path): + return BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path) + ) + + def test_returns_none_if_no_meta_file(self, tmp_path): + dl = self._make_dl(tmp_path) + assert dl._load_meta(str(tmp_path / "nonexistent.parquet")) is None + + def test_returns_dict_for_valid_meta(self, tmp_path): + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "f.parquet") + open(file_path, "wb").close() + meta_data = {"etag": '"abc"', "last_checked": "2026-01-01T00:00:00+00:00"} + with open(file_path + ".meta", "w") as f: + json.dump(meta_data, f) + + result = dl._load_meta(file_path) + assert result == meta_data + + def test_returns_none_for_corrupt_meta(self, tmp_path): + dl = self._make_dl(tmp_path) + file_path = str(tmp_path / "corrupt.parquet") + open(file_path, "wb").close() + with open(file_path + ".meta", "w") as f: + f.write("not valid json {{{") + + assert dl._load_meta(file_path) is None + + +class TestIsWithinFreshness: + """Tests for _is_within_freshness.""" + + def _make_dl(self, tmp_path): + return BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path) + ) + + def test_returns_true_when_recent(self, tmp_path): + dl = self._make_dl(tmp_path) + recent = datetime.now(UTC).isoformat() + meta = {"last_checked": recent} + assert dl._is_within_freshness(meta, 3600) is True + + def test_returns_false_when_stale(self, tmp_path): + dl = self._make_dl(tmp_path) + old = (datetime.now(UTC) - timedelta(hours=5)).isoformat() + meta = {"last_checked": old} + assert dl._is_within_freshness(meta, 3600) is False + + def test_returns_false_when_missing_last_checked(self, tmp_path): + dl = self._make_dl(tmp_path) + assert dl._is_within_freshness({}, 3600) is False + + def test_returns_true_when_freshness_is_inf(self, tmp_path): + dl = self._make_dl(tmp_path) + old = (datetime.now(UTC) - timedelta(days=365)).isoformat() + meta = {"last_checked": old} + assert dl._is_within_freshness(meta, float("inf")) is True + + def test_returns_false_when_missing_last_checked_even_if_inf(self, tmp_path): + """`--check-download never` must not resurrect a sidecar the version change expired. + + sync_cache_version clears last_checked to force a re-check. If float('inf') + short-circuited ahead of that test, `never` would return the previous release's + Parquet with no network call at all. + """ + dl = self._make_dl(tmp_path) + assert dl._is_within_freshness({"etag": '"old"'}, float("inf")) is False + + def test_returns_false_when_freshness_is_zero(self, tmp_path): + dl = self._make_dl(tmp_path) + just_now = datetime.now(UTC).isoformat() + meta = {"last_checked": just_now} + # Even with freshness=0, age >= 0 so it's not < 0 + assert dl._is_within_freshness(meta, 0) is False + + +class TestRemoteUnchanged: + """Tests for _remote_unchanged.""" + + def _make_dl(self, tmp_path): + return BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path) + ) + + def test_returns_true_on_matching_etag(self, tmp_path): + dl = self._make_dl(tmp_path) + meta = {"etag": '"abc123"'} + mock_resp = Mock() + mock_resp.headers = {"ETag": '"abc123"'} + mock_resp.raise_for_status = Mock() + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_resp + ): + assert dl._remote_unchanged("https://example.com/f.parquet", meta) is True + + def test_returns_false_on_different_etag(self, tmp_path): + dl = self._make_dl(tmp_path) + meta = {"etag": '"old"'} + mock_resp = Mock() + mock_resp.headers = {"ETag": '"new"'} + mock_resp.raise_for_status = Mock() + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_resp + ): + assert dl._remote_unchanged("https://example.com/f.parquet", meta) is False + + def test_fallback_last_modified_match(self, tmp_path): + dl = self._make_dl(tmp_path) + lm = "Wed, 03 Dec 2025 15:54:19 GMT" + meta = {"last_modified": lm, "content_length": 100} + mock_resp = Mock() + mock_resp.headers = {"Last-Modified": lm, "Content-Length": "100"} + mock_resp.raise_for_status = Mock() + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_resp + ): + assert dl._remote_unchanged("https://example.com/f.parquet", meta) is True + + def test_returns_none_on_request_error(self, tmp_path): + """A failed HEAD is 'unknown', not 'unchanged' — the caller keeps the cached + file but must not restamp last_checked on the strength of it.""" + dl = self._make_dl(tmp_path) + meta = {"etag": '"abc"'} + with patch( + "babel_explorer.core.downloader.requests.head", + side_effect=requests.ConnectionError("fail"), + ): + assert dl._remote_unchanged("https://example.com/f.parquet", meta) is None + + +class TestGetDownloadedFileTiers: + """Tests for the three-tier logic in get_downloaded_file.""" + + def _make_dl(self, tmp_path, freshness=3600): + return BabelDownloader( + url_base="https://example.com/", + local_path=str(tmp_path), + freshness_seconds=freshness, + ) + + # --- Tier 1: within freshness window --- + + def test_tier1_returns_immediately_no_http(self, tmp_path): + """File + fresh .meta → no network calls at all.""" + dl = self._make_dl(tmp_path, freshness=3600) + test_file = "duckdb/test.parquet" + local = tmp_path / "duckdb" / "test.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data") + + meta = {"etag": '"abc"', "last_checked": datetime.now(UTC).isoformat()} + with open(str(local) + ".meta", "w") as f: + json.dump(meta, f) + + with patch("babel_explorer.core.downloader.requests.head") as mock_head: + with patch("babel_explorer.core.downloader.requests.get") as mock_get: + result = dl.get_downloaded_file(test_file) + mock_head.assert_not_called() + mock_get.assert_not_called() + assert result == str(local) + + def test_never_still_rechecks_after_a_release_change(self, tmp_path): + """`--check-download never` must not defeat the cross-release refresh. + + End-to-end version of the _is_within_freshness ordering: the cache holds the + previous release, sync_cache_version has expired the sidecar, and `never` still + has to issue the HEAD that notices the ETag changed. + """ + dl = self._make_dl(tmp_path, freshness=float("inf")) + test_file = "duckdb/test.parquet" + local = tmp_path / "duckdb" / "test.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data from the previous release") + + # No last_checked: exactly what sync_cache_version leaves behind. + with open(str(local) + ".meta", "w") as f: + json.dump({"etag": '"old"'}, f) + + mock_head_resp = Mock() + mock_head_resp.headers = {"ETag": '"new"'} + mock_head_resp.raise_for_status = Mock() + + def fake_download(url, tmp_path_, chunk_size): + """Stand in for the real download by writing the .tmp it would have left.""" + with open(tmp_path_, "wb") as f: + f.write(b"the new release") + return {"ETag": '"new"'} + + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp + ) as mock_head: + with patch.object( + dl, "_download_with_retry", side_effect=fake_download + ) as mock_download: + dl.get_downloaded_file(test_file) + + mock_head.assert_called_once() + mock_download.assert_called_once() + assert local.read_bytes() == b"the new release" + + # --- Tier 2: stale .meta, ETag matches --- + + def test_tier2_head_check_no_redownload(self, tmp_path): + """Stale .meta + matching ETag → HEAD only, no GET.""" + dl = self._make_dl(tmp_path, freshness=0) + test_file = "duckdb/test.parquet" + local = tmp_path / "duckdb" / "test.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data") + + old_ts = (datetime.now(UTC) - timedelta(hours=5)).isoformat() + meta = {"etag": '"abc"', "last_checked": old_ts} + with open(str(local) + ".meta", "w") as f: + json.dump(meta, f) + + mock_head_resp = Mock() + mock_head_resp.headers = {"ETag": '"abc"'} + mock_head_resp.raise_for_status = Mock() + + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp + ): + with patch("babel_explorer.core.downloader.requests.get") as mock_get: + result = dl.get_downloaded_file(test_file) + mock_get.assert_not_called() + assert result == str(local) + + def test_tier2_updates_last_checked_after_head(self, tmp_path): + """After successful HEAD match, last_checked in .meta is updated.""" + dl = self._make_dl(tmp_path, freshness=0) + test_file = "duckdb/upd.parquet" + local = tmp_path / "duckdb" / "upd.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data") + + old_ts = (datetime.now(UTC) - timedelta(hours=5)).isoformat() + meta = {"etag": '"abc"', "last_checked": old_ts} + with open(str(local) + ".meta", "w") as f: + json.dump(meta, f) + + mock_head_resp = Mock() + mock_head_resp.headers = {"ETag": '"abc"'} + mock_head_resp.raise_for_status = Mock() + + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp + ): + dl.get_downloaded_file(test_file) + + with open(str(local) + ".meta") as f: + updated_meta = json.load(f) + updated_ts = datetime.fromisoformat(updated_meta["last_checked"]) + assert (datetime.now(UTC) - updated_ts).total_seconds() < 5 + + def test_tier2_failed_head_does_not_refresh_last_checked(self, tmp_path): + """A HEAD that never happened must not mark the file freshly validated. + + sync_cache_version clears last_checked when the Babel release changes so + every cached file is re-checked. If one flaky HEAD restamped it, the old + release's Parquet would look current for the whole freshness window under a + .babel-version marker naming the new release. + """ + dl = self._make_dl(tmp_path, freshness=3600) + test_file = "duckdb/unreachable.parquet" + local = tmp_path / "duckdb" / "unreachable.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"data from the previous release") + + # No last_checked: exactly what sync_cache_version leaves behind. + with open(str(local) + ".meta", "w") as f: + json.dump({"etag": '"old"'}, f) + + with patch( + "babel_explorer.core.downloader.requests.head", + side_effect=requests.ConnectionError("network down"), + ): + with patch("babel_explorer.core.downloader.requests.get") as mock_get: + result = dl.get_downloaded_file(test_file) + mock_get.assert_not_called() + + assert result == str(local) + with open(str(local) + ".meta") as f: + assert "last_checked" not in json.load(f) + + # --- Tier 3: ETag changed, re-download --- + + def test_tier3_redownloads_when_etag_changed(self, tmp_path): + """Changed ETag → file deleted and re-downloaded.""" + dl = self._make_dl(tmp_path, freshness=0) + test_file = "duckdb/changed.parquet" + local = tmp_path / "duckdb" / "changed.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"old data") + + old_ts = (datetime.now(UTC) - timedelta(hours=5)).isoformat() + meta = {"etag": '"old"', "last_checked": old_ts} + with open(str(local) + ".meta", "w") as f: + json.dump(meta, f) + + mock_head_resp = Mock() + mock_head_resp.headers = {"ETag": '"new"'} + mock_head_resp.raise_for_status = Mock() + + new_content = b"new data" + + def fake_download(url, path, chunk_size): + with open(path, "wb") as f: + f.write(new_content) + return {"ETag": '"new"', "Content-Length": str(len(new_content))} + + with patch( + "babel_explorer.core.downloader.requests.head", return_value=mock_head_resp + ): + with patch.object(dl, "_download_with_retry", side_effect=fake_download): + result = dl.get_downloaded_file(test_file) + + assert open(result, "rb").read() == new_content + + # --- No .meta: fresh download --- + + def test_downloads_when_no_meta(self, tmp_path): + """No file and no .meta → download happens, .meta is saved.""" + dl = self._make_dl(tmp_path) + test_file = "duckdb/new.parquet" + content = b"fresh download" + + def fake_download(url, path, chunk_size): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(content) + return {"ETag": '"fresh"', "Content-Length": str(len(content))} + + with patch.object( + dl, "_download_with_retry", side_effect=fake_download + ) as mock_dl: + result = dl.get_downloaded_file(test_file) + mock_dl.assert_called_once() + + assert os.path.exists(result) + assert open(result, "rb").read() == content + # .meta should be saved + meta_path = result + ".meta" + assert os.path.exists(meta_path) + with open(meta_path) as f: + saved_meta = json.load(f) + assert saved_meta["etag"] == '"fresh"' + + def test_downloads_when_file_exists_but_no_meta(self, tmp_path): + """File exists but no .meta → treats as unknown, triggers full download flow.""" + dl = self._make_dl(tmp_path, freshness=3600) + test_file = "duckdb/nometa.parquet" + local = tmp_path / "duckdb" / "nometa.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"old content") + # No .meta file + + new_content = b"refreshed" + + def fake_download(url, path, chunk_size): + with open(path, "wb") as f: + f.write(new_content) + return {"ETag": '"new"'} + + with patch.object( + dl, "_download_with_retry", side_effect=fake_download + ) as mock_dl: + result = dl.get_downloaded_file(test_file) + mock_dl.assert_called_once() + + assert open(result, "rb").read() == new_content + + +class TestPartialDownloadSafety: + """A .tmp must never be resumed across two different versions of a remote file.""" + + def test_leftover_tmp_from_an_earlier_run_is_discarded(self, tmp_path): + """A .tmp of unknown provenance is deleted before the download starts. + + get_downloaded_file only reaches the download block when the remote bytes + changed, so resuming an orphaned .tmp (left by a killed process) would + append the new file's tail to the old file's prefix and then stamp the + splice with the new ETag. + """ + dl = BabelDownloader( + url_base="https://example.com/", + local_path=str(tmp_path), + freshness_seconds=0, + ) + test_file = "duckdb/spliced.parquet" + local = tmp_path / "duckdb" / "spliced.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"old version") + with open(str(local) + ".meta", "w") as f: + json.dump({"etag": '"old"'}, f) + tmp_file = local.parent / "spliced.parquet.tmp" + tmp_file.write_bytes(b"PREFIX-OF-OLD-VERSION") + + seen_sizes = [] + + def fake_download(url, path, chunk_size): + seen_sizes.append(os.path.getsize(path) if os.path.exists(path) else None) + with open(path, "wb") as f: + f.write(b"new version") + return {"ETag": '"new"'} + + head = Mock(headers={"ETag": '"new"'}, raise_for_status=Mock()) + with ( + patch("babel_explorer.core.downloader.requests.head", return_value=head), + patch.object(dl, "_download_with_retry", side_effect=fake_download), + ): + result = dl.get_downloaded_file(test_file) + + assert seen_sizes == [None], "the stale .tmp was still on disk" + assert open(result, "rb").read() == b"new version" + + def test_keyboard_interrupt_removes_the_partial_file(self, tmp_path): + """Ctrl-C is a BaseException; the .tmp must still be cleaned up.""" + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + tmp_file = tmp_path / "interrupted.parquet.tmp" + + def fake_download(url, path, chunk_size): + with open(path, "wb") as f: + f.write(b"half a file") + raise KeyboardInterrupt + + with patch.object(dl, "_download_with_retry", side_effect=fake_download): + with pytest.raises(KeyboardInterrupt): + dl.get_downloaded_file("interrupted.parquet") + + assert not tmp_file.exists() + + def test_resume_sends_if_range_once_a_validator_is_known(self, tmp_path): + """After the first response, a resume is conditional on the file not changing.""" + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = str(tmp_path / "conditional.bin") + + # First attempt streams 4 of 10 bytes and then trips the size check. + first = TestDownloadWithRetry._make_response( + 200, {"Content-Length": "10", "ETag": '"v1"'}, [b"abcd"] + ) + second = TestDownloadWithRetry._make_response( + 206, {"Content-Length": "6", "ETag": '"v1"'}, [b"efghij"] + ) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[first, second], + ) as mock_get, + patch("babel_explorer.core.downloader.time.sleep"), + ): + dl._download_with_retry("https://example.com/file", out_path, 1024) + + assert mock_get.call_args_list[1].kwargs["headers"] == { + "Range": "bytes=4-", + "If-Range": '"v1"', + } + assert open(out_path, "rb").read() == b"abcdefghij" + + +class TestDownloadCompleteness: + """A short stream must be retried, never promoted as the finished file.""" + + def test_truncated_stream_raises(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = str(tmp_path / "short.bin") + + mock_response = Mock() + mock_response.headers = {"Content-Length": "10"} + mock_response.iter_content = Mock(return_value=[b"only4"]) + + with pytest.raises(IncompleteDownloadError, match="expected 10 bytes"): + dl._stream_download(mock_response, out_path, 0, 1024) + + def test_truncated_stream_is_retried_and_resumed(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = str(tmp_path / "resumed.bin") + + # The ETag is what makes the resume conditional, and so allowed at all: + # without a validator the retry restarts from zero instead. + first = TestDownloadWithRetry._make_response( + 200, {"Content-Length": "10", "ETag": '"v1"'}, [b"abcd"] + ) + second = TestDownloadWithRetry._make_response( + 206, {"Content-Length": "6", "ETag": '"v1"'}, [b"efghij"] + ) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[first, second], + ), + patch("babel_explorer.core.downloader.time.sleep"), + ): + dl._download_with_retry("https://example.com/file", out_path, 1024) + + assert open(out_path, "rb").read() == b"abcdefghij" + + def test_encoded_body_skips_the_size_check(self, tmp_path): + """With Content-Encoding set, iter_content yields decoded bytes whose count + has nothing to do with Content-Length.""" + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = str(tmp_path / "gzipped.bin") + + mock_response = Mock() + mock_response.headers = {"Content-Length": "4", "Content-Encoding": "gzip"} + mock_response.iter_content = Mock(return_value=[b"decompressed"]) + + dl._stream_download(mock_response, out_path, 0, 1024) + assert open(out_path, "rb").read() == b"decompressed" + + def test_416_with_a_shorter_remote_file_restarts(self, tmp_path): + """A remote rebuild that shrank the file also answers 416; the over-long + local file must not be promoted as complete.""" + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = tmp_path / "shrunk.bin" + + # Attempt 1 ends short, leaving 20 bytes and a validator behind. Only then is + # there a Range to send, and so a 416 to receive. + truncated = TestDownloadWithRetry._make_response( + 200, {"Content-Length": "30", "ETag": '"old"'}, [b"the old, longer file"] + ) + too_long = TestDownloadWithRetry._make_response(416) + fresh = TestDownloadWithRetry._make_response( + 200, {"Content-Length": "5", "ETag": '"new"'}, [b"short"] + ) + head = MagicMock(status_code=200, headers={"Content-Length": "5"}) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[truncated, too_long, fresh], + ), + patch("babel_explorer.core.downloader.requests.head", return_value=head), + patch("babel_explorer.core.downloader.time.sleep"), + ): + headers = dl._download_with_retry( + "https://example.com/file", str(out_path), 1024 + ) + + assert out_path.read_bytes() == b"short" + assert headers["ETag"] == '"new"' + + +class TestFullContentLength: + """A 206 Content-Length is a range length, not the file's length.""" + + def test_partial_response_records_the_total_from_content_range(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + file_path = str(tmp_path / "resumed.parquet") + with open(file_path, "wb") as f: + f.write(b"0123456789") + + dl._save_meta( + file_path, + {"Content-Length": "6", "Content-Range": "bytes 4-9/10", "ETag": '"e"'}, + ) + with open(file_path + ".meta") as f: + assert json.load(f)["content_length"] == 10 + + def test_partial_response_with_unknown_total_falls_back_to_the_file(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + file_path = str(tmp_path / "unknown_total.parquet") + with open(file_path, "wb") as f: + f.write(b"0123456789") + + dl._save_meta( + file_path, {"Content-Length": "6", "Content-Range": "bytes 4-9/*"} + ) + with open(file_path + ".meta") as f: + assert json.load(f)["content_length"] == 10 + + +class TestGetDownloadedFileCaching: + """Tests that repeated calls within the freshness window avoid redundant downloads.""" + + def test_second_call_within_freshness_skips_download(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + content = b"cached content" + + def fake_download(url, path, chunk_size): + with open(path, "wb") as f: + f.write(content) + return {} + + with patch.object( + dl, "_download_with_retry", side_effect=fake_download + ) as mock_dl: + r1 = dl.get_downloaded_file("cached.txt") + r2 = dl.get_downloaded_file("cached.txt") + assert r1 == r2 + mock_dl.assert_called_once() # freshness window prevents second download + + +class TestDownloadWithRetry: + """Tests for _download_with_retry.""" + + @staticmethod + def _make_response(status_code, headers=None, content=None): + m = MagicMock() + m.__enter__.return_value = m + m.status_code = status_code + m.headers = headers or {} + if content is not None: + m.iter_content = Mock(return_value=content) + return m + + def test_retries_exhausted_raises_runtime_error(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=2 + ) + with patch( + "babel_explorer.core.downloader.requests.get", + side_effect=requests.ConnectionError("fail"), + ): + with patch("babel_explorer.core.downloader.time.sleep"): # skip waiting + with pytest.raises(RuntimeError, match="Failed to download"): + dl._download_with_retry( + "https://example.com/file", str(tmp_path / "f"), 1024 + ) + + def test_succeeds_on_second_attempt(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = str(tmp_path / "retry_success.bin") + + mock_response = self._make_response(200, {"Content-Length": "5"}, [b"hello"]) + side_effects = [requests.ConnectionError("first fail"), mock_response] + + with patch( + "babel_explorer.core.downloader.requests.get", side_effect=side_effects + ): + with patch("babel_explorer.core.downloader.time.sleep"): + dl._download_with_retry("https://example.com/file", out_path, 1024) + assert os.path.exists(out_path) + + def test_resume_without_a_validator_restarts_from_zero(self, tmp_path): + """A bare Range is a splice waiting to happen. + + With neither an ETag nor a Last-Modified there is no If-Range to send, so + nothing stops a server that rebuilt the file from handing back the new + version's tail to append to the old version's prefix — and the result would + be stamped with the new validator and pass every later check. Restarting + costs a re-download; resuming costs silent corruption. + """ + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = tmp_path / "partial.bin" + out_path.write_bytes(b"partial") # 7 bytes + + mock_response = self._make_response(200, {"Content-Length": "5"}, [b"whole"]) + with patch( + "babel_explorer.core.downloader.requests.get", return_value=mock_response + ) as mock_get: + dl._download_with_retry("https://example.com/file", str(out_path), 1024) + _, kwargs = mock_get.call_args + assert kwargs["headers"] == {}, "no validator means no conditional resume" + assert out_path.read_bytes() == b"whole" + + def test_http_416_file_already_complete(self, tmp_path): + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = tmp_path / "complete.bin" + + # Attempt 1 over-declares Content-Length and delivers the whole 9-byte file, + # so the size check retries it; attempt 2 then asks for bytes past the end of + # a file it already holds in full, which is what a 416 legitimately means. + over_declared = self._make_response( + 200, {"Content-Length": "20", "ETag": '"v1"'}, [b"full file"] + ) + mock_response = self._make_response(416) + head = MagicMock(status_code=200, headers={"Content-Length": "9"}) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[over_declared, mock_response], + ), + patch("babel_explorer.core.downloader.requests.head", return_value=head), + patch("babel_explorer.core.downloader.time.sleep"), + ): + headers = dl._download_with_retry( + "https://example.com/file", str(out_path), 1024 + ) + assert out_path.read_bytes() == b"full file" + # The 416 response describes the error body; saving its length as the file's + # metadata would fail every later freshness check and re-download the file. + assert headers == {"Content-Length": "9"} + + def test_416_without_a_content_length_restarts(self, tmp_path): + """416 alone does not prove completeness — it is also how a shrunk file answers. + + With no remote length to compare against there is no way to tell the two + apart, so the local file must not be promoted as complete. + """ + dl = BabelDownloader( + url_base="https://example.com/", local_path=str(tmp_path), retries=3 + ) + out_path = tmp_path / "unverifiable.bin" + + # Attempt 1 ends short, leaving bytes and a validator behind, so attempt 2 + # sends the Range that draws the 416. + truncated = self._make_response( + 200, {"Content-Length": "30", "ETag": '"old"'}, [b"possibly stale bytes"] + ) + range_rejected = self._make_response(416) + fresh = self._make_response( + 200, {"Content-Length": "5", "ETag": '"new"'}, [b"fresh"] + ) + head = MagicMock(status_code=200, headers={}) + with ( + patch( + "babel_explorer.core.downloader.requests.get", + side_effect=[truncated, range_rejected, fresh], + ), + patch("babel_explorer.core.downloader.requests.head", return_value=head), + patch("babel_explorer.core.downloader.time.sleep"), + ): + headers = dl._download_with_retry( + "https://example.com/file", str(out_path), 1024 + ) + + assert out_path.read_bytes() == b"fresh" + assert headers["ETag"] == '"new"' + + def test_server_no_resume_restarts_download(self, tmp_path): + """When server responds 200 (instead of 206), partial file is removed and download restarts.""" + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = tmp_path / "no_resume.bin" + out_path.write_bytes(b"partial") + + mock_response = self._make_response( + 200, {"Content-Length": "12"}, [b"full content"] + ) + with patch( + "babel_explorer.core.downloader.requests.get", return_value=mock_response + ): + dl._download_with_retry("https://example.com/file", str(out_path), 1024) + assert out_path.read_bytes() == b"full content" + + def test_returns_response_headers(self, tmp_path): + """_download_with_retry should return response headers.""" + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = str(tmp_path / "headers.bin") + + mock_response = self._make_response( + 200, {"Content-Length": "5", "ETag": '"abc"'}, [b"hello"] + ) + with patch( + "babel_explorer.core.downloader.requests.get", return_value=mock_response + ): + headers = dl._download_with_retry( + "https://example.com/file", out_path, 1024 + ) + assert headers["ETag"] == '"abc"' + + +class TestStreamDownload: + """Tests for _stream_download.""" + + def test_writes_chunks(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = str(tmp_path / "stream.bin") + + mock_response = Mock() + mock_response.headers = {"Content-Length": "10"} + mock_response.iter_content = Mock(return_value=[b"hello", b"world"]) + + dl._stream_download(mock_response, out_path, resume_byte_pos=0, chunk_size=1024) + with open(out_path, "rb") as f: + assert f.read() == b"helloworld" + + def test_append_mode_on_resume(self, tmp_path): + dl = BabelDownloader(url_base="https://example.com/", local_path=str(tmp_path)) + out_path = tmp_path / "append.bin" + out_path.write_bytes(b"start") + + mock_response = Mock() + mock_response.headers = {"Content-Length": "3"} + mock_response.iter_content = Mock(return_value=[b"end"]) + + dl._stream_download( + mock_response, str(out_path), resume_byte_pos=5, chunk_size=1024 + ) + assert out_path.read_bytes() == b"startend" + + +# ========================================================================== +# Integration Tests — require network access +# ========================================================================== + + +@pytest.mark.integration +def test_download_concord_parquet(downloaded_concord): + """Verify Concord.parquet downloads and is > 100 MB.""" + assert os.path.isfile(downloaded_concord) + size = os.path.getsize(downloaded_concord) + assert size > 100 * 1024 * 1024, f"Concord.parquet too small: {size} bytes" + + +@pytest.mark.integration +def test_download_metadata_parquet(downloaded_metadata): + """Verify Metadata.parquet downloads and is non-empty.""" + assert os.path.isfile(downloaded_metadata) + assert os.path.getsize(downloaded_metadata) > 0 + + +@pytest.mark.integration +def test_download_creates_meta_file(downloaded_concord): + """After download, a .meta sidecar file should exist.""" + meta_path = downloaded_concord + ".meta" + assert os.path.isfile(meta_path), f"Missing .meta file: {meta_path}" + with open(meta_path) as f: + meta = json.load(f) + assert "last_checked" in meta + + +@pytest.mark.integration +def test_download_caching_real_files(shared_downloader, downloaded_concord): + """Second call returns same path and file is not re-downloaded.""" + path2 = shared_downloader.get_downloaded_file(CONCORD_FILE) + assert path2 == downloaded_concord + assert os.path.getmtime(downloaded_concord) == os.path.getmtime(path2) + + +@pytest.mark.integration +@pytest.mark.slow +def test_download_identifiers_parquet(downloaded_identifiers): + """Verify Identifiers.parquet downloads and is > 2 GB.""" + assert os.path.isfile(downloaded_identifiers) + size = os.path.getsize(downloaded_identifiers) + assert size > 2 * 1024 * 1024 * 1024, f"Identifiers.parquet too small: {size} bytes" diff --git a/tests/test_formatting.py b/tests/test_formatting.py new file mode 100644 index 0000000..2a6629f --- /dev/null +++ b/tests/test_formatting.py @@ -0,0 +1,416 @@ +""" +Unit tests for formatting.py — no network, no mocking required. +""" + +import io +import json + +import pytest +from rich.console import Console + +from babel_explorer.core.babel_xrefs import ( + CrossReference, + IdentifierRecord, + LabeledCrossReference, +) +from babel_explorer.core.nodenorm import Identifier +from babel_explorer.formatting import ( + curie_with_label, + escape_label, + format_identifier_record, + hl_curie, + make_console, + record_to_dict, + write_records, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def xref(): + return CrossReference( + filename="Concord.parquet", subj="A:1", pred="skos:exactMatch", obj="B:2" + ) + + +@pytest.fixture +def labeled_xref(): + return LabeledCrossReference( + filename="Concord.parquet", + subj="A:1", + pred="skos:exactMatch", + obj="B:2", + subj_label="Alpha", + subj_biolink_type=("biolink:Disease",), + obj_label="Beta", + obj_biolink_type=("biolink:Gene", "biolink:NamedThing"), + ) + + +@pytest.fixture +def id_record(): + return IdentifierRecord( + curie="A:1", + extra_fields=(("type", "gene"), ("label", "Alpha")), + ) + + +@pytest.fixture +def identifier(): + return Identifier( + curie="MONDO:0004979", + label="asthma", + biolink_type=("biolink:Disease",), + taxa=("NCBITaxon:9606",), + description=("A chronic inflammatory disease",), + ) + + +# --------------------------------------------------------------------------- +# Tests for make_console and hl_curie +# --------------------------------------------------------------------------- + + +class TestConsoleUtilities: + def test_make_console_returns_console(self): + console = make_console() + assert isinstance(console, Console) + + def test_make_console_accepts_file(self): + out = io.StringIO() + console = make_console(file=out) + assert isinstance(console, Console) + console.print("hello") + assert "hello" in out.getvalue() + + def test_hl_curie_highlighted_contains_markup(self): + result = hl_curie("HGNC:1100", 0) + assert "bold cyan" in result + assert "HGNC:1100" in result + + def test_hl_curie_not_highlighted_is_plain(self): + result = hl_curie("HGNC:1100", None) + assert result == "HGNC:1100" + assert "[" not in result + + def test_hl_curie_highlighted_renders_correctly(self): + """Markup renders to plain text on a non-TTY console.""" + out = io.StringIO() + console = Console(file=out, highlight=False, no_color=True) + console.print(hl_curie("HGNC:1100", 0)) + assert "HGNC:1100" in out.getvalue() + + def test_hl_curie_highlighted_renders_with_color(self): + """On a forced-TTY console, ANSI codes are emitted.""" + out = io.StringIO() + console = Console(file=out, highlight=False, force_terminal=True) + console.print(hl_curie("HGNC:1100", 0)) + output = out.getvalue() + assert "HGNC:1100" in output + assert "\x1b[" in output # ANSI escape present + + +# --------------------------------------------------------------------------- +# Tests for record_to_dict +# --------------------------------------------------------------------------- + + +class TestRecordToDict: + def test_cross_reference(self, xref): + d = record_to_dict(xref) + assert d == { + "filename": "Concord.parquet", + "subj": "A:1", + "pred": "skos:exactMatch", + "obj": "B:2", + } + + def test_labeled_cross_reference_has_all_eight_fields(self, labeled_xref): + d = record_to_dict(labeled_xref) + assert set(d.keys()) == { + "filename", + "subj", + "pred", + "obj", + "subj_label", + "subj_biolink_type", + "obj_label", + "obj_biolink_type", + } + # dataclasses.asdict() preserves tuple types + assert d["subj_biolink_type"] == ("biolink:Disease",) + assert d["obj_biolink_type"] == ("biolink:Gene", "biolink:NamedThing") + + def test_identifier_record_extra_fields_expanded(self, id_record): + d = record_to_dict(id_record) + assert "extra_fields" not in d + assert d["curie"] == "A:1" + assert d["type"] == "gene" + assert d["label"] == "Alpha" + + def test_identifier_record_no_extra_fields(self): + rec = IdentifierRecord(curie="X:1") + d = record_to_dict(rec) + assert d == {"curie": "X:1"} + + def test_plain_dict_passthrough(self): + data = {"a": 1, "b": "hello"} + assert record_to_dict(data) is data + + def test_identifier_dataclass(self, identifier): + d = record_to_dict(identifier) + assert d["curie"] == "MONDO:0004979" + assert d["label"] == "asthma" + # dataclasses.asdict() preserves tuple types + assert d["biolink_type"] == ("biolink:Disease",) + assert d["taxa"] == ("NCBITaxon:9606",) + + +# --------------------------------------------------------------------------- +# Tests for write_records +# --------------------------------------------------------------------------- + + +class TestWriteRecords: + # -- json format -- + + def test_json_is_valid_list(self, xref): + out = io.StringIO() + write_records([xref], "json", file=out) + data = json.loads(out.getvalue()) + assert isinstance(data, list) + assert len(data) == 1 + assert data[0]["subj"] == "A:1" + + def test_json_empty_list(self): + out = io.StringIO() + write_records([], "json", file=out) + assert json.loads(out.getvalue()) == [] + + def test_json_indent_controls_formatting(self, xref): + out_pretty = io.StringIO() + write_records([xref], "json", indent=2, file=out_pretty) + + out_compact = io.StringIO() + write_records([xref], "json", indent=None, file=out_compact) + + # Pretty-printed output has more lines (has newlines per field) + assert out_pretty.getvalue().count("\n") > out_compact.getvalue().count("\n") + + def test_json_tuple_fields_serialized_as_arrays(self, labeled_xref): + # json.dump converts tuples to JSON arrays, so json.loads gives back lists + out = io.StringIO() + write_records([labeled_xref], "json", file=out) + data = json.loads(out.getvalue()) + assert isinstance(data[0]["subj_biolink_type"], list) + assert data[0]["obj_biolink_type"] == ["biolink:Gene", "biolink:NamedThing"] + + def test_json_plain_dict(self): + out = io.StringIO() + write_records([{"a": 1, "b": "x"}], "json", file=out) + assert json.loads(out.getvalue()) == [{"a": 1, "b": "x"}] + + # -- tsv format -- + + def test_tsv_has_header_row(self, xref): + out = io.StringIO() + write_records([xref], "tsv", file=out) + lines = out.getvalue().splitlines() + assert lines[0] == "filename\tsubj\tpred\tobj" + + def test_tsv_data_row(self, xref): + out = io.StringIO() + write_records([xref], "tsv", file=out) + lines = out.getvalue().splitlines() + assert lines[1] == "Concord.parquet\tA:1\tskos:exactMatch\tB:2" + + def test_tsv_tuple_fields_pipe_joined(self, labeled_xref): + out = io.StringIO() + write_records([labeled_xref], "tsv", file=out) + lines = out.getvalue().splitlines() + # Header row + assert "subj_biolink_type" in lines[0] + # Data row: multi-value tuple joined with pipe + assert "biolink:Gene|biolink:NamedThing" in lines[1] + + def test_tsv_empty_no_output(self): + out = io.StringIO() + write_records([], "tsv", file=out) + assert out.getvalue() == "" + + def test_tsv_identifier_record_extra_fields_expanded(self, id_record): + out = io.StringIO() + write_records([id_record], "tsv", file=out) + lines = out.getvalue().splitlines() + assert "curie" in lines[0] + assert "type" in lines[0] + assert "label" in lines[0] + assert "A:1" in lines[1] + + # -- csv format -- + + def test_csv_has_header_row(self, xref): + out = io.StringIO() + write_records([xref], "csv", file=out) + lines = out.getvalue().splitlines() + assert lines[0] == "filename,subj,pred,obj" + + def test_csv_data_row(self, xref): + out = io.StringIO() + write_records([xref], "csv", file=out) + lines = out.getvalue().splitlines() + assert lines[1] == "Concord.parquet,A:1,skos:exactMatch,B:2" + + def test_csv_empty_no_output(self): + out = io.StringIO() + write_records([], "csv", file=out) + assert out.getvalue() == "" + + def test_csv_tuple_fields_pipe_joined(self, labeled_xref): + out = io.StringIO() + write_records([labeled_xref], "csv", file=out) + lines = out.getvalue().splitlines() + assert "biolink:Gene|biolink:NamedThing" in lines[1] + + # -- invalid formats (including console, which is handled at CLI layer) -- + + def test_text_format_raises_value_error(self, xref): + out = io.StringIO() + with pytest.raises(ValueError, match="Unknown format"): + write_records([xref], "text", file=out) + + def test_console_format_raises_value_error(self, xref): + """Console format is handled by the CLI, not write_records.""" + out = io.StringIO() + with pytest.raises(ValueError, match="Unknown format"): + write_records([xref], "console", file=out) + + def test_unknown_format_raises_value_error(self, xref): + out = io.StringIO() + with pytest.raises(ValueError, match="Unknown format"): + write_records([xref], "xml", file=out) + + +# --------------------------------------------------------------------------- +# Tests for the label convention (escape_label / curie_with_label) +# --------------------------------------------------------------------------- + + +class TestLabelConvention: + """CLAUDE.md: a label follows its CURIE in double quotes, or is omitted.""" + + def test_label_follows_curie_in_double_quotes(self): + assert curie_with_label("MONDO:1", None, "asthma") == 'MONDO:1 "asthma"' + + @pytest.mark.parametrize("label", [None, ""]) + def test_absent_label_is_omitted_entirely(self, label): + """No placeholder — a CURIE with no label renders as the bare CURIE.""" + assert curie_with_label("MONDO:1", None, label) == "MONDO:1" + + def test_backslashes_escape_before_quotes(self): + assert escape_label(r'a\b"c') == r"a\\b\"c" + + def test_escaped_label_matches_the_documented_regex(self): + import re + + rendered = curie_with_label("MONDO:1", None, r'say "hi" \ bye') + assert re.search(r'"([^"\\]|\\.)*"', rendered).group(0) == ( + r'"say \"hi\" \\ bye"' + ) + + def test_query_curie_is_highlighted_at_depth_zero(self): + assert curie_with_label("MONDO:1", 0, "asthma").startswith("[bold cyan]") + + def test_identifier_record_uses_the_same_convention(self): + rec = IdentifierRecord( + curie="A:1", extra_fields=(("n", 1),), nodenorm_label='a"b' + ) + rendered = format_identifier_record(rec) + assert r'nodenorm_label="a\"b"' in rendered + + def test_identifier_record_omits_absent_label(self): + rec = IdentifierRecord(curie="A:1", extra_fields=(("n", 1),)) + assert "label=" not in format_identifier_record(rec) + + def test_nodenorm_label_does_not_collide_with_the_parquet_label_column(self): + """Identifiers.parquet carries its own label column; both must survive.""" + rec = IdentifierRecord( + curie="MONDO:0004979", + extra_fields=(("label", "asthma (Babel)"),), + nodenorm_label="asthma (NodeNorm)", + ) + d = record_to_dict(rec) + assert d["nodenorm_label"] == "asthma (NodeNorm)" + assert d["label"] == "asthma (Babel)" + + +class TestLabelOmissionInRecords: + """The omit-when-absent rule covers the NodeNorm label fields, and only those.""" + + def test_an_empty_parquet_label_column_is_kept(self): + """Identifiers.parquet's own `label` column is data, not an absent lookup. + + Dropping it when empty would give some rows of a json/tsv/csv run a `label` + key and others none, so `row["label"]` raises KeyError downstream. + """ + rec = IdentifierRecord(curie="A:1", extra_fields=(("label", ""),)) + d = record_to_dict(rec) + assert d["label"] == "" + assert "nodenorm_label" not in d + + def test_json_rows_all_carry_the_parquet_label_column(self): + out = io.StringIO() + write_records( + [ + IdentifierRecord(curie="A:1", extra_fields=(("label", ""),)), + IdentifierRecord(curie="B:2", extra_fields=(("label", "asthma"),)), + ], + "json", + file=out, + ) + rows = json.loads(out.getvalue()) + assert [r["label"] for r in rows] == ["", "asthma"] + + def test_empty_subj_and_obj_labels_are_dropped(self): + xref = LabeledCrossReference( + filename="f", + subj="A:1", + pred="p", + obj="B:2", + subj_label="", + subj_biolink_type=(), + obj_label="", + obj_biolink_type=(), + ) + d = record_to_dict(xref) + assert "subj_label" not in d and "obj_label" not in d + + def test_present_labels_are_kept(self): + xref = LabeledCrossReference( + filename="f", + subj="A:1", + pred="p", + obj="B:2", + subj_label="asthma", + subj_biolink_type=(), + obj_label="", + obj_biolink_type=(), + ) + d = record_to_dict(xref) + assert d["subj_label"] == "asthma" and "obj_label" not in d + + def test_tabular_output_survives_rows_with_differing_keys(self): + """A labelled row after an unlabelled one must not blow up DictWriter.""" + rows = [ + IdentifierRecord(curie="A:1", extra_fields=()), + IdentifierRecord(curie="B:2", extra_fields=(), nodenorm_label="asthma"), + ] + out = io.StringIO() + write_records(rows, "csv", file=out) + lines = out.getvalue().splitlines() + assert lines[0] == "curie,nodenorm_label" + assert lines[1] == "A:1," + assert lines[2] == "B:2,asthma" diff --git a/tests/test_nodenorm.py b/tests/test_nodenorm.py new file mode 100644 index 0000000..47c58fc --- /dev/null +++ b/tests/test_nodenorm.py @@ -0,0 +1,487 @@ +""" +Tests for NodeNorm and Identifier classes. + +Unit tests use mocks; integration tests call the real NodeNorm API. +""" + +from unittest.mock import Mock, patch + +import pytest +import requests + +from babel_explorer.core.nodenorm import NORMALIZE_BATCH_SIZE, Identifier, NodeNorm +from tests.constants import load_curies + +VALID_CURIES = load_curies() + + +# ========================================================================== +# Unit Tests — Identifier +# ========================================================================== + + +class TestIdentifier: + """Tests for the Identifier dataclass.""" + + def test_creation_with_defaults(self): + ident = Identifier(curie="MONDO:0004979") + assert ident.curie == "MONDO:0004979" + assert ident.label == "" + assert ident.biolink_type == () + assert ident.taxa == () + assert ident.description == () + + def test_full_creation(self): + ident = Identifier( + curie="MONDO:0004979", + label="asthma", + biolink_type=("biolink:Disease",), + taxa=("NCBITaxon:9606",), + description=("A chronic respiratory disease",), + ) + assert ident.label == "asthma" + assert ident.biolink_type == ("biolink:Disease",) + assert ident.taxa == ("NCBITaxon:9606",) + + def test_from_dict_minimal(self): + d = {"identifier": "X:1"} + ident = Identifier.from_dict(d) + assert ident.curie == "X:1" + assert ident.label == "" + + def test_from_dict_full(self): + d = { + "identifier": "X:1", + "label": "Alpha", + "type": ["biolink:NamedThing"], + "taxa": ["NCBITaxon:9606"], + "description": ["Some thing"], + } + ident = Identifier.from_dict(d) + assert ident.curie == "X:1" + assert ident.label == "Alpha" + assert ident.biolink_type == ("biolink:NamedThing",) + assert ident.taxa == ("NCBITaxon:9606",) + + def test_from_dict_partial(self): + d = {"identifier": "X:1", "label": "Beta"} + ident = Identifier.from_dict(d) + assert ident.curie == "X:1" + assert ident.label == "Beta" + assert ident.biolink_type == () + + def test_from_dict_type_as_string(self): + """NodeNorm may return 'type' as a bare string for individual identifiers.""" + d = {"identifier": "X:1", "type": "biolink:Disease"} + ident = Identifier.from_dict(d) + assert ident.biolink_type == ("biolink:Disease",), ( + "biolink_type should be a 1-tuple, not a tuple of characters" + ) + + def test_from_dict_description_as_string(self): + """NodeNorm may return 'description' as a bare string.""" + d = {"identifier": "X:1", "description": "A chronic disease"} + ident = Identifier.from_dict(d) + assert ident.description == ("A chronic disease",), ( + "description should be a 1-tuple, not a tuple of characters" + ) + + def test_from_dict_taxa_as_string(self): + """NodeNorm may return 'taxa' as a bare string.""" + d = {"identifier": "X:1", "taxa": "NCBITaxon:9606"} + ident = Identifier.from_dict(d) + assert ident.taxa == ("NCBITaxon:9606",), ( + "taxa should be a 1-tuple, not a tuple of characters" + ) + + def test_from_dict_all_fields_as_strings(self): + """All three tuple fields as strings produce correct single-element tuples.""" + d = { + "identifier": "X:1", + "label": "Alpha", + "type": "biolink:NamedThing", + "taxa": "NCBITaxon:9606", + "description": "Some description", + } + ident = Identifier.from_dict(d) + assert ident.biolink_type == ("biolink:NamedThing",) + assert ident.taxa == ("NCBITaxon:9606",) + assert ident.description == ("Some description",) + + def test_lt_ordering(self): + a = Identifier(curie="A:1") + b = Identifier(curie="B:2") + assert a < b + + def test_sorting(self): + items = [ + Identifier(curie="C:3"), + Identifier(curie="A:1"), + Identifier(curie="B:2"), + ] + result = sorted(items) + assert [x.curie for x in result] == ["A:1", "B:2", "C:3"] + + +# ========================================================================== +# Unit Tests — NodeNorm (mocked) +# ========================================================================== + + +class TestNodeNormInit: + """Tests for NodeNorm constructor and URL normalisation.""" + + def test_default_url(self): + nn = NodeNorm() + assert nn.nodenorm_url == "" + + def test_custom_url(self): + nn = NodeNorm(nodenorm_url="https://custom.api/") + assert nn.nodenorm_url == "https://custom.api/" + + def test_empty_url_normalize_curie_returns_none_without_network(self): + """NodeNorm('') must not make any HTTP calls and must return None.""" + nn = NodeNorm("") + with patch("babel_explorer.core.nodenorm.requests.get") as mock_get: + result = nn.normalize_curie("MONDO:0004979") + mock_get.assert_not_called() + assert result is None + + +class TestNormalizeCurieMocked: + """Unit tests for NodeNorm.normalize_curie() with mocked HTTP responses.""" + + def _make_nn(self): + return NodeNorm(nodenorm_url="https://example.com/") + + def test_correct_api_endpoint_and_params(self): + nn = self._make_nn() + mock_resp = Mock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"X:1": {"id": {"identifier": "X:1"}}} + mock_resp.raise_for_status = Mock() + + with patch( + "babel_explorer.core.nodenorm.requests.get", return_value=mock_resp + ) as mock_get: + nn.normalize_curie("X:1") + mock_get.assert_called_once() + args, kwargs = mock_get.call_args + assert args[0] == "https://example.com/get_normalized_nodes" + # CURIEs are always sent as a batch, even when there is only one. + assert kwargs["params"]["curie"] == ["X:1"] + + def test_returns_result_for_curie(self): + nn = self._make_nn() + expected = {"id": {"identifier": "X:1"}, "equivalent_identifiers": []} + mock_resp = Mock() + mock_resp.json.return_value = {"X:1": expected} + mock_resp.raise_for_status = Mock() + + with patch("babel_explorer.core.nodenorm.requests.get", return_value=mock_resp): + result = nn.normalize_curie("X:1") + assert result == expected + + def test_caching(self): + nn = self._make_nn() + mock_resp = Mock() + mock_resp.json.return_value = {"X:1": {"id": "X:1"}} + mock_resp.raise_for_status = Mock() + + with patch( + "babel_explorer.core.nodenorm.requests.get", return_value=mock_resp + ) as mock_get: + nn.normalize_curie("X:1") + nn.normalize_curie("X:1") + mock_get.assert_called_once() + + def test_http_error_raises(self): + nn = self._make_nn() + mock_resp = Mock() + mock_resp.raise_for_status.side_effect = requests.HTTPError("500 Server Error") + + with patch("babel_explorer.core.nodenorm.requests.get", return_value=mock_resp): + with pytest.raises(requests.HTTPError): + nn.normalize_curie("BAD:1") + + +class TestGetIdentifierMocked: + """Unit tests for NodeNorm.get_identifier() with mocked normalize_curie.""" + + def _make_nn(self): + return NodeNorm(nodenorm_url="https://example.com/") + + def test_exact_match_found(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [ + {"identifier": "X:1", "label": "Alpha", "type": ["biolink:Disease"]}, + {"identifier": "X:2", "label": "Beta"}, + ], + } + with patch.object(nn, "normalize_curie", return_value=api_result): + ident = nn.get_identifier("X:1") + assert ident.curie == "X:1" + assert ident.label == "Alpha" + + def test_no_match_returns_bare_identifier(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [ + {"identifier": "X:2", "label": "Beta"}, + ], + } + with patch.object(nn, "normalize_curie", return_value=api_result): + ident = nn.get_identifier("X:1") + assert ident.curie == "X:1" + assert ident.label == "" + + def test_falsy_result_returns_bare_identifier(self): + nn = self._make_nn() + with patch.object(nn, "normalize_curie", return_value=None): + ident = nn.get_identifier("X:1") + assert ident.curie == "X:1" + assert ident.label == "" + + def test_caching(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [ + {"identifier": "X:1", "label": "Alpha"}, + ], + } + with patch.object(nn, "normalize_curie", return_value=api_result) as mock_norm: + nn.get_identifier("X:1") + nn.get_identifier("X:1") + mock_norm.assert_called_once() + + +class TestGetCliqueIdentifiersMocked: + """Unit tests for NodeNorm.get_clique_identifiers() with mocked normalize_curie.""" + + def _make_nn(self): + return NodeNorm(nodenorm_url="https://example.com/") + + def test_success_returns_list(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [ + {"identifier": "X:1", "label": "Alpha"}, + {"identifier": "X:2", "label": "Beta"}, + ], + } + with patch.object(nn, "normalize_curie", return_value=api_result): + result = nn.get_clique_identifiers("X:1") + assert len(result) == 2 + assert all(isinstance(x, Identifier) for x in result) + + def test_missing_key_returns_none(self): + nn = self._make_nn() + api_result = {"id": {"identifier": "X:1"}} # no equivalent_identifiers + with patch.object(nn, "normalize_curie", return_value=api_result): + result = nn.get_clique_identifiers("X:1") + assert result == [] + + def test_caching(self): + nn = self._make_nn() + api_result = { + "equivalent_identifiers": [{"identifier": "X:1"}], + } + with patch.object(nn, "normalize_curie", return_value=api_result) as mock_norm: + nn.get_clique_identifiers("X:1") + nn.get_clique_identifiers("X:1") + mock_norm.assert_called_once() + + +class TestGetBabelVersionMocked: + """Tests for get_babel_version().""" + + @staticmethod + def _status_response(payload): + response = Mock() + response.json = Mock(return_value=payload) + response.raise_for_status = Mock() + return response + + def test_reads_babel_version_from_status(self): + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._status_response({"babel_version": "2026jul22"}), + ) as mock_get: + assert nn.get_babel_version() == "2026jul22" + assert mock_get.call_args[0][0] == "https://example.com/nn/status" + + def test_offline_mode_makes_no_request(self): + """An empty URL short-circuits every lookup, including this one.""" + nn = NodeNorm(nodenorm_url="") + with patch("babel_explorer.core.nodenorm.requests.get") as mock_get: + assert nn.get_babel_version() is None + mock_get.assert_not_called() + + def test_unreachable_status_returns_none(self): + """A version check must never take down the command that called it.""" + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + side_effect=requests.ConnectionError("boom"), + ): + assert nn.get_babel_version() is None + + def test_status_without_babel_version_returns_none(self): + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._status_response({"biolink_model": {"tag": "v4.2.6"}}), + ): + assert nn.get_babel_version() is None + + def test_result_is_cached(self): + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._status_response({"babel_version": "2026jul22"}), + ) as mock_get: + nn.get_babel_version() + nn.get_babel_version() + mock_get.assert_called_once() + + def test_failure_is_cached_too(self): + """A failed lookup must not be retried on every subsequent call.""" + nn = NodeNorm(nodenorm_url="https://example.com/nn") + with patch( + "babel_explorer.core.nodenorm.requests.get", + side_effect=requests.ConnectionError("boom"), + ) as mock_get: + nn.get_babel_version() + nn.get_babel_version() + mock_get.assert_called_once() + + +# ========================================================================== +# Integration Tests — require real NodeNorm API +# ========================================================================== + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_normalize_curie_real_api(nodenorm, curie): + """normalize_curie returns a dict with expected keys.""" + result = nodenorm.normalize_curie(curie) + assert isinstance(result, dict) + assert "id" in result + assert "equivalent_identifiers" in result + assert "type" in result + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_identifier_real_api(nodenorm, curie): + """get_identifier returns an Identifier with non-empty label and biolink_type.""" + ident = nodenorm.get_identifier(curie) + assert isinstance(ident, Identifier) + assert ident.curie == curie + assert ident.label != "" + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_clique_identifiers_real_api(nodenorm, curie): + """get_clique_identifiers returns a non-empty list of Identifiers.""" + result = nodenorm.get_clique_identifiers(curie) + assert result is not None + assert len(result) > 0 + assert all(isinstance(x, Identifier) for x in result) + + +@pytest.mark.integration +@pytest.mark.parametrize("curie", VALID_CURIES) +def test_get_clique_identifiers_has_known_ids(nodenorm, curie): + """At least one equivalent identifier is returned.""" + result = nodenorm.get_clique_identifiers(curie) + assert len(result) >= 1 + + +@pytest.mark.integration +def test_normalize_curie_nonexistent(nodenorm): + """A made-up CURIE returns None.""" + result = nodenorm.normalize_curie("FAKENS:9999999999") + assert result is None + + +class TestNormalizeCuriesBatching: + """Many CURIEs must cost a handful of requests, not one each.""" + + @staticmethod + def _resp(payload): + r = Mock() + r.json.return_value = payload + r.raise_for_status = Mock() + return r + + def test_one_request_for_many_curies(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + curies = [f"X:{i}" for i in range(50)] + payload = {c: {"id": {"identifier": c}} for c in curies} + + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._resp(payload), + ) as mock_get: + nn.normalize_curies(curies) + assert mock_get.call_count == 1 + assert sorted(mock_get.call_args.kwargs["params"]["curie"]) == sorted( + curies + ) + + def test_chunks_above_the_batch_size(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + curies = [f"X:{i:04d}" for i in range(NORMALIZE_BATCH_SIZE * 2 + 1)] + + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._resp({}), + ) as mock_get: + nn.normalize_curies(curies) + assert mock_get.call_count == 3 + sizes = [len(c.kwargs["params"]["curie"]) for c in mock_get.call_args_list] + assert sizes == [NORMALIZE_BATCH_SIZE, NORMALIZE_BATCH_SIZE, 1] + + def test_prefetched_curies_are_not_refetched(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + payload = {"X:1": {"id": {"identifier": "X:1"}}} + + with patch( + "babel_explorer.core.nodenorm.requests.get", + return_value=self._resp(payload), + ) as mock_get: + nn.normalize_curies(["X:1"]) + nn.normalize_curie("X:1") + nn.get_identifier("X:1") + assert mock_get.call_count == 1 + + def test_unrecognised_curie_caches_none(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + with patch( + "babel_explorer.core.nodenorm.requests.get", return_value=self._resp({}) + ) as mock_get: + nn.normalize_curies(["MISSING:1"]) + assert nn.normalize_curie("MISSING:1") is None + assert mock_get.call_count == 1 + + def test_offline_mode_makes_no_requests(self): + nn = NodeNorm(nodenorm_url="") + with patch("babel_explorer.core.nodenorm.requests.get") as mock_get: + nn.normalize_curies(["X:1", "X:2"]) + assert nn.normalize_curie("X:1") is None + mock_get.assert_not_called() + + def test_http_error_is_not_cached(self): + nn = NodeNorm(nodenorm_url="https://example.com/") + bad = Mock() + bad.raise_for_status.side_effect = requests.HTTPError("500 Server Error") + + with patch("babel_explorer.core.nodenorm.requests.get", return_value=bad): + with pytest.raises(requests.HTTPError): + nn.normalize_curies(["X:1"]) + assert "X:1" not in nn._normalize_cache diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..f69e0e0 --- /dev/null +++ b/uv.lock @@ -0,0 +1,410 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "babel-explorer" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "click" }, + { name = "duckdb" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "rich" }, + { name = "tqdm" }, +] + +[package.dev-dependencies] +dev = [ + { name = "filelock" }, + { name = "pytest" }, + { name = "pytest-xdist", extra = ["psutil"] }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.3.1" }, + { name = "duckdb", specifier = ">=1.4.2" }, + { name = "python-dotenv", specifier = ">=1.0" }, + { name = "requests", specifier = ">=2.32.5" }, + { name = "rich", specifier = ">=13" }, + { name = "tqdm", specifier = ">=4.67.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "filelock", specifier = ">=3.16" }, + { name = "pytest", specifier = ">=8.3.5" }, + { name = "pytest-xdist", extras = ["psutil"], specifier = ">=3.6" }, + { name = "ruff", specifier = ">=0.11.0" }, +] + +[[package]] +name = "certifi" +version = "2026.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/27/c6491ff4954e58a10f69ad90aca8a1b6fe9c5d3c6f380907af3c37435b59/charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8", size = 206988, upload-time = "2025-10-14T04:40:33.79Z" }, + { url = "https://files.pythonhosted.org/packages/94/59/2e87300fe67ab820b5428580a53cad894272dbb97f38a7a814a2a1ac1011/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0", size = 147324, upload-time = "2025-10-14T04:40:34.961Z" }, + { url = "https://files.pythonhosted.org/packages/07/fb/0cf61dc84b2b088391830f6274cb57c82e4da8bbc2efeac8c025edb88772/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3", size = 142742, upload-time = "2025-10-14T04:40:36.105Z" }, + { url = "https://files.pythonhosted.org/packages/62/8b/171935adf2312cd745d290ed93cf16cf0dfe320863ab7cbeeae1dcd6535f/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc", size = 160863, upload-time = "2025-10-14T04:40:37.188Z" }, + { url = "https://files.pythonhosted.org/packages/09/73/ad875b192bda14f2173bfc1bc9a55e009808484a4b256748d931b6948442/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897", size = 157837, upload-time = "2025-10-14T04:40:38.435Z" }, + { url = "https://files.pythonhosted.org/packages/6d/fc/de9cce525b2c5b94b47c70a4b4fb19f871b24995c728e957ee68ab1671ea/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381", size = 151550, upload-time = "2025-10-14T04:40:40.053Z" }, + { url = "https://files.pythonhosted.org/packages/55/c2/43edd615fdfba8c6f2dfbd459b25a6b3b551f24ea21981e23fb768503ce1/charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815", size = 149162, upload-time = "2025-10-14T04:40:41.163Z" }, + { url = "https://files.pythonhosted.org/packages/03/86/bde4ad8b4d0e9429a4e82c1e8f5c659993a9a863ad62c7df05cf7b678d75/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0", size = 150019, upload-time = "2025-10-14T04:40:42.276Z" }, + { url = "https://files.pythonhosted.org/packages/1f/86/a151eb2af293a7e7bac3a739b81072585ce36ccfb4493039f49f1d3cae8c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161", size = 143310, upload-time = "2025-10-14T04:40:43.439Z" }, + { url = "https://files.pythonhosted.org/packages/b5/fe/43dae6144a7e07b87478fdfc4dbe9efd5defb0e7ec29f5f58a55aeef7bf7/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4", size = 162022, upload-time = "2025-10-14T04:40:44.547Z" }, + { url = "https://files.pythonhosted.org/packages/80/e6/7aab83774f5d2bca81f42ac58d04caf44f0cc2b65fc6db2b3b2e8a05f3b3/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89", size = 149383, upload-time = "2025-10-14T04:40:46.018Z" }, + { url = "https://files.pythonhosted.org/packages/4f/e8/b289173b4edae05c0dde07f69f8db476a0b511eac556dfe0d6bda3c43384/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569", size = 159098, upload-time = "2025-10-14T04:40:47.081Z" }, + { url = "https://files.pythonhosted.org/packages/d8/df/fe699727754cae3f8478493c7f45f777b17c3ef0600e28abfec8619eb49c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224", size = 152991, upload-time = "2025-10-14T04:40:48.246Z" }, + { url = "https://files.pythonhosted.org/packages/1a/86/584869fe4ddb6ffa3bd9f491b87a01568797fb9bd8933f557dba9771beaf/charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a", size = 99456, upload-time = "2025-10-14T04:40:49.376Z" }, + { url = "https://files.pythonhosted.org/packages/65/f6/62fdd5feb60530f50f7e38b4f6a1d5203f4d16ff4f9f0952962c044e919a/charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016", size = 106978, upload-time = "2025-10-14T04:40:50.844Z" }, + { url = "https://files.pythonhosted.org/packages/7a/9d/0710916e6c82948b3be62d9d398cb4fcf4e97b56d6a6aeccd66c4b2f2bd5/charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1", size = 99969, upload-time = "2025-10-14T04:40:52.272Z" }, + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "duckdb" +version = "1.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/9d/ab66a06e416d71b7bdcb9904cdf8d4db3379ef632bb8e9495646702d9718/duckdb-1.4.4.tar.gz", hash = "sha256:8bba52fd2acb67668a4615ee17ee51814124223de836d9e2fdcbc4c9021b3d3c", size = 18419763, upload-time = "2026-01-26T11:50:37.68Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/68/19233412033a2bc5a144a3f531f64e3548d4487251e3f16b56c31411a06f/duckdb-1.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5ba684f498d4e924c7e8f30dd157da8da34c8479746c5011b6c0e037e9c60ad2", size = 28883816, upload-time = "2026-01-26T11:49:01.009Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3e/cec70e546c298ab76d80b990109e111068d82cca67942c42328eaa7d6fdb/duckdb-1.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5536eb952a8aa6ae56469362e344d4e6403cc945a80bc8c5c2ebdd85d85eb64b", size = 15339662, upload-time = "2026-01-26T11:49:04.058Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f0/cf4241a040ec4f571859a738007ec773b642fbc27df4cbcf34b0c32ea559/duckdb-1.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:47dd4162da6a2be59a0aef640eb08d6360df1cf83c317dcc127836daaf3b7f7c", size = 13670044, upload-time = "2026-01-26T11:49:06.627Z" }, + { url = "https://files.pythonhosted.org/packages/11/64/de2bb4ec1e35ec9ebf6090a95b930fc56934a0ad6f34a24c5972a14a77ef/duckdb-1.4.4-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6cb357cfa3403910e79e2eb46c8e445bb1ee2fd62e9e9588c6b999df4256abc1", size = 18409951, upload-time = "2026-01-26T11:49:09.808Z" }, + { url = "https://files.pythonhosted.org/packages/79/a2/ac0f5ee16df890d141304bcd48733516b7202c0de34cd3555634d6eb4551/duckdb-1.4.4-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c25d5b0febda02b7944e94fdae95aecf952797afc8cb920f677b46a7c251955", size = 20411739, upload-time = "2026-01-26T11:49:12.652Z" }, + { url = "https://files.pythonhosted.org/packages/37/a2/9a3402edeedaecf72de05fe9ff7f0303d701b8dfc136aea4a4be1a5f7eee/duckdb-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:6703dd1bb650025b3771552333d305d62ddd7ff182de121483d4e042ea6e2e00", size = 12256972, upload-time = "2026-01-26T11:49:15.468Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/052ea6dcdf35b259fd182eff3efd8d75a071de4010c9807556098df137b9/duckdb-1.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:bf138201f56e5d6fc276a25138341b3523e2f84733613fc43f02c54465619a95", size = 13006696, upload-time = "2026-01-26T11:49:18.054Z" }, + { url = "https://files.pythonhosted.org/packages/58/33/beadaa69f8458afe466126f2c5ee48c4759cc9d5d784f8703d44e0b52c3c/duckdb-1.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ddcfd9c6ff234da603a1edd5fd8ae6107f4d042f74951b65f91bc5e2643856b3", size = 28896535, upload-time = "2026-01-26T11:49:21.232Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/82413f386df10467affc87f65bac095b7c88dbd9c767584164d5f4dc4cb8/duckdb-1.4.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6792ca647216bd5c4ff16396e4591cfa9b4a72e5ad7cdd312cec6d67e8431a7c", size = 15349716, upload-time = "2026-01-26T11:49:23.989Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/c13d396fd4e9bf970916dc5b4fea410c1b10fe531069aea65f1dcf849a71/duckdb-1.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1f8d55843cc940e36261689054f7dfb6ce35b1f5b0953b0d355b6adb654b0d52", size = 13672403, upload-time = "2026-01-26T11:49:26.741Z" }, + { url = "https://files.pythonhosted.org/packages/db/77/2446a0b44226bb95217748d911c7ca66a66ca10f6481d5178d9370819631/duckdb-1.4.4-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c65d15c440c31e06baaebfd2c06d71ce877e132779d309f1edf0a85d23c07e92", size = 18419001, upload-time = "2026-01-26T11:49:29.353Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a3/97715bba30040572fb15d02c26f36be988d48bc00501e7ac02b1d65ef9d0/duckdb-1.4.4-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b297eff642503fd435a9de5a9cb7db4eccb6f61d61a55b30d2636023f149855f", size = 20437385, upload-time = "2026-01-26T11:49:32.302Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0a/18b9167adf528cbe3867ef8a84a5f19f37bedccb606a8a9e59cfea1880c8/duckdb-1.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:d525de5f282b03aa8be6db86b1abffdceae5f1055113a03d5b50cd2fb8cf2ef8", size = 12267343, upload-time = "2026-01-26T11:49:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/f8/15/37af97f5717818f3d82d57414299c293b321ac83e048c0a90bb8b6a09072/duckdb-1.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:50f2eb173c573811b44aba51176da7a4e5c487113982be6a6a1c37337ec5fa57", size = 13007490, upload-time = "2026-01-26T11:49:37.413Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fe/64810fee20030f2bf96ce28b527060564864ce5b934b50888eda2cbf99dd/duckdb-1.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:337f8b24e89bc2e12dadcfe87b4eb1c00fd920f68ab07bc9b70960d6523b8bc3", size = 28899349, upload-time = "2026-01-26T11:49:40.294Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9b/3c7c5e48456b69365d952ac201666053de2700f5b0144a699a4dc6854507/duckdb-1.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0509b39ea7af8cff0198a99d206dca753c62844adab54e545984c2e2c1381616", size = 15350691, upload-time = "2026-01-26T11:49:43.242Z" }, + { url = "https://files.pythonhosted.org/packages/a6/7b/64e68a7b857ed0340045501535a0da99ea5d9d5ea3708fec0afb8663eb27/duckdb-1.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fb94de6d023de9d79b7edc1ae07ee1d0b4f5fa8a9dcec799650b5befdf7aafec", size = 13672311, upload-time = "2026-01-26T11:49:46.069Z" }, + { url = "https://files.pythonhosted.org/packages/09/5b/3e7aa490841784d223de61beb2ae64e82331501bf5a415dc87a0e27b4663/duckdb-1.4.4-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d636ceda422e7babd5e2f7275f6a0d1a3405e6a01873f00d38b72118d30c10b", size = 18422740, upload-time = "2026-01-26T11:49:49.034Z" }, + { url = "https://files.pythonhosted.org/packages/53/32/256df3dbaa198c58539ad94f9a41e98c2c8ff23f126b8f5f52c7dcd0a738/duckdb-1.4.4-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7df7351328ffb812a4a289732f500d621e7de9942a3a2c9b6d4afcf4c0e72526", size = 20435578, upload-time = "2026-01-26T11:49:51.946Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f0/620323fd87062ea43e527a2d5ed9e55b525e0847c17d3b307094ddab98a2/duckdb-1.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:6fb1225a9ea5877421481d59a6c556a9532c32c16c7ae6ca8d127e2b878c9389", size = 12268083, upload-time = "2026-01-26T11:49:54.615Z" }, + { url = "https://files.pythonhosted.org/packages/e5/07/a397fdb7c95388ba9c055b9a3d38dfee92093f4427bc6946cf9543b1d216/duckdb-1.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:f28a18cc790217e5b347bb91b2cab27aafc557c58d3d8382e04b4fe55d0c3f66", size = 13006123, upload-time = "2026-01-26T11:49:57.092Z" }, + { url = "https://files.pythonhosted.org/packages/97/a6/f19e2864e651b0bd8e4db2b0c455e7e0d71e0d4cd2cd9cc052f518e43eb3/duckdb-1.4.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:25874f8b1355e96178079e37312c3ba6d61a2354f51319dae860cf21335c3a20", size = 28909554, upload-time = "2026-01-26T11:50:00.107Z" }, + { url = "https://files.pythonhosted.org/packages/0e/93/8a24e932c67414fd2c45bed83218e62b73348996bf859eda020c224774b2/duckdb-1.4.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:452c5b5d6c349dc5d1154eb2062ee547296fcbd0c20e9df1ed00b5e1809089da", size = 15353804, upload-time = "2026-01-26T11:50:03.382Z" }, + { url = "https://files.pythonhosted.org/packages/62/13/e5378ff5bb1d4397655d840b34b642b1b23cdd82ae19599e62dc4b9461c9/duckdb-1.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8e5c2d8a0452df55e092959c0bfc8ab8897ac3ea0f754cb3b0ab3e165cd79aff", size = 13676157, upload-time = "2026-01-26T11:50:06.232Z" }, + { url = "https://files.pythonhosted.org/packages/2d/94/24364da564b27aeebe44481f15bd0197a0b535ec93f188a6b1b98c22f082/duckdb-1.4.4-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1af6e76fe8bd24875dc56dd8e38300d64dc708cd2e772f67b9fbc635cc3066a3", size = 18426882, upload-time = "2026-01-26T11:50:08.97Z" }, + { url = "https://files.pythonhosted.org/packages/26/0a/6ae31b2914b4dc34243279b2301554bcbc5f1a09ccc82600486c49ab71d1/duckdb-1.4.4-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0440f59e0cd9936a9ebfcf7a13312eda480c79214ffed3878d75947fc3b7d6d", size = 20435641, upload-time = "2026-01-26T11:50:12.188Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b1/fd5c37c53d45efe979f67e9bd49aaceef640147bb18f0699a19edd1874d6/duckdb-1.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:59c8d76016dde854beab844935b1ec31de358d4053e792988108e995b18c08e7", size = 12762360, upload-time = "2026-01-26T11:50:14.76Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2d/13e6024e613679d8a489dd922f199ef4b1d08a456a58eadd96dc2f05171f/duckdb-1.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:53cd6423136ab44383ec9955aefe7599b3fb3dd1fe006161e6396d8167e0e0d4", size = 13458633, upload-time = "2026-01-26T11:50:17.657Z" }, +] + +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + +[[package]] +name = "filelock" +version = "3.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/18/a1fd2231c679dcb9726204645721b12498aeac28e1ad0601038f94b42556/filelock-3.25.0.tar.gz", hash = "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3", size = 40158, upload-time = "2026-03-01T15:08:45.916Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/0b/de6f54d4a8bedfe8645c41497f3c18d749f0bd3218170c667bf4b81d0cdd/filelock-3.25.0-py3-none-any.whl", hash = "sha256:5ccf8069f7948f494968fc0713c10e5c182a9c9d9eef3a636307a20c2490f047", size = 26427, upload-time = "2026-03-01T15:08:44.593Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + +[package.optional-dependencies] +psutil = [ + { name = "psutil" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "requests" +version = "2.32.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, +] + +[[package]] +name = "rich" +version = "14.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/04/eab13a954e763b0606f460443fcbf6bb5a0faf06890ea3754ff16523dce5/ruff-0.15.2.tar.gz", hash = "sha256:14b965afee0969e68bb871eba625343b8673375f457af4abe98553e8bbb98342", size = 4558148, upload-time = "2026-02-19T22:32:20.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/70/3a4dc6d09b13cb3e695f28307e5d889b2e1a66b7af9c5e257e796695b0e6/ruff-0.15.2-py3-none-linux_armv6l.whl", hash = "sha256:120691a6fdae2f16d65435648160f5b81a9625288f75544dc40637436b5d3c0d", size = 10430565, upload-time = "2026-02-19T22:32:41.824Z" }, + { url = "https://files.pythonhosted.org/packages/71/0b/bb8457b56185ece1305c666dc895832946d24055be90692381c31d57466d/ruff-0.15.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a89056d831256099658b6bba4037ac6dd06f49d194199215befe2bb10457ea5e", size = 10820354, upload-time = "2026-02-19T22:32:07.366Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c1/e0532d7f9c9e0b14c46f61b14afd563298b8b83f337b6789ddd987e46121/ruff-0.15.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e36dee3a64be0ebd23c86ffa3aa3fd3ac9a712ff295e192243f814a830b6bd87", size = 10170767, upload-time = "2026-02-19T22:32:13.188Z" }, + { url = "https://files.pythonhosted.org/packages/47/e8/da1aa341d3af017a21c7a62fb5ec31d4e7ad0a93ab80e3a508316efbcb23/ruff-0.15.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9fb47b6d9764677f8c0a193c0943ce9a05d6763523f132325af8a858eadc2b9", size = 10529591, upload-time = "2026-02-19T22:32:02.547Z" }, + { url = "https://files.pythonhosted.org/packages/93/74/184fbf38e9f3510231fbc5e437e808f0b48c42d1df9434b208821efcd8d6/ruff-0.15.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f376990f9d0d6442ea9014b19621d8f2aaf2b8e39fdbfc79220b7f0c596c9b80", size = 10260771, upload-time = "2026-02-19T22:32:36.938Z" }, + { url = "https://files.pythonhosted.org/packages/05/ac/605c20b8e059a0bc4b42360414baa4892ff278cec1c91fff4be0dceedefd/ruff-0.15.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dcc987551952d73cbf5c88d9fdee815618d497e4df86cd4c4824cc59d5dd75f", size = 11045791, upload-time = "2026-02-19T22:32:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/fd/52/db6e419908f45a894924d410ac77d64bdd98ff86901d833364251bd08e22/ruff-0.15.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:42a47fd785cbe8c01b9ff45031af875d101b040ad8f4de7bbb716487c74c9a77", size = 11879271, upload-time = "2026-02-19T22:32:29.305Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d8/7992b18f2008bdc9231d0f10b16df7dda964dbf639e2b8b4c1b4e91b83af/ruff-0.15.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbe9f49354866e575b4c6943856989f966421870e85cd2ac94dccb0a9dcb2fea", size = 11303707, upload-time = "2026-02-19T22:32:22.492Z" }, + { url = "https://files.pythonhosted.org/packages/d7/02/849b46184bcfdd4b64cde61752cc9a146c54759ed036edd11857e9b8443b/ruff-0.15.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7a672c82b5f9887576087d97be5ce439f04bbaf548ee987b92d3a7dede41d3a", size = 11149151, upload-time = "2026-02-19T22:32:44.234Z" }, + { url = "https://files.pythonhosted.org/packages/70/04/f5284e388bab60d1d3b99614a5a9aeb03e0f333847e2429bebd2aaa1feec/ruff-0.15.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ecc64f46f7019e2bcc3cdc05d4a7da958b629a5ab7033195e11a438403d956", size = 11091132, upload-time = "2026-02-19T22:32:24.691Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ae/88d844a21110e14d92cf73d57363fab59b727ebeabe78009b9ccb23500af/ruff-0.15.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8dcf243b15b561c655c1ef2f2b0050e5d50db37fe90115507f6ff37d865dc8b4", size = 10504717, upload-time = "2026-02-19T22:32:26.75Z" }, + { url = "https://files.pythonhosted.org/packages/64/27/867076a6ada7f2b9c8292884ab44d08fd2ba71bd2b5364d4136f3cd537e1/ruff-0.15.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dab6941c862c05739774677c6273166d2510d254dac0695c0e3f5efa1b5585de", size = 10263122, upload-time = "2026-02-19T22:32:10.036Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ef/faf9321d550f8ebf0c6373696e70d1758e20ccdc3951ad7af00c0956be7c/ruff-0.15.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1b9164f57fc36058e9a6806eb92af185b0697c9fe4c7c52caa431c6554521e5c", size = 10735295, upload-time = "2026-02-19T22:32:39.227Z" }, + { url = "https://files.pythonhosted.org/packages/2f/55/e8089fec62e050ba84d71b70e7834b97709ca9b7aba10c1a0b196e493f97/ruff-0.15.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:80d24fcae24d42659db7e335b9e1531697a7102c19185b8dc4a028b952865fd8", size = 11241641, upload-time = "2026-02-19T22:32:34.617Z" }, + { url = "https://files.pythonhosted.org/packages/23/01/1c30526460f4d23222d0fabd5888868262fd0e2b71a00570ca26483cd993/ruff-0.15.2-py3-none-win32.whl", hash = "sha256:fd5ff9e5f519a7e1bd99cbe8daa324010a74f5e2ebc97c6242c08f26f3714f6f", size = 10507885, upload-time = "2026-02-19T22:32:15.635Z" }, + { url = "https://files.pythonhosted.org/packages/5c/10/3d18e3bbdf8fc50bbb4ac3cc45970aa5a9753c5cb51bf9ed9a3cd8b79fa3/ruff-0.15.2-py3-none-win_amd64.whl", hash = "sha256:d20014e3dfa400f3ff84830dfb5755ece2de45ab62ecea4af6b7262d0fb4f7c5", size = 11623725, upload-time = "2026-02-19T22:32:04.947Z" }, + { url = "https://files.pythonhosted.org/packages/6d/78/097c0798b1dab9f8affe73da9642bb4500e098cb27fd8dc9724816ac747b/ruff-0.15.2-py3-none-win_arm64.whl", hash = "sha256:cabddc5822acdc8f7b5527b36ceac55cc51eec7b1946e60181de8fe83ca8876e", size = 10941649, upload-time = "2026-02-19T22:32:18.108Z" }, +] + +[[package]] +name = "tqdm" +version = "4.67.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +]