From 23a04e820b9403abe85b4b61c3accf67522c51a0 Mon Sep 17 00:00:00 2001 From: jonas Date: Tue, 8 Sep 2026 11:40:14 +0200 Subject: [PATCH 1/6] added test to see if each database builds on autobumb --- .github/workflows/herpesdrg-autobump.yml | 52 ++++++++++++++++++ .../workflows/hsv-daehne-jaki-autobump.yml | 52 ++++++++++++++++++ .github/workflows/stanford-hiv-autobump.yml | 54 ++++++++++++++++++- .gitignore | 2 +- README.md | 2 + 5 files changed, 160 insertions(+), 2 deletions(-) diff --git a/.github/workflows/herpesdrg-autobump.yml b/.github/workflows/herpesdrg-autobump.yml index def2b66..1a7e411 100644 --- a/.github/workflows/herpesdrg-autobump.yml +++ b/.github/workflows/herpesdrg-autobump.yml @@ -100,6 +100,50 @@ jobs: run: | python scripts/build_metadata_manifest.py + - name: Set up respro (bioconda) + if: steps.update_check.outputs.should_update == 'true' + uses: mamba-org/setup-micromamba@v2 + with: + environment-name: respro + condarc: | + channels: + - conda-forge + - bioconda + create-args: >- + respro==0.1.1 + init-shell: bash + cache-environment: true + + - name: respro build QC + id: respro_qc + if: steps.update_check.outputs.should_update == 'true' + shell: bash -el {0} + continue-on-error: true + run: | + python scripts/respro_build_qc.py \ + --source-name herpesdrg \ + --output-dir databases/herpesdrg/output \ + --work-dir /tmp/respro_qc + echo "qc_exit=$?" >> "$GITHUB_OUTPUT" + env: + NCBI_EUTILS_API_KEY: ${{ secrets.NCBI_EUTILS_API_KEY }} + + - name: Read respro QC summary + id: qc_summary + if: steps.update_check.outputs.should_update == 'true' + shell: bash + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "block<<${EOF}" >> "$GITHUB_OUTPUT" + cat /tmp/respro_qc/qc_summary.md >> "$GITHUB_OUTPUT" + echo "${EOF}" >> "$GITHUB_OUTPUT" + qc_failed="false" + if [ -f /tmp/respro_qc/qc_summary.json ]; then + status=$(python -c "import json;print(json.load(open('/tmp/respro_qc/qc_summary.json'))['status'])" 2>/dev/null || echo "") + if [ "$status" = "failed" ]; then qc_failed="true"; fi + fi + echo "qc_failed=${qc_failed}" >> "$GITHUB_OUTPUT" + - name: No update detected if: steps.update_check.outputs.should_update != 'true' run: | @@ -122,6 +166,8 @@ jobs: - Source commit SHA: ${{ steps.source.outputs.source_commit_sha }} - Source URL: ${{ steps.source.outputs.source_url }} + ${{ steps.qc_summary.outputs.block }} + This PR was generated by the autobump workflow. labels: autobump add-paths: | @@ -130,3 +176,9 @@ jobs: databases/herpesdrg/output/metadata.json databases/herpesdrg/output/non-migrated-rules.txt databases/manifest.json + + - name: Mark workflow failed on respro build error + if: steps.update_check.outputs.should_update == 'true' && steps.qc_summary.outputs.qc_failed == 'true' + run: | + echo "respro build QC failed for herpesdrg; see the PR body and the 'respro build QC' step log." + exit 1 diff --git a/.github/workflows/hsv-daehne-jaki-autobump.yml b/.github/workflows/hsv-daehne-jaki-autobump.yml index dded91b..2d874fe 100644 --- a/.github/workflows/hsv-daehne-jaki-autobump.yml +++ b/.github/workflows/hsv-daehne-jaki-autobump.yml @@ -117,6 +117,50 @@ jobs: run: | python scripts/build_metadata_manifest.py + - name: Set up respro (bioconda) + if: steps.update_check.outputs.should_update == 'true' + uses: mamba-org/setup-micromamba@v2 + with: + environment-name: respro + condarc: | + channels: + - conda-forge + - bioconda + create-args: >- + respro==0.1.1 + init-shell: bash + cache-environment: true + + - name: respro build QC + id: respro_qc + if: steps.update_check.outputs.should_update == 'true' + shell: bash -el {0} + continue-on-error: true + run: | + python scripts/respro_build_qc.py \ + --source-name hsv_daehne_jaki \ + --output-dir databases/hsv_daehne_jaki/output \ + --work-dir /tmp/respro_qc + echo "qc_exit=$?" >> "$GITHUB_OUTPUT" + env: + NCBI_EUTILS_API_KEY: ${{ secrets.NCBI_EUTILS_API_KEY }} + + - name: Read respro QC summary + id: qc_summary + if: steps.update_check.outputs.should_update == 'true' + shell: bash + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "block<<${EOF}" >> "$GITHUB_OUTPUT" + cat /tmp/respro_qc/qc_summary.md >> "$GITHUB_OUTPUT" + echo "${EOF}" >> "$GITHUB_OUTPUT" + qc_failed="false" + if [ -f /tmp/respro_qc/qc_summary.json ]; then + status=$(python -c "import json;print(json.load(open('/tmp/respro_qc/qc_summary.json'))['status'])" 2>/dev/null || echo "") + if [ "$status" = "failed" ]; then qc_failed="true"; fi + fi + echo "qc_failed=${qc_failed}" >> "$GITHUB_OUTPUT" + - name: No update detected if: steps.update_check.outputs.should_update != 'true' run: | @@ -139,6 +183,8 @@ jobs: - Zenodo updated timestamp: ${{ steps.zenodo.outputs.updated }} - Source file: ${{ steps.zenodo.outputs.source_url }} + ${{ steps.qc_summary.outputs.block }} + This PR was generated by the autobump workflow. labels: autobump add-paths: | @@ -147,3 +193,9 @@ jobs: databases/hsv_daehne_jaki/output/metadata.json databases/hsv_daehne_jaki/output/non-migrated-rules.txt databases/manifest.json + + - name: Mark workflow failed on respro build error + if: steps.update_check.outputs.should_update == 'true' && steps.qc_summary.outputs.qc_failed == 'true' + run: | + echo "respro build QC failed for hsv_daehne_jaki; see the PR body and the 'respro build QC' step log." + exit 1 diff --git a/.github/workflows/stanford-hiv-autobump.yml b/.github/workflows/stanford-hiv-autobump.yml index 72f3e12..1407c8b 100644 --- a/.github/workflows/stanford-hiv-autobump.yml +++ b/.github/workflows/stanford-hiv-autobump.yml @@ -120,6 +120,50 @@ jobs: run: | python scripts/build_metadata_manifest.py + - name: Set up respro (bioconda) + if: steps.update_check.outputs.should_update == 'true' + uses: mamba-org/setup-micromamba@v2 + with: + environment-name: respro + condarc: | + channels: + - conda-forge + - bioconda + create-args: >- + respro==0.1.1 + init-shell: bash + cache-environment: true + + - name: respro build QC + id: respro_qc + if: steps.update_check.outputs.should_update == 'true' + shell: bash -el {0} + continue-on-error: true + run: | + python scripts/respro_build_qc.py \ + --source-name stanford-hiv \ + --output-dir databases/stanford-hiv/output \ + --work-dir /tmp/respro_qc + echo "qc_exit=$?" >> "$GITHUB_OUTPUT" + env: + NCBI_EUTILS_API_KEY: ${{ secrets.NCBI_EUTILS_API_KEY }} + + - name: Read respro QC summary + id: qc_summary + if: steps.update_check.outputs.should_update == 'true' + shell: bash + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "block<<${EOF}" >> "$GITHUB_OUTPUT" + cat /tmp/respro_qc/qc_summary.md >> "$GITHUB_OUTPUT" + echo "${EOF}" >> "$GITHUB_OUTPUT" + qc_failed="false" + if [ -f /tmp/respro_qc/qc_summary.json ]; then + status=$(python -c "import json;print(json.load(open('/tmp/respro_qc/qc_summary.json'))['status'])" 2>/dev/null || echo "") + if [ "$status" = "failed" ]; then qc_failed="true"; fi + fi + echo "qc_failed=${qc_failed}" >> "$GITHUB_OUTPUT" + - name: No update detected if: steps.update_check.outputs.should_update != 'true' run: | @@ -142,6 +186,8 @@ jobs: - Source version: ${{ steps.source.outputs.source_version }} - Source URL: ${{ steps.source.outputs.source_url }} + ${{ steps.qc_summary.outputs.block }} + This PR was generated by the autobump workflow. labels: autobump add-paths: | @@ -149,4 +195,10 @@ jobs: databases/stanford-hiv/output/formula-rules.tsv databases/stanford-hiv/output/metadata.json databases/stanford-hiv/output/non-migrated-rules.txt - databases/manifest.json \ No newline at end of file + databases/manifest.json + + - name: Mark workflow failed on respro build error + if: steps.update_check.outputs.should_update == 'true' && steps.qc_summary.outputs.qc_failed == 'true' + run: | + echo "respro build QC failed for stanford-hiv; see the PR body and the 'respro build QC' step log." + exit 1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index fe698b6..487fea5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ __pycache__/ .pytest_cache/ .mypy_cache/ - +databases/charmd/ .act/ .DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 133505b..cfed825 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ databases/ manifest.json # generated discovery file, see below / + example/ + example.fasta scripts/ convert.py requirements.txt From b16d7f8aed0eb971ed3b6e4d245cb3f1e9d8cfec Mon Sep 17 00:00:00 2001 From: jonas Date: Tue, 8 Sep 2026 11:40:36 +0200 Subject: [PATCH 2/6] added qc script --- scripts/respro_build_qc.py | 505 +++++++++++++++++++++++++++++++++++++ 1 file changed, 505 insertions(+) create mode 100644 scripts/respro_build_qc.py diff --git a/scripts/respro_build_qc.py b/scripts/respro_build_qc.py new file mode 100644 index 0000000..2d77a04 --- /dev/null +++ b/scripts/respro_build_qc.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 +"""Build a ResPro project database from a source's converted artifacts. + +This is the autobump quality-control step: it takes a database's generated +``rules.tsv`` (plus optional ``formula-rules.tsv`` and ``metadata.json``), +fetches the GenBank reference record(s) named by the ``reference_identifier`` +column from NCBI, runs ``respro init`` to build a SQLite project database, and +emits a structured summary of the build outcome (success / hard-failure / +skipped rules). The summary is consumed by the autobump workflow and rendered +into the pull-request body so a failed or degraded build is explicit. + +The script intentionally does NOT modify any repository artifacts. It writes +the GenBank files, the built ``.db``, and the summary into a working directory +(default: a fresh temp directory) that the caller may discard after the run. + +Network access is required only for the NCBI GenBank fetch; ``respro init`` is +invoked with ``--no-additional-info`` so no PubChem/PubMed enrichment happens +and the build is deterministic given fixed inputs. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + +NCBI_EFETCH_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" +# Matches a single NCBI nucleotide accession with an optional version suffix. +# Used to validate path safety before writing .gb files. +ACCESSION_RE = re.compile(r"^[A-Za-z0-9_]+(\.\d+)?$") + +# respro log signals we parse to classify the build outcome. These are matched +# against the combined stdout+stderr of ``respro init``. +HARD_FAIL_RE = re.compile(r"Error:\s*Rules validation failed", re.MULTILINE) +REF_AA_SKIP_RE = re.compile( + r"(\d+)\s+position\(s\) with rules have reference AA mismatches", re.MULTILINE +) +FEATURE_SKIP_RE = re.compile( + r"(\d+)\s+rule\(s\) skipped\s*\u2014\s*feature\(s\) not found", re.MULTILINE +) +FORMULA_SKIP_RE = re.compile( + r"(\d+)\s+formula rule\(s\) skipped", re.MULTILINE +) +SUCCESS_RE = re.compile(r"Project initialised:\s*(\S+)", re.MULTILINE) + + +def eprint(message: str) -> None: + print(message, file=sys.stderr) + + +def norm(value: object) -> str: + if value is None: + return "" + return str(value).strip() + + +def collect_reference_identifiers(rules_path: Path) -> list[str]: + """Return the unique, non-empty ``reference_identifier`` values in order.""" + seen: set[str] = set() + accessions: list[str] = [] + with rules_path.open("r", encoding="utf-8", newline="") as fh: + reader = csv.DictReader(fh, delimiter="\t") + if reader.fieldnames is None or "reference_identifier" not in reader.fieldnames: + raise ValueError( + f"{rules_path} is missing a 'reference_identifier' column" + ) + for row in reader: + acc = norm(row.get("reference_identifier")) + if not acc or acc in seen: + continue + if not ACCESSION_RE.match(acc): + raise ValueError( + f"{rules_path} contains an invalid reference_identifier: {acc!r}" + ) + seen.add(acc) + accessions.append(acc) + if not accessions: + raise ValueError(f"{rules_path} contains no reference_identifier values") + return accessions + + +def fetch_genbank( + accession: str, + dest_dir: Path, + api_key: str = "", + retries: int = 3, + backoff: float = 2.0, +) -> Path: + """Fetch one GenBank record from NCBI eutils into ``dest_dir``. + + Uses ``rettype=gbwithparts`` so segmented/large records are returned in + full. Retries with exponential backoff on transient HTTP errors. + """ + dest_dir.mkdir(parents=True, exist_ok=True) + out_path = dest_dir / f"{accession}.gb" + params = { + "db": "nuccore", + "id": accession, + "rettype": "gbwithparts", + "retmode": "text", + } + if api_key: + params["api_key"] = api_key + url = f"{NCBI_EFETCH_URL}?{urllib.parse.urlencode(params)}" + + last_error: str = "" + for attempt in range(1, retries + 1): + try: + request = urllib.request.Request( + url, + headers={"User-Agent": "respro-databases-qc/1.0"}, + ) + with urllib.request.urlopen(request, timeout=120) as response: + payload = response.read() + if not payload.strip(): + raise RuntimeError("empty response from NCBI efetch") + out_path.write_bytes(payload) + return out_path + except Exception as exc: # noqa: BLE001 - retry any fetch failure + last_error = str(exc) + if attempt < retries: + wait = backoff ** attempt + eprint( + f"WARNING: NCBI fetch for {accession} failed (attempt " + f"{attempt}/{retries}): {last_error}; retrying in {wait:.0f}s" + ) + time.sleep(wait) + raise RuntimeError(f"Could not fetch GenBank for {accession}: {last_error}") + + +def build_respro_command( + source_name: str, + rules_path: Path, + genbank_paths: list[Path], + output_db: Path, + formula_rules_path: Path | None, + metadata_path: Path | None, + respro_bin: str, +) -> list[str]: + cmd: list[str] = [ + respro_bin, + "init", + "--name", + source_name, + "--rules", + str(rules_path), + "--output", + str(output_db), + "--no-additional-info", + "--overwrite", + ] + for gb in genbank_paths: + cmd.extend(["--genbank", str(gb)]) + if formula_rules_path and formula_rules_path.is_file(): + cmd.extend(["--formula-rules", str(formula_rules_path)]) + if metadata_path and metadata_path.is_file(): + cmd.extend(["--metadata", str(metadata_path)]) + return cmd + + +def parse_build_log(log: str) -> dict[str, Any]: + """Classify a ``respro init`` log + exit code into a structured result. + + respro logs through ``rich``, which wraps long lines at the terminal width. + We collapse all whitespace (including newlines) into single spaces before + regex matching so wrapped WARNING/Error messages still match. + """ + # Collapse runs of whitespace (incl. newlines) into single spaces for + # robust matching against rich-wrapped log output. + flat = re.sub(r"\s+", " ", log) + + ref_aa = 0 + feature_skip = 0 + formula_skip = 0 + m = REF_AA_SKIP_RE.search(flat) + if m: + ref_aa = int(m.group(1)) + m = FEATURE_SKIP_RE.search(flat) + if m: + feature_skip = int(m.group(1)) + m = FORMULA_SKIP_RE.search(flat) + if m: + formula_skip = int(m.group(1)) + + hard_fail = HARD_FAIL_RE.search(flat) is not None + success = SUCCESS_RE.search(flat) is not None + + # Extract a concise error excerpt for hard failures. + error_excerpt = "" + if hard_fail: + lines = log.splitlines() + for i, line in enumerate(lines): + if "Error:" in line: + # Grab the error line plus up to 3 following indented detail lines. + excerpt = [line.strip()] + for follow in lines[i + 1 : i + 4]: + if not follow.strip(): + break + excerpt.append(follow.strip()) + error_excerpt = "\n".join(excerpt) + break + + return { + "ref_aa_skipped": ref_aa, + "feature_skipped": feature_skip, + "formula_skipped": formula_skip, + "hard_fail": hard_fail, + "success": success, + "error_excerpt": error_excerpt, + } + + +def render_summary_markdown( + source_name: str, + exit_code: int, + parsed: dict[str, Any], + log_path: Path, + imported_rules: int | None, +) -> str: + """Render the ``## respro build QC`` markdown block for the PR body.""" + total_skipped = ( + parsed["ref_aa_skipped"] + + parsed["feature_skipped"] + + parsed["formula_skipped"] + ) + + if parsed["hard_fail"] or exit_code != 0: + status_line = "\u26a0\ufe0f **`respro init` FAILED** \u2014 the regenerated TSVs do not build and must be fixed before merge." + elif total_skipped > 0: + status_line = ( + f"\u26a0\ufe0f `respro init` succeeded with {total_skipped} skipped rule(s)." + ) + else: + status_line = "\u2705 `respro init` succeeded (all rules imported)." + + lines = [ + "## respro build QC", + "", + f"**Source:** `{source_name}`", + f"**Status:** {status_line}", + f"**Exit code:** `{exit_code}`", + ] + + if imported_rules is not None: + lines.append(f"**Imported single rules:** {imported_rules}") + + lines.extend( + [ + f"**Skipped \u2014 reference AA mismatch:** {parsed['ref_aa_skipped']}", + f"**Skipped \u2014 feature not in GenBank:** {parsed['feature_skipped']}", + f"**Skipped \u2014 formula member missing:** {parsed['formula_skipped']}", + f"**Full log:** `{log_path}`", + ] + ) + + if parsed["error_excerpt"]: + lines.extend(["", "```", parsed["error_excerpt"], "```"]) + + return "\n".join(lines) + + +def count_imported_rules(db_path: Path, respro_bin: str) -> int | None: + """Best-effort count of imported single rules via ``respro manage``.""" + if not db_path.is_file(): + return None + try: + result = subprocess.run( + [respro_bin, "manage", "database", str(db_path), "--list-single"], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if result.returncode != 0: + return None + # Count non-empty output lines (the table body); this is an approximate + # count sufficient for a QC summary, not a precise audit. + return sum(1 for line in result.stdout.splitlines() if line.strip()) + except Exception: # noqa: BLE001 - count is best-effort + return None + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--source-name", + required=True, + help="Source database name (e.g. stanford-hiv). Used as the respro project name.", + ) + parser.add_argument( + "--output-dir", + required=True, + help="Directory containing the source's rules.tsv / metadata.json / formula-rules.tsv.", + ) + parser.add_argument( + "--respro-bin", + default="respro", + help="Path to the respro executable (default: looks up `respro` on PATH).", + ) + parser.add_argument( + "--work-dir", + default=None, + help="Working directory for GenBank fetch + built .db + logs (default: fresh temp dir).", + ) + parser.add_argument( + "--ncbi-api-key", + default="", + help=( + "Optional NCBI eutils API key (raises the rate limit). " + "Falls back to the NCBI_EUTILS_API_KEY environment variable." + ), + ) + parser.add_argument( + "--summary-md", + default=None, + help="Path to write the markdown summary (default: /qc_summary.md).", + ) + parser.add_argument( + "--summary-json", + default=None, + help="Path to write the machine-readable summary (default: /qc_summary.json).", + ) + return parser.parse_args() + + +def write_failure_summary( + source_name: str, + message: str, + summary_md_path: Path, + summary_json_path: Path, +) -> None: + """Emit a failed QC summary for infrastructure errors (exit 2 path). + + Ensures the workflow's summary-reading step always has a markdown file to + interpolate into the PR body, even when the build never ran. + """ + summary_md = ( + "## respro build QC\n\n" + f"**Source:** `{source_name}`\n" + f"**Status:** \u26a0\ufe0f **`respro init` FAILED** \u2014 the QC step " + f"could not complete.\n" + f"**Exit code:** `2`\n" + f"**Error:** {message}\n" + ) + summary_md_path.write_text(summary_md + "\n", encoding="utf-8") + summary_json = { + "source_name": source_name, + "status": "failed", + "exit_code": 2, + "imported_rules": None, + "ref_aa_skipped": 0, + "feature_skipped": 0, + "formula_skipped": 0, + "error_excerpt": message, + "log_path": "", + "db_path": "", + "summary_md_path": str(summary_md_path), + } + summary_json_path.write_text( + json.dumps(summary_json, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output_dir).resolve() + rules_path = output_dir / "rules.tsv" + formula_rules_path = output_dir / "formula-rules.tsv" + metadata_path = output_dir / "metadata.json" + + ncbi_api_key = args.ncbi_api_key or os.environ.get("NCBI_EUTILS_API_KEY", "") + + work_dir = Path(args.work_dir).resolve() if args.work_dir else Path( + tempfile.mkdtemp(prefix="respro_qc_") + ) + work_dir.mkdir(parents=True, exist_ok=True) + genbank_dir = work_dir / "genbank" + db_path = work_dir / f"{args.source_name}.db" + log_path = work_dir / "respro_init.log" + summary_md_path = Path(args.summary_md) if args.summary_md else work_dir / "qc_summary.md" + summary_json_path = ( + Path(args.summary_json) if args.summary_json else work_dir / "qc_summary.json" + ) + + if not rules_path.is_file(): + msg = f"rules.tsv not found: {rules_path}" + eprint(f"ERROR: {msg}") + write_failure_summary(args.source_name, msg, summary_md_path, summary_json_path) + return 2 + + try: + accessions = collect_reference_identifiers(rules_path) + except ValueError as exc: + msg = str(exc) + eprint(f"ERROR: {msg}") + write_failure_summary(args.source_name, msg, summary_md_path, summary_json_path) + return 2 + + eprint(f"Source: {args.source_name}") + eprint(f"Reference accessions: {', '.join(accessions)}") + + # Fetch GenBank references from NCBI. + genbank_paths: list[Path] = [] + try: + for acc in accessions: + eprint(f"Fetching GenBank {acc} \u2026") + gb = fetch_genbank(acc, genbank_dir, api_key=ncbi_api_key) + genbank_paths.append(gb) + except RuntimeError as exc: + msg = str(exc) + eprint(f"ERROR: {msg}") + write_failure_summary(args.source_name, msg, summary_md_path, summary_json_path) + return 2 + + # Build the respro command and run it. + cmd = build_respro_command( + source_name=args.source_name, + rules_path=rules_path, + genbank_paths=genbank_paths, + output_db=db_path, + formula_rules_path=formula_rules_path, + metadata_path=metadata_path, + respro_bin=args.respro_bin, + ) + eprint(f"Running: {' '.join(cmd)}") + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=600, + check=False, + ) + log = result.stdout + result.stderr + log_path.write_text(log, encoding="utf-8") + + parsed = parse_build_log(log) + imported_rules = count_imported_rules(db_path, args.respro_bin) + + summary_md = render_summary_markdown( + source_name=args.source_name, + exit_code=result.returncode, + parsed=parsed, + log_path=log_path, + imported_rules=imported_rules, + ) + summary_md_path.write_text(summary_md + "\n", encoding="utf-8") + + summary_json: dict[str, Any] = { + "source_name": args.source_name, + "status": ( + "failed" if parsed["hard_fail"] or result.returncode != 0 + else "skipped" if ( + parsed["ref_aa_skipped"] + + parsed["feature_skipped"] + + parsed["formula_skipped"] + ) + else "ok" + ), + "exit_code": result.returncode, + "imported_rules": imported_rules, + "ref_aa_skipped": parsed["ref_aa_skipped"], + "feature_skipped": parsed["feature_skipped"], + "formula_skipped": parsed["formula_skipped"], + "error_excerpt": parsed["error_excerpt"], + "log_path": str(log_path), + "db_path": str(db_path), + "summary_md_path": str(summary_md_path), + } + summary_json_path.write_text( + json.dumps(summary_json, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + eprint(f"Written {summary_md_path}") + eprint(f"Written {summary_json_path}") + + # Exit non-zero ONLY on hard respro failure. Soft skips are exit 0 so the + # PR step still runs; the workflow separately marks the run red on failure. + if parsed["hard_fail"] or result.returncode != 0: + eprint(f"ERROR: respro init failed (exit {result.returncode}). See {log_path}") + return 1 + + eprint("Done.") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + finally: + # The workflow passes an explicit --work-dir it manages; for ad-hoc + # local runs we leave the temp dir in place for inspection. + pass From 10e0fa5e5161c756cec18721bb5b120f4c951920 Mon Sep 17 00:00:00 2001 From: jonas Date: Tue, 8 Sep 2026 12:35:34 +0200 Subject: [PATCH 3/6] added version to manifest --- .gitignore | 2 +- README.md | 7 +++++++ databases/manifest.json | 2 +- scripts/build_metadata_manifest.py | 10 +++++++++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 487fea5..fd5dc60 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ __pycache__/ .pytest_cache/ .mypy_cache/ -databases/charmd/ +charmd/ .act/ .DS_Store \ No newline at end of file diff --git a/README.md b/README.md index cfed825..43491d0 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,13 @@ It's built from all `databases/*/output/metadata.json` files, and each entry con name, relative paths to `metadata.json`, `rules.tsv`, and `formula-rules.tsv` (empty when absent), and the embedded metadata content. +The manifest's top-level `respro_version` field is a PEP 440 version specifier (e.g. `">=0.1.3"`) +stating the minimum ResPro release required to build the listed databases. ResPro clients compare +their own installed version against this specifier and prompt the user to upgrade when too old, +so breaking changes to the TSV contract or metadata schema can be gated on a known ResPro release. +Bump this value in `scripts/build_metadata_manifest.py` (`RESPRO_VERSION_REQUIREMENT`) whenever a +newer ResPro feature becomes a hard requirement. + This is the primariy file that is used to display supported databases via the ResPro command: ```bash diff --git a/databases/manifest.json b/databases/manifest.json index a72a639..dc2304e 100644 --- a/databases/manifest.json +++ b/databases/manifest.json @@ -1,5 +1,5 @@ { - "manifest_version": 1, + "respro_version": ">=0.1.3", "databases": [ { "source_name": "herpesdrg", diff --git a/scripts/build_metadata_manifest.py b/scripts/build_metadata_manifest.py index 6239895..3dc6c61 100644 --- a/scripts/build_metadata_manifest.py +++ b/scripts/build_metadata_manifest.py @@ -25,6 +25,14 @@ "interpretation_algorithms", } +# Minimum ResPro version required to build the databases listed in this +# manifest. Expressed as a PEP 440 version specifier string (without the +# leading package name) so ResPro clients can compare it against their own +# installed version and prompt the user to upgrade when too old. Bump this +# whenever a change to the TSV contract or metadata schema relies on +# behaviour only present in a newer ResPro release. +RESPRO_VERSION_REQUIREMENT = ">=0.1.3" + def load_metadata(metadata_path: Path) -> dict[str, Any]: with metadata_path.open("r", encoding="utf-8") as fh: @@ -85,7 +93,7 @@ def build_manifest(repo_root: Path) -> dict[str, Any]: entries.append(entry) return { - "manifest_version": 1, + "respro_version": RESPRO_VERSION_REQUIREMENT, "databases": entries, } From 7858b1aa6d1475200b93b959d1209185122c1ec2 Mon Sep 17 00:00:00 2001 From: jonas Date: Tue, 8 Sep 2026 12:43:40 +0200 Subject: [PATCH 4/6] added better repsro build qc --- .github/actions/respro-build-qc/action.yml | 118 ++++++++++++++++++ .github/workflows/herpesdrg-autobump.yml | 51 ++------ .../workflows/hsv-daehne-jaki-autobump.yml | 51 ++------ .github/workflows/pr-build-check.yml | 97 ++++++++++++++ .github/workflows/stanford-hiv-autobump.yml | 51 ++------ scripts/build_metadata_manifest.py | 2 +- 6 files changed, 240 insertions(+), 130 deletions(-) create mode 100644 .github/actions/respro-build-qc/action.yml create mode 100644 .github/workflows/pr-build-check.yml diff --git a/.github/actions/respro-build-qc/action.yml b/.github/actions/respro-build-qc/action.yml new file mode 100644 index 0000000..d3789a4 --- /dev/null +++ b/.github/actions/respro-build-qc/action.yml @@ -0,0 +1,118 @@ +name: respro build QC +description: | + Install ResistanceProfiler from a GitHub release tag (no conda, no autobump + of a pinned version) and run the build quality-control step against a + converted database's output artifacts. Used by both the autobump workflows + and the PR-triggered build-check workflow so every build is validated + against the same respro version. + +inputs: + source-name: + description: Source database name (e.g. stanford-hiv). Passed to respro as the project name. + required: true + output-dir: + description: Directory containing rules.tsv / metadata.json / formula-rules.tsv. + required: true + work-dir: + description: Working directory for GenBank fetch + built .db + logs. + required: false + default: /tmp/respro_qc + respro-version: + description: | + ResistanceProfiler version to build from. Defaults to "latest", which + resolves the newest tag on the the-foxlab/ResistanceProfiler repository + at run time. Set to a concrete tag (e.g. "v0.1.3") to pin a specific + release. + required: false + default: latest + +outputs: + qc_failed: + description: '"true" when the respro build QC reported a hard failure, otherwise "false".' + value: ${{ steps.qc_summary.outputs.qc_failed }} + summary: + description: 'Multi-line markdown summary of the build QC, suitable for PR bodies.' + value: ${{ steps.qc_summary.outputs.block }} + +runs: + using: composite + + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Resolve latest respro release tag + if: inputs.respro-version == 'latest' + id: respro_tag + shell: bash + run: | + set -euo pipefail + # Fetch the newest non-web tag from the ResistanceProfiler repo. + # Tags are prefixed with "v" (e.g. v0.1.3); web releases use "web-v". + tag=$(curl -fsSL \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/the-foxlab/ResistanceProfiler/tags" \ + | python -c ' + import json, sys + tags = json.load(sys.stdin) + for t in tags: + name = t.get("name", "") + if name.startswith("v") and not name.startswith("web-"): + print(name) + break + else: + sys.exit("No non-web respro release tag found") + ') + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + echo "Resolved latest respro release tag: ${tag}" + + - name: Determine respro version + id: respro_version + shell: bash + run: | + if [ "${{ inputs.respro-version }}" = "latest" ]; then + echo "version=${{ steps.respro_tag.outputs.tag }}" >> "$GITHUB_OUTPUT" + else + echo "version=${{ inputs.respro-version }}" >> "$GITHUB_OUTPUT" + fi + + - name: Install respro from release tag + shell: bash + run: | + set -euo pipefail + python -m pip install --upgrade pip + # Build from the tagged source tarball. The runner has a C compiler + # so the mappy native dependency builds from source without conda. + version="${{ steps.respro_version.outputs.version }}" + echo "Installing ResistanceProfiler ${version} from source" + python -m pip install \ + "respro @ git+https://github.com/the-foxlab/ResistanceProfiler.git@${version}" + respro --version + + - name: respro build QC + id: respro_qc + shell: bash + continue-on-error: true + run: | + python scripts/respro_build_qc.py \ + --source-name "${{ inputs.source-name }}" \ + --output-dir "${{ inputs.output-dir }}" \ + --work-dir "${{ inputs.work-dir }}" + echo "qc_exit=$?" >> "$GITHUB_OUTPUT" + + - name: Read respro QC summary + id: qc_summary + shell: bash + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "block<<${EOF}" >> "$GITHUB_OUTPUT" + cat "${{ inputs.work-dir }}/qc_summary.md" >> "$GITHUB_OUTPUT" + echo "${EOF}" >> "$GITHUB_OUTPUT" + qc_failed="false" + if [ -f "${{ inputs.work-dir }}/qc_summary.json" ]; then + status=$(python -c "import json;print(json.load(open('${{ inputs.work-dir }}/qc_summary.json'))['status'])" 2>/dev/null || echo "") + if [ "$status" = "failed" ]; then qc_failed="true"; fi + fi + echo "qc_failed=${qc_failed}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/herpesdrg-autobump.yml b/.github/workflows/herpesdrg-autobump.yml index 1a7e411..44dc277 100644 --- a/.github/workflows/herpesdrg-autobump.yml +++ b/.github/workflows/herpesdrg-autobump.yml @@ -100,49 +100,14 @@ jobs: run: | python scripts/build_metadata_manifest.py - - name: Set up respro (bioconda) - if: steps.update_check.outputs.should_update == 'true' - uses: mamba-org/setup-micromamba@v2 - with: - environment-name: respro - condarc: | - channels: - - conda-forge - - bioconda - create-args: >- - respro==0.1.1 - init-shell: bash - cache-environment: true - - name: respro build QC - id: respro_qc + id: qc if: steps.update_check.outputs.should_update == 'true' - shell: bash -el {0} - continue-on-error: true - run: | - python scripts/respro_build_qc.py \ - --source-name herpesdrg \ - --output-dir databases/herpesdrg/output \ - --work-dir /tmp/respro_qc - echo "qc_exit=$?" >> "$GITHUB_OUTPUT" - env: - NCBI_EUTILS_API_KEY: ${{ secrets.NCBI_EUTILS_API_KEY }} - - - name: Read respro QC summary - id: qc_summary - if: steps.update_check.outputs.should_update == 'true' - shell: bash - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "block<<${EOF}" >> "$GITHUB_OUTPUT" - cat /tmp/respro_qc/qc_summary.md >> "$GITHUB_OUTPUT" - echo "${EOF}" >> "$GITHUB_OUTPUT" - qc_failed="false" - if [ -f /tmp/respro_qc/qc_summary.json ]; then - status=$(python -c "import json;print(json.load(open('/tmp/respro_qc/qc_summary.json'))['status'])" 2>/dev/null || echo "") - if [ "$status" = "failed" ]; then qc_failed="true"; fi - fi - echo "qc_failed=${qc_failed}" >> "$GITHUB_OUTPUT" + uses: ./.github/actions/respro-build-qc + with: + source-name: herpesdrg + output-dir: databases/herpesdrg/output + work-dir: /tmp/respro_qc - name: No update detected if: steps.update_check.outputs.should_update != 'true' @@ -166,7 +131,7 @@ jobs: - Source commit SHA: ${{ steps.source.outputs.source_commit_sha }} - Source URL: ${{ steps.source.outputs.source_url }} - ${{ steps.qc_summary.outputs.block }} + ${{ steps.qc.outputs.summary }} This PR was generated by the autobump workflow. labels: autobump @@ -178,7 +143,7 @@ jobs: databases/manifest.json - name: Mark workflow failed on respro build error - if: steps.update_check.outputs.should_update == 'true' && steps.qc_summary.outputs.qc_failed == 'true' + if: steps.update_check.outputs.should_update == 'true' && steps.qc.outputs.qc_failed == 'true' run: | echo "respro build QC failed for herpesdrg; see the PR body and the 'respro build QC' step log." exit 1 diff --git a/.github/workflows/hsv-daehne-jaki-autobump.yml b/.github/workflows/hsv-daehne-jaki-autobump.yml index 2d874fe..f2b7444 100644 --- a/.github/workflows/hsv-daehne-jaki-autobump.yml +++ b/.github/workflows/hsv-daehne-jaki-autobump.yml @@ -117,49 +117,14 @@ jobs: run: | python scripts/build_metadata_manifest.py - - name: Set up respro (bioconda) - if: steps.update_check.outputs.should_update == 'true' - uses: mamba-org/setup-micromamba@v2 - with: - environment-name: respro - condarc: | - channels: - - conda-forge - - bioconda - create-args: >- - respro==0.1.1 - init-shell: bash - cache-environment: true - - name: respro build QC - id: respro_qc + id: qc if: steps.update_check.outputs.should_update == 'true' - shell: bash -el {0} - continue-on-error: true - run: | - python scripts/respro_build_qc.py \ - --source-name hsv_daehne_jaki \ - --output-dir databases/hsv_daehne_jaki/output \ - --work-dir /tmp/respro_qc - echo "qc_exit=$?" >> "$GITHUB_OUTPUT" - env: - NCBI_EUTILS_API_KEY: ${{ secrets.NCBI_EUTILS_API_KEY }} - - - name: Read respro QC summary - id: qc_summary - if: steps.update_check.outputs.should_update == 'true' - shell: bash - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "block<<${EOF}" >> "$GITHUB_OUTPUT" - cat /tmp/respro_qc/qc_summary.md >> "$GITHUB_OUTPUT" - echo "${EOF}" >> "$GITHUB_OUTPUT" - qc_failed="false" - if [ -f /tmp/respro_qc/qc_summary.json ]; then - status=$(python -c "import json;print(json.load(open('/tmp/respro_qc/qc_summary.json'))['status'])" 2>/dev/null || echo "") - if [ "$status" = "failed" ]; then qc_failed="true"; fi - fi - echo "qc_failed=${qc_failed}" >> "$GITHUB_OUTPUT" + uses: ./.github/actions/respro-build-qc + with: + source-name: hsv_daehne_jaki + output-dir: databases/hsv_daehne_jaki/output + work-dir: /tmp/respro_qc - name: No update detected if: steps.update_check.outputs.should_update != 'true' @@ -183,7 +148,7 @@ jobs: - Zenodo updated timestamp: ${{ steps.zenodo.outputs.updated }} - Source file: ${{ steps.zenodo.outputs.source_url }} - ${{ steps.qc_summary.outputs.block }} + ${{ steps.qc.outputs.summary }} This PR was generated by the autobump workflow. labels: autobump @@ -195,7 +160,7 @@ jobs: databases/manifest.json - name: Mark workflow failed on respro build error - if: steps.update_check.outputs.should_update == 'true' && steps.qc_summary.outputs.qc_failed == 'true' + if: steps.update_check.outputs.should_update == 'true' && steps.qc.outputs.qc_failed == 'true' run: | echo "respro build QC failed for hsv_daehne_jaki; see the PR body and the 'respro build QC' step log." exit 1 diff --git a/.github/workflows/pr-build-check.yml b/.github/workflows/pr-build-check.yml new file mode 100644 index 0000000..e1013e8 --- /dev/null +++ b/.github/workflows/pr-build-check.yml @@ -0,0 +1,97 @@ +name: PR database build check + +on: + pull_request: + paths: + - "databases/**" + - "scripts/respro_build_qc.py" + - ".github/actions/respro-build-qc/**" + - ".github/workflows/pr-build-check.yml" + +permissions: + contents: read + pull-requests: write + +jobs: + detect-changed-databases: + runs-on: ubuntu-latest + outputs: + herpesdrg: ${{ steps.filter.outputs.herpesdrg }} + hsv_daehne_jaki: ${{ steps.filter.outputs.hsv_daehne_jaki }} + stanford-hiv: ${{ steps.filter.outputs.stanford-hiv }} + any: ${{ steps.filter.outputs.any }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Detect changed databases + id: filter + uses: dorny/paths-filter@v3 + with: + # `any` covers shared infrastructure (QC script, composite action, + # this workflow) so a change there triggers a full build check. + filters: | + herpesdrg: + - 'databases/herpesdrg/**' + hsv_daehne_jaki: + - 'databases/hsv_daehne_jaki/**' + stanford-hiv: + - 'databases/stanford-hiv/**' + any: + - 'databases/herpesdrg/**' + - 'databases/hsv_daehne_jaki/**' + - 'databases/stanford-hiv/**' + - 'scripts/respro_build_qc.py' + - '.github/actions/respro-build-qc/**' + - '.github/workflows/pr-build-check.yml' + + build-check: + needs: detect-changed-databases + if: needs.detect-changed-databases.outputs.any == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # One job per database; the `if` guard on each step skips databases + # that were not touched by the PR. + database: + - name: herpesdrg + dir: databases/herpesdrg/output + changed: ${{ needs.detect-changed-databases.outputs.herpesdrg }} + - name: hsv_daehne_jaki + dir: databases/hsv_daehne_jaki/output + changed: ${{ needs.detect-changed-databases.outputs.hsv_daehne_jaki }} + - name: stanford-hiv + dir: databases/stanford-hiv/output + changed: ${{ needs.detect-changed-databases.outputs.stanford-hiv }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run respro build QC + if: matrix.database.changed == 'true' + id: qc + uses: ./.github/actions/respro-build-qc + with: + source-name: ${{ matrix.database.name }} + output-dir: ${{ matrix.database.dir }} + work-dir: /tmp/respro_qc_${{ matrix.database.name }} + + - name: Comment QC summary on PR + if: matrix.database.changed == 'true' && always() + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + ### respro build QC — `${{ matrix.database.name }}` + + ${{ steps.qc.outputs.summary }} + + _Failed builds must be fixed before merge._ + + - name: Fail on respro build error + if: matrix.database.changed == 'true' && steps.qc.outputs.qc_failed == 'true' + run: | + echo "respro build QC failed for ${{ matrix.database.name }}; see the PR comment and the build logs." + exit 1 diff --git a/.github/workflows/stanford-hiv-autobump.yml b/.github/workflows/stanford-hiv-autobump.yml index 1407c8b..477c3c0 100644 --- a/.github/workflows/stanford-hiv-autobump.yml +++ b/.github/workflows/stanford-hiv-autobump.yml @@ -120,49 +120,14 @@ jobs: run: | python scripts/build_metadata_manifest.py - - name: Set up respro (bioconda) - if: steps.update_check.outputs.should_update == 'true' - uses: mamba-org/setup-micromamba@v2 - with: - environment-name: respro - condarc: | - channels: - - conda-forge - - bioconda - create-args: >- - respro==0.1.1 - init-shell: bash - cache-environment: true - - name: respro build QC - id: respro_qc - if: steps.update_check.outputs.should_update == 'true' - shell: bash -el {0} - continue-on-error: true - run: | - python scripts/respro_build_qc.py \ - --source-name stanford-hiv \ - --output-dir databases/stanford-hiv/output \ - --work-dir /tmp/respro_qc - echo "qc_exit=$?" >> "$GITHUB_OUTPUT" - env: - NCBI_EUTILS_API_KEY: ${{ secrets.NCBI_EUTILS_API_KEY }} - - - name: Read respro QC summary - id: qc_summary + id: qc if: steps.update_check.outputs.should_update == 'true' - shell: bash - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "block<<${EOF}" >> "$GITHUB_OUTPUT" - cat /tmp/respro_qc/qc_summary.md >> "$GITHUB_OUTPUT" - echo "${EOF}" >> "$GITHUB_OUTPUT" - qc_failed="false" - if [ -f /tmp/respro_qc/qc_summary.json ]; then - status=$(python -c "import json;print(json.load(open('/tmp/respro_qc/qc_summary.json'))['status'])" 2>/dev/null || echo "") - if [ "$status" = "failed" ]; then qc_failed="true"; fi - fi - echo "qc_failed=${qc_failed}" >> "$GITHUB_OUTPUT" + uses: ./.github/actions/respro-build-qc + with: + source-name: stanford-hiv + output-dir: databases/stanford-hiv/output + work-dir: /tmp/respro_qc - name: No update detected if: steps.update_check.outputs.should_update != 'true' @@ -186,7 +151,7 @@ jobs: - Source version: ${{ steps.source.outputs.source_version }} - Source URL: ${{ steps.source.outputs.source_url }} - ${{ steps.qc_summary.outputs.block }} + ${{ steps.qc.outputs.summary }} This PR was generated by the autobump workflow. labels: autobump @@ -198,7 +163,7 @@ jobs: databases/manifest.json - name: Mark workflow failed on respro build error - if: steps.update_check.outputs.should_update == 'true' && steps.qc_summary.outputs.qc_failed == 'true' + if: steps.update_check.outputs.should_update == 'true' && steps.qc.outputs.qc_failed == 'true' run: | echo "respro build QC failed for stanford-hiv; see the PR body and the 'respro build QC' step log." exit 1 \ No newline at end of file diff --git a/scripts/build_metadata_manifest.py b/scripts/build_metadata_manifest.py index 3dc6c61..d96d0cc 100644 --- a/scripts/build_metadata_manifest.py +++ b/scripts/build_metadata_manifest.py @@ -39,7 +39,7 @@ def load_metadata(metadata_path: Path) -> dict[str, Any]: metadata = json.load(fh) if not isinstance(metadata, dict): - raise ValueError(f"Metadata must be a JSON object: {metadata_path}") + raise ValueError(f"Metadata must be a JSON object: {metadata_path}") # noqa: TRY004 keys = set(metadata.keys()) required = set(REQUIRED_METADATA_KEYS) From 68e7392b5de5016b3c3ce4f2b5e6257e8f804855 Mon Sep 17 00:00:00 2001 From: jonas Date: Tue, 8 Sep 2026 16:02:36 +0200 Subject: [PATCH 5/6] multiple fixes --- databases/herpesdrg/output/metadata.json | 10 ++-- databases/herpesdrg/scripts/convert.py | 8 ++- .../hsv_daehne_jaki/output/metadata.json | 6 +- databases/hsv_daehne_jaki/scripts/convert.py | 1 - databases/stanford-hiv/output/metadata.json | 7 ++- .../output/non-migrated-rules.txt | 4 +- databases/stanford-hiv/scripts/convert.py | 56 ++++++++++++++----- 7 files changed, 59 insertions(+), 33 deletions(-) diff --git a/databases/herpesdrg/output/metadata.json b/databases/herpesdrg/output/metadata.json index 04ac5f5..5bffa9b 100644 --- a/databases/herpesdrg/output/metadata.json +++ b/databases/herpesdrg/output/metadata.json @@ -44,18 +44,16 @@ }, { "name": "drug_interpretation", - "method": "by_phenotype", - "thresholds": { - "resistant": 1, - "intermediate": 1 - } + "method": "by_phenotype" }, { "name": "drug_interpretation", "method": "by_fold_ic50", "thresholds": { + "susceptible": 0.0, + "low-level resistance": 2.0, "intermediate": 5.0, - "resistant": 15.0 + "high-level resistance": 15.0 } }, { diff --git a/databases/herpesdrg/scripts/convert.py b/databases/herpesdrg/scripts/convert.py index b7491fb..a4a2412 100644 --- a/databases/herpesdrg/scripts/convert.py +++ b/databases/herpesdrg/scripts/convert.py @@ -744,12 +744,16 @@ def main() -> None: { "name": "drug_interpretation", "method": "by_phenotype", - "thresholds": {"resistant": 1, "intermediate": 1}, }, { "name": "drug_interpretation", "method": "by_fold_ic50", - "thresholds": {"intermediate": 5.0, "resistant": 15.0}, + "thresholds": { + "susceptible": 0.0, + "low-level resistance": 2.0, + "intermediate": 5.0, + "high-level resistance": 15.0, + }, }, { "name": "drug_alias", diff --git a/databases/hsv_daehne_jaki/output/metadata.json b/databases/hsv_daehne_jaki/output/metadata.json index d7dc598..865ad7d 100644 --- a/databases/hsv_daehne_jaki/output/metadata.json +++ b/databases/hsv_daehne_jaki/output/metadata.json @@ -34,11 +34,7 @@ }, { "name": "drug_interpretation", - "method": "by_phenotype", - "thresholds": { - "resistant": 1, - "intermediate": 1 - } + "method": "by_phenotype" }, { "name": "drug_alias", diff --git a/databases/hsv_daehne_jaki/scripts/convert.py b/databases/hsv_daehne_jaki/scripts/convert.py index 56f274f..0b46be8 100644 --- a/databases/hsv_daehne_jaki/scripts/convert.py +++ b/databases/hsv_daehne_jaki/scripts/convert.py @@ -876,7 +876,6 @@ def main(): { "name": "drug_interpretation", "method": "by_phenotype", - "thresholds": {"resistant": 1, "intermediate": 1}, }, { "name": "drug_alias", diff --git a/databases/stanford-hiv/output/metadata.json b/databases/stanford-hiv/output/metadata.json index cf926e4..20ea2be 100644 --- a/databases/stanford-hiv/output/metadata.json +++ b/databases/stanford-hiv/output/metadata.json @@ -57,8 +57,11 @@ "name": "drug_interpretation", "method": "by_score", "thresholds": { - "intermediate": 15, - "resistant": 60 + "susceptible": 0, + "potential low-level resistance": 10, + "low-level resistance": 15, + "intermediate resistance": 30, + "high-level resistance": 60 } }, { diff --git a/databases/stanford-hiv/output/non-migrated-rules.txt b/databases/stanford-hiv/output/non-migrated-rules.txt index bd74d51..b045e68 100644 --- a/databases/stanford-hiv/output/non-migrated-rules.txt +++ b/databases/stanford-hiv/output/non-migrated-rules.txt @@ -1,4 +1,4 @@ # Non-migrated Stanford HIVDB rules -# Columns: reason drug gene score raw_rule details +# Columns: reason drug feature score raw_rule details -reason drug gene score raw_rule details +reason drug feature score raw_rule details diff --git a/databases/stanford-hiv/scripts/convert.py b/databases/stanford-hiv/scripts/convert.py index 06be1d1..e99421a 100644 --- a/databases/stanford-hiv/scripts/convert.py +++ b/databases/stanford-hiv/scripts/convert.py @@ -279,15 +279,25 @@ def extract_score_thresholds(root: ET.Element) -> dict[str, int]: (-INF TO 9 => 1, 10 TO 14 => 2, 15 TO 29 => 3, 30 TO 59 => 4, 60 TO INF => 5) LEVEL_DEFINITION maps ORDER integers to SIR codes (S/I/R). - Returns a dict with the minimum score for each non-S level name used in - the ResPro drug_interpretation block, e.g. {"intermediate": 15, "resistant": 60}. - The threshold is the lowest finite lower-bound score whose level order maps to R or I. + Returns a dict mapping each rank-vocabulary label to the lowest finite + lower-bound score for its GLOBALRANGE bucket. The rank-1 ``susceptible`` + label is always present (value 0) as the baseline fallback for scores + below all higher-rank breakpoints, mirroring the ResPro ``by_score`` + contract that requires at least one rank-1 label. + + Stanford's five ORDERs map to the ResPro rank vocabulary as: + ORDER 1 (S, -INF..9) -> susceptible (rank 1, value 0) + ORDER 2 (I?, 10..14) -> potential low-level resistance (rank 2) + ORDER 3 (I?, 15..29) -> low-level resistance (rank 3) + ORDER 4 (I, 30..59) -> intermediate resistance (rank 4) + ORDER 5 (R, 60..INF) -> high-level resistance (rank 5) """ defs = root.find("DEFINITIONS") if defs is None: raise ValueError("XML missing DEFINITIONS element") - # Build order -> SIR mapping from LEVEL_DEFINITION elements. + # Build order -> SIR mapping from LEVEL_DEFINITION elements (kept for + # validation that the XML structure is as expected). order_to_sir: dict[int, str] = {} for ld in defs.findall("LEVEL_DEFINITION"): order_text = text_of(ld, "ORDER") @@ -309,24 +319,40 @@ def extract_score_thresholds(root: ET.Element) -> dict[str, int]: re.IGNORECASE, ) - # Collect lowest finite lower-bound for each SIR category. - sir_min: dict[str, int] = {} + # Map each ORDER to its rank-vocabulary label. Stanford uses five ordered + # levels (1..5); we project them onto the ResPro rank vocabulary so the + # report colours and strongest-wins aggregation use the full multi-tier + # scale rather than collapsing to S/I/R. + order_to_label: dict[int, str] = { + 1: "susceptible", + 2: "potential low-level resistance", + 3: "low-level resistance", + 4: "intermediate resistance", + 5: "high-level resistance", + } + + # Collect the lowest finite lower-bound score per rank label. The + # susceptible baseline is always 0 so scores below the first non-S + # breakpoint resolve to rank 1. + label_min: dict[str, int] = {"susceptible": 0} for m in bucket_re.finditer(range_text): lb_raw, _, order_raw = m.group(1), m.group(2), m.group(3) order = int(order_raw) - sir = order_to_sir.get(order) - if sir is None or sir == "S": + label = order_to_label.get(order) + if label is None: continue - lb = None if lb_raw.upper() == "-INF" else int(lb_raw) - if lb is not None: - sir_key = "resistant" if sir == "R" else "intermediate" - if sir_key not in sir_min or lb < sir_min[sir_key]: - sir_min[sir_key] = lb + if lb_raw.upper() == "-INF": + # -INF lower bound: the susceptible baseline bucket. + label_min[label] = 0 + else: + lb = int(lb_raw) + if label not in label_min or lb < label_min[label]: + label_min[label] = lb - if not sir_min: + if not any(k != "susceptible" for k in label_min): raise ValueError("Could not derive any non-susceptible thresholds from GLOBALRANGE") - return sir_min + return label_min def build_drug_to_gene(root: ET.Element) -> dict[str, str]: From 5618bfd0c0d12da80011794d38589431a36e634d Mon Sep 17 00:00:00 2001 From: jonas Date: Wed, 9 Sep 2026 14:29:06 +0200 Subject: [PATCH 6/6] small change --- .github/actions/respro-build-qc/action.yml | 24 ++++++++++++++++---- databases/hsv_daehne_jaki/scripts/convert.py | 3 ++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/actions/respro-build-qc/action.yml b/.github/actions/respro-build-qc/action.yml index d3789a4..ae2c51d 100644 --- a/.github/actions/respro-build-qc/action.yml +++ b/.github/actions/respro-build-qc/action.yml @@ -106,13 +106,27 @@ runs: id: qc_summary shell: bash run: | + summary_path="${{ inputs.work-dir }}/qc_summary.md" EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) echo "block<<${EOF}" >> "$GITHUB_OUTPUT" - cat "${{ inputs.work-dir }}/qc_summary.md" >> "$GITHUB_OUTPUT" - echo "${EOF}" >> "$GITHUB_OUTPUT" qc_failed="false" - if [ -f "${{ inputs.work-dir }}/qc_summary.json" ]; then - status=$(python -c "import json;print(json.load(open('${{ inputs.work-dir }}/qc_summary.json'))['status'])" 2>/dev/null || echo "") - if [ "$status" = "failed" ]; then qc_failed="true"; fi + if [ ! -f "$summary_path" ]; then + # The QC script exited before writing a summary (infrastructure + # error: crash, missing respro, import failure). Emit a synthetic + # failure block so the PR comment still renders and the workflow + # fails cleanly via qc_failed=true. + echo "## respro build QC" >> "$GITHUB_OUTPUT" + echo "" >> "$GITHUB_OUTPUT" + echo "**Source:** \`${{ inputs.source-name }}\`" >> "$GITHUB_OUTPUT" + echo "**Status:** ⚠️ **\`respro init\` FAILED** — the QC step did not produce a summary (infrastructure error before QC ran)." >> "$GITHUB_OUTPUT" + echo "**Exit code:** \`unknown\`" >> "$GITHUB_OUTPUT" + qc_failed="true" + else + cat "$summary_path" >> "$GITHUB_OUTPUT" + if [ -f "${{ inputs.work-dir }}/qc_summary.json" ]; then + status=$(python -c "import json;print(json.load(open('${{ inputs.work-dir }}/qc_summary.json'))['status'])" 2>/dev/null || echo "") + if [ "$status" = "failed" ]; then qc_failed="true"; fi + fi fi + echo "${EOF}" >> "$GITHUB_OUTPUT" echo "qc_failed=${qc_failed}" >> "$GITHUB_OUTPUT" diff --git a/databases/hsv_daehne_jaki/scripts/convert.py b/databases/hsv_daehne_jaki/scripts/convert.py index 0b46be8..17a6131 100644 --- a/databases/hsv_daehne_jaki/scripts/convert.py +++ b/databases/hsv_daehne_jaki/scripts/convert.py @@ -813,7 +813,8 @@ def main(): single_rows, formula_rows, non_migrated_rows = extract_rows(wb) finally: tmp_path.unlink(missing_ok=True) - eprint(f"Parsed {len(single_rows) + len(formula_rows) + len(non_migrated_rows)} source rows") + # Source is a non-tabular XLSX workbook, so the "Parsed source rows" + # line is omitted per the converter output contract (README). # --- Deduplication --- single_rows = deduplicate_single_rows(single_rows, non_migrated_rows)