diff --git a/cnvlib/commands.py b/cnvlib/commands.py
index 5ccbcd7b..946c970a 100644
--- a/cnvlib/commands.py
+++ b/cnvlib/commands.py
@@ -273,7 +273,7 @@ def first_paths(fnames: list[str]) -> dict[str, str]:
def _cmd_batch(args: argparse.Namespace) -> None:
- """Run the complete CNVkit pipeline on one or more BAM or bedGraph files."""
+ """Run the complete CNVkit pipeline on one or more BAM, CRAM, or bedGraph files."""
logging.info("CNVkit %s", __version__)
# Validate/restrict options, beyond what argparse mutual exclusion can do
bad_args_msg = ""
@@ -426,7 +426,7 @@ def _cmd_batch(args: argparse.Namespace) -> None:
P_batch.add_argument(
"sample_fnames",
nargs="*",
- help="""Mapped sequence reads (.bam) or pre-computed per-base depth
+ help="""Mapped sequence reads (.bam or .cram) or pre-computed per-base depth
(bedGraph .bed.gz with tabix index .tbi or .csi)""",
)
P_batch.add_argument(
@@ -491,10 +491,10 @@ def _cmd_batch(args: argparse.Namespace) -> None:
"--normal",
nargs="*",
metavar="FILES",
- help="""Normal samples (.bam or bedGraph .bed.gz) used to construct the pooled, paired,
- or flat reference. If this option is used but no filenames are given, a "flat"
- reference will be built. Otherwise, all filenames following this option will
- be used.""",
+ help="""Normal samples (.bam, .cram, or bedGraph .bed.gz) used to construct the
+ pooled, paired, or flat reference. If this option is used but no filenames
+ are given, a "flat" reference will be built. Otherwise, all filenames
+ following this option will be used.""",
)
add_fasta(P_batch_newref)
P_batch_newref.add_argument(
@@ -749,7 +749,7 @@ def _cmd_antitarget(args: argparse.Namespace) -> None:
def _cmd_autobin(args: argparse.Namespace) -> None:
- """Quickly calculate reasonable bin sizes from BAM read counts."""
+ """Quickly calculate reasonable bin sizes from BAM or CRAM read counts."""
if args.method in ("hybrid", "amplicon") and not args.targets:
raise RuntimeError(f"Sequencing method {args.method!r} requires targets (-t)")
if args.method == "wgs":
@@ -830,7 +830,8 @@ def read_regions(bed_fname):
P_autobin.add_argument(
"bams",
nargs="+",
- help="""Sample BAM file(s) or bedGraph file(s) (.bed.gz) to test for target coverage""",
+ help="""Sample BAM or CRAM file(s), or bedGraph file(s) (.bed.gz), to test for
+ target coverage""",
)
add_fasta(P_autobin)
P_autobin.add_argument(
@@ -925,7 +926,7 @@ def read_regions(bed_fname):
def _cmd_coverage(args: argparse.Namespace) -> None:
- """Calculate coverage in the given regions from BAM read depths or bedGraph."""
+ """Calculate coverage in the given regions from BAM/CRAM read depths or bedGraph."""
pset = coverage.do_coverage(
args.interval,
args.bam_or_bg_file,
@@ -953,7 +954,7 @@ def _cmd_coverage(args: argparse.Namespace) -> None:
P_coverage.add_argument(
"bam_or_bg_file",
metavar="input_file",
- help="""Mapped sequence reads (.bam) or pre-computed per-base depth
+ help="""Mapped sequence reads (.bam or .cram) or pre-computed per-base depth
(bedGraph .bed.gz with tabix index .tbi or .csi)""",
)
P_coverage.add_argument("interval", help="Intervals (.bed or .list)")
diff --git a/doc/pipeline.rst b/doc/pipeline.rst
index 87723240..64ddfaca 100644
--- a/doc/pipeline.rst
+++ b/doc/pipeline.rst
@@ -422,7 +422,7 @@ command.
``coverage``
------------
-Calculate coverage in the given regions from BAM read depths.
+Calculate coverage in the given regions from BAM or CRAM read depths.
By default, coverage is calculated via mean read depth from a pileup.
Alternatively, the ``--count`` option scans each read individually with pysam
@@ -474,6 +474,12 @@ these reads when calculating read depth.
physically removed. (Exception: for amplicon sequencing, do *not* mark
duplicates at all -- see :doc:`nonhybrid`.)
+CNVkit does not read base quality scores when calculating coverage, so base
+quality score recalibration does not affect it. GATK ApplyBQSR rewrites the
+quality scores and, apart from GATK's default filter for malformed reads, passes
+every read through with its alignment and flags unchanged, so the MarkDuplicates
+output and the recalibrated output yield the same coverage.
+
You will probably want to index the finished BAM file using `samtools
`_ or SAMBAMBA. But if you haven't done this
beforehand, CNVkit will automatically do it for you.
@@ -511,6 +517,77 @@ beforehand, CNVkit will automatically do it for you.
or manual indexing are required (equivalent to running ``samtools index
-c`` yourself).
+CRAM input
+``````````
+
+CNVkit accepts CRAM files wherever it accepts BAM files, including the
+:ref:`batch`, :ref:`coverage`, and :ref:`autobin` commands. Most CRAM files store
+each read as its differences from an external reference genome, which is then
+needed to read the file. Supply the FASTA file that the CRAM files were written
+against with the ``-f``/``--fasta`` option::
+
+ cnvkit.py coverage Sample.cram targets.bed -f hg38.fa -o Sample.targetcoverage.cnn
+
+ cnvkit.py batch *Tumor.cram --normal *Normal.cram --targets my_targets.bed \
+ --fasta hg38.fa --output-reference my_reference.cnn --output-dir results/
+
+The :ref:`batch` command also accepts ``-f`` together with ``-r``/``--reference``,
+for applying an existing copy number reference to new CRAM files. As with BAM
+files, CNVkit builds a missing or outdated CRAM index (``.crai``) automatically.
+
+.. note::
+ **Without** ``-f``, **reading a CRAM file can take many hours or fail.**
+ CNVkit reads alignments through pysam and HTSlib. HTSlib looks for each
+ reference sequence, in order: by name in the FASTA file given with ``-f``;
+ by the MD5 checksum in the ``M5`` tag of the CRAM header's ``@SQ`` line,
+ first in the ``REF_CACHE`` directory and then in each location listed in
+ ``REF_PATH``; and by name in the local FASTA file named in the ``@SQ``
+ line's ``UR`` tag. If ``REF_PATH`` is set and none of these supplies the
+ sequence, HTSlib reports ``Unable to fetch reference`` and ``Failure to
+ decode slice``, and CNVkit stops with an error. If ``REF_PATH`` is not set,
+ the outcome depends on the HTSlib version:
+
+ - HTSlib 1.21 and earlier (bundled with pysam 0.23 and earlier) use the
+ European Bioinformatics Institute's reference server as the default
+ ``REF_PATH``, downloading each sequence and storing it in a local cache,
+ ``~/.cache/hts-ref`` unless ``REF_CACHE`` is set. Fetching every chromosome
+ of a large genome this way can take many hours, and is a common reason for
+ CRAM processing to run far slower than BAM processing of the same data.
+ HTSlib logs a warning beginning ``[W::cram_populate_ref] Creating
+ reference cache directory`` only when it creates the default cache
+ directory; later downloads into that directory are silent.
+ - Later HTSlib versions (bundled with pysam 0.24 and later) have no default
+ server, and fail with the errors above.
+
+ Passing the matching FASTA file with ``-f`` avoids both outcomes. Where that
+ is not possible, the ``seq_cache_populate.pl`` script distributed with
+ samtools builds a local directory of sequences named by their MD5
+ checksums, and prints the path template to assign to ``REF_CACHE``, such
+ as ``/data/ref_cache/%2s/%2s/%s``. Assign the same template to ``REF_PATH``,
+ so that HTSlib 1.21 and earlier do not fall back to the EBI server for a
+ sequence missing from the cache. HTSlib's `reference sequence
+ documentation `_ describes
+ these variables in full.
+
+.. note::
+ **The FASTA file must contain the same sequences the CRAM files were written
+ against.** Where it differs, HTSlib reports an ``MD5 checksum reference
+ mismatch``, and CNVkit stops with an error rather than computing coverage
+ from misread alignments. A warning such as ``[W::sanitise_SQ_lines] Header
+ @SQ length mismatch for ref chr1, 249250621 vs 248956422`` means that the
+ FASTA file's sequence lengths disagree with the CRAM header, which typically
+ indicates a different genome assembly (here, a CRAM header from hg19 and a
+ FASTA file from hg38). Decoding can still succeed where the two sequences
+ happen to agree over the region being read, so treat the warning as a sign
+ that the wrong FASTA file was given, even when no error follows.
+
+A CRAM file and a BAM file holding the same alignments yield identical coverage,
+in both the default and ``--count`` calculations, and CNVkit's test suite checks
+this on a fixture that includes duplicate-marked reads. Duplicate flags are
+honored in CRAM input exactly as in BAM input (see the note on marking
+duplicates above), and since coverage does not depend on base quality scores,
+CRAM's lossy quality-score compression options do not affect it either.
+
.. _reference:
diff --git a/test/test_preprocessing.py b/test/test_preprocessing.py
index f4a2e239..1509b2ee 100644
--- a/test/test_preprocessing.py
+++ b/test/test_preprocessing.py
@@ -231,43 +231,71 @@ def test_bedcov_depth_independent_of_process_count(self):
def test_coverage_skips_marked_duplicates(self):
"""Coverage ignores reads flagged as duplicates in both the depth
(bedcov) and by-count paths, so marking duplicates (e.g. Picard
- MarkDuplicates) is equivalent to physically removing them (#689)."""
+ MarkDuplicates) is equivalent to physically removing them (#689).
+
+ The same holds for CRAM input (#1105). Base qualities play no part
+ either: half of the surviving reads carry Phred 2 at every base, so
+ a recalibration that rewrites only base qualities, such as GATK
+ ApplyBQSR, cannot change the result.
+ """
seqlen = 50
- n_total, n_dup = 10, 6 # 4 reads survive the duplicate filter
+ n_total, n_dup = 10, 6
+ n_kept = n_total - n_dup
header = {
"HD": {"VN": "1.6", "SO": "coordinate"},
"SQ": [{"SN": "chr1", "LN": 1000}],
}
tmpdir = tempfile.mkdtemp()
- bam = os.path.join(tmpdir, "dup.bam")
- with pysam.AlignmentFile(bam, "wb", header=header) as out:
- for i in range(n_total):
- a = pysam.AlignedSegment()
- a.query_name = f"r{i}"
- a.query_sequence = "A" * seqlen
- a.flag = 0
- a.reference_id = 0
- a.reference_start = 100
- a.mapping_quality = 60
- a.cigarstring = f"{seqlen}M"
- a.query_qualities = pysam.qualitystring_to_array("I" * seqlen)
- a.is_duplicate = i >= (n_total - n_dup)
- out.write(a)
- pysam.index(bam)
- bed = os.path.join(tmpdir, "region.bed")
- with open(bed, "w") as fh:
- fh.write("chr1\t90\t160\tregionA\n")
- n_kept = n_total - n_dup
try:
- # Depth path (default): basecount counts only non-duplicate reads
- table = coverage.bedcov(bed, bam, 0)
- self.assertEqual(float(table["basecount"].iloc[0]), n_kept * seqlen)
- # By-count path: same surviving read count
- bamfile = pysam.AlignmentFile(bam, "rb")
- count, _row = coverage.region_depth_count(
- bamfile, "chr1", 90, 160, "regionA", 0
- )
- self.assertEqual(count, n_kept)
+ # The CRAM is written against a FASTA that is then moved, so its
+ # @SQ UR tag names a missing file. With REF_PATH and REF_CACHE
+ # pointing at a nonexistent directory, HTSlib has no other source (and
+ # does not fall back to the EBI server), so decoding depends on the
+ # FASTA passed to CNVkit.
+ written_fasta = os.path.join(tmpdir, "written.fa")
+ with open(written_fasta, "w") as fh:
+ fh.write(">chr1\n" + "A" * 1000 + "\n")
+ alns = {}
+ for ext, mode in (("bam", "wb"), ("cram", "wc")):
+ aln = alns[ext] = os.path.join(tmpdir, f"dup.{ext}")
+ with pysam.AlignmentFile(
+ aln, mode, header=header, reference_filename=written_fasta
+ ) as out:
+ for i in range(n_total):
+ a = pysam.AlignedSegment()
+ a.query_name = f"r{i}"
+ a.query_sequence = "A" * seqlen
+ a.flag = 0
+ a.reference_id = 0
+ a.reference_start = 100
+ a.mapping_quality = 60
+ a.cigarstring = f"{seqlen}M"
+ a.query_qualities = pysam.qualitystring_to_array(
+ ("I" if i % 2 else "#") * seqlen
+ )
+ a.is_duplicate = i >= n_kept
+ out.write(a)
+ pysam.index(aln)
+ fasta = os.path.join(tmpdir, "ref.fa")
+ os.rename(written_fasta, fasta)
+ bed = os.path.join(tmpdir, "region.bed")
+ with open(bed, "w") as fh:
+ fh.write("chr1\t90\t160\tregionA\n")
+ no_cache = os.path.join(tmpdir, "no-cache", "%s")
+ ref_env = {"REF_PATH": no_cache, "REF_CACHE": no_cache}
+ for ext, aln in alns.items():
+ with self.subTest(fmt=ext), mock.patch.dict(os.environ, ref_env):
+ # Depth path (default): only non-duplicate reads contribute
+ table = coverage.bedcov(bed, aln, 0, fasta)
+ self.assertEqual(float(table["basecount"].iloc[0]), n_kept * seqlen)
+ # By-count path: same surviving read count
+ with pysam.AlignmentFile(
+ aln, "rb", reference_filename=fasta
+ ) as alnfile:
+ count, _row = coverage.region_depth_count(
+ alnfile, "chr1", 90, 160, "regionA", 0
+ )
+ self.assertEqual(count, n_kept)
finally:
shutil.rmtree(tmpdir)
@@ -554,18 +582,31 @@ def test_coverage(self):
self.assertGreater(cna.log2.nunique(), 1)
def test_coverage_cram(self):
- """The 'coverage' command with CRAM input."""
+ """CRAM input yields the same coverage as a BAM of the same alignments.
+
+ CRAM decoding reproduces the alignments exactly, so the two formats
+ must agree exactly in both the depth and by-count paths. The fixture
+ holds duplicate-marked reads within the chrM target, so the comparison
+ also requires both formats to apply the duplicate flag alike (#1105).
+
+ The fixture's @SQ UR tags name an absolute path on the machine that
+ wrote it, so where that path exists, this test cannot show that the
+ FASTA argument is used; test_coverage_skips_marked_duplicates does.
+ """
bed = "formats/my-targets.bed"
cram = "formats/na12878-chrM-Y-trunc.cram"
fasta = "formats/chrM-Y-trunc.hg19.fa"
bam = "formats/na12878-chrM-Y-trunc.bam"
- # Verify CRAM coverage matches BAM coverage
- cna_cram = commands.do_coverage(bed, cram, fasta=fasta)
- cna_bam = commands.do_coverage(bed, bam)
- self.assertEqual(len(cna_cram), len(cna_bam))
- np.testing.assert_array_almost_equal(
- cna_cram["depth"].values, cna_bam["depth"].values, decimal=1
- )
+ for by_count in (False, True):
+ with self.subTest(by_count=by_count):
+ cna_cram = commands.do_coverage(
+ bed, cram, by_count=by_count, fasta=fasta
+ )
+ cna_bam = commands.do_coverage(bed, bam, by_count=by_count)
+ self.assertEqual(len(cna_cram), 4) # One row per BED region
+ np.testing.assert_array_equal(
+ cna_cram["depth"].values, cna_bam["depth"].values
+ )
def test_coverage_bedgraph(self):
"""The 'coverage' command with bedGraph input."""