diff --git a/.gitignore b/.gitignore index b9434da..5ab1374 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,9 @@ Makefile.bak minibwa tmp + +# api-test helper programs. `make test` builds these from tracked sources in +# api-test/, so the binaries are artifacts. +api-test/ex-*-check +api-test/mbmap-batch +api-test/mbmap-one diff --git a/Makefile b/Makefile index 0e20bd8..ea7fe2e 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ ifeq ($(ARCH), x86_64) endif .SUFFIXES:.c .o -.PHONY:all clean depend +.PHONY:all clean depend test .c.o: $(CC) -c $(CFLAGS) $(CPPFLAGS) $(INCLUDES) $< -o $@ @@ -55,6 +55,23 @@ libminibwa.a:$(LOBJS) minibwa:libminibwa.a $(MALLOC_O) $(AOBJS) main.o $(CC) $(CFLAGS) $(LDFLAGS) $(MALLOC_O) $(AOBJS) main.o -o $@ -L. -lminibwa $(LIBS) +# Run the ALT liftover-group suite. Each script builds its own fixtures, but several +# drive the api-test helpers, which live in their own Makefile and are not part of `all` +# -- without them those scripts fail on a fresh checkout. Set MB_STOCK to a stock +# minibwa to enable the no-.alt byte-identity comparisons; they are skipped without it. +test:minibwa + $(MAKE) -C api-test + @if [ -z "$$MB_STOCK" ]; then \ + echo "NOTE: MB_STOCK unset -> the no-.alt byte-identity-vs-stock comparisons are SKIPPED."; \ + echo " Set MB_STOCK= to exercise that invariant."; \ + fi + @fail=0; for t in test/altlg/test-*.sh; do \ + printf '%s: ' "$$t"; \ + if sh "$$t" > /tmp/altlg-log.$$$$ 2>&1; then echo PASS; \ + else echo FAIL; cat /tmp/altlg-log.$$$$; fail=1; fi; \ + rm -f /tmp/altlg-log.$$$$; \ + done; exit $$fail + clean: rm -fr *.o a.out $(PROG) *~ *.a *.dSYM diff --git a/README.md b/README.md index 1926712..e68acfd 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,10 @@ structural changes anyway. Minibwa is designed for mapping short reads and accurate long reads. It does not support spliced alignment and has not been tuned for aligning long contigs. -For now, minibwa does not properly work with alternate contigs in the reference -genome. Please use a version of the reference without such contigs. +Alternate contigs are supported: when a `.alt` file sits beside the index +it is loaded automatically and ALT hits are reconciled with their primary +counterparts, so an ALT twin no longer collapses the primary's mapping quality. +Pass `--no-alt` to ignore it and align exactly as if no ALT file existed. ### Installation @@ -130,7 +132,6 @@ include GPL'd source code. * Minibwa does not work with noisy long reads or spliced RNA-seq reads. * Minibwa does not support undirectional bisulfite sequencing data. -* Minibwa does not recognize alternate haplotypes. [apache2]: https://github.com/lh3/minibwa/tree/Apache2 [zlib]: https://zlib.net/ diff --git a/api-test/Makefile b/api-test/Makefile index 32c578a..e2f8885 100644 --- a/api-test/Makefile +++ b/api-test/Makefile @@ -1,6 +1,6 @@ CC= gcc CFLAGS= -std=gnu99 -Wall -O3 -EXE= mbmap-one mbmap-batch +EXE= mbmap-one mbmap-batch ex-lift-check ex-place-check ex-group-check .PHONY:all clean @@ -10,10 +10,19 @@ all:$(EXE) (cd ..; make libminibwa.a) mbmap-one:ex-one.c ../libminibwa.a - $(CC) $(CFLAGS) -o $@ -I.. $^ -lz -lm + $(CC) $(CFLAGS) -o $@ -I.. $^ -lz -lm $(LIBS) mbmap-batch:ex-batch.c ../libminibwa.a - $(CC) $(CFLAGS) -o $@ -I.. $^ -lz -lm + $(CC) $(CFLAGS) -o $@ -I.. $^ -lz -lm $(LIBS) + +ex-lift-check:ex-lift-check.c ../libminibwa.a + $(CC) $(CFLAGS) -o $@ -I.. $^ -lz -lm $(LIBS) + +ex-place-check:ex-place-check.c ../libminibwa.a + $(CC) $(CFLAGS) -o $@ -I.. $^ -lz -lm $(LIBS) + +ex-group-check:ex-group-check.c ../libminibwa.a + $(CC) $(CFLAGS) -o $@ -I.. $^ -lz -lm $(LIBS) clean: rm -f $(EXE) diff --git a/api-test/ex-group-check.c b/api-test/ex-group-check.c new file mode 100644 index 0000000..dee806f --- /dev/null +++ b/api-test/ex-group-check.c @@ -0,0 +1,59 @@ +#include +#include +#include +#include +#include "minibwa.h" +#include "kseq.h" +KSEQ_INIT(gzFile, gzread) + +/* Probe for Task 4 (mb_reconcile_alt grouping). mb_map() runs the full SE + * pipeline, including the gated mb_reconcile_alt pass, so the per-hit `parent` + * field already reflects the FINAL liftover grouping: a hit is its own group + * representative iff parent == id, otherwise it is a subordinate whose parent is + * the id of its group's representative. For every hit of every read it prints: + * + * ctg= qs= qe= rev= is_alt= \ + * id= parent= subsc= dp_max2= + * + * so the test can assert directly on the grouping (parent == id => independent + * representative; parent == => merged subordinate) without depending + * on SAM-flag heuristics that are insensitive to the merge decision. */ +int main(int argc, char *argv[]) +{ + mb_opt_t opt; + mb_opt_init(&opt); + + if (argc < 3) { + fprintf(stderr, "Usage: ex-group-check \n"); + return 1; + } + + gzFile f = gzopen(argv[2], "r"); + assert(f); + kseq_t *ks = kseq_init(f); + + mb_idx_t *idx = mb_idx_load(argv[1], 0); + assert(idx); + /* The loaders do not resolve .alt; the caller does. This example needs it. */ + mb_idx_set_alt_auto(idx, argv[1]); + + while (kseq_read(ks) >= 0) { + mb_hit_t *hit; + int32_t j, n_hit; + hit = mb_map(&opt, idx, ks->seq.l, ks->seq.s, 0, &n_hit, 0, ks->name.s); + for (j = 0; j < n_hit; ++j) { + mb_hit_t *h = &hit[j]; + printf("%s\tctg=%s\tqs=%d\tqe=%d\trev=%d\tis_alt=%d\tid=%d\tparent=%d\tsubsc=%d\tdp_max2=", + ks->name.s, mb_idx_ctg_name(idx, h->tid), + h->qs, h->qe, h->rev, h->is_alt, h->id, h->parent, h->subsc); + if (h->p) printf("%d\n", h->p->dp_max2); + else printf(".\n"); + free(h->p); + } + free(hit); + } + mb_idx_destroy(idx); + kseq_destroy(ks); + gzclose(f); + return 0; +} diff --git a/api-test/ex-lift-check.c b/api-test/ex-lift-check.c new file mode 100644 index 0000000..ffe3141 --- /dev/null +++ b/api-test/ex-lift-check.c @@ -0,0 +1,49 @@ +#include +#include +#include +#include "minibwa.h" +#include "l2bit.h" +/* Loads .l2b, sets .alt, then either: + * (default, no extra args) prints lift of (chrP_alt, pos) -> (chrP, pos): + * for pos<300 expect identity; for pos>=305 expect pos-5 (the 5bp insertion). + * (check mode) with trailing " " triples, asserts each + * lift's primary position equals (or expected<0 means "in a hole / + * not liftable"); prints one line per triple and exits nonzero on any + * mismatch. Also verifies the contig's lift[] is sorted by alt_st. */ +static int64_t find_tid(const l2b_t *l2b, const char *name){ + int64_t i; for(i=0;i<(int64_t)l2b->n_ctg;i++) if(!strcmp(l2b->ctg[i].name,name)) return i; return -1; +} +int main(int argc, char **argv){ + if (argc<2){fprintf(stderr,"usage: ex-lift-check [ ...]\n");return 2;} + char buf[1024]; snprintf(buf,sizeof buf,"%s.l2b",argv[1]); + l2b_t *l2b=l2b_load(buf); if(!l2b){fprintf(stderr,"no l2b\n");return 2;} + snprintf(buf,sizeof buf,"%s.alt",argv[1]); l2b_set_alt(l2b,buf); + if (argc>2){ /* check mode: (ctg,pos,expected) triples */ + int rc=0, a; + if ((argc-2)%3!=0){fprintf(stderr,"check mode needs triples\n");return 2;} + for(a=2;a+2ctg[at].is_alt){fprintf(stderr,"%s not flagged ALT\n",ctg);return 1;} + unsigned k; int sorted=1; + for(k=1;kctg[at].n_lift;k++) if(l2b->ctg[at].lift[k].alt_st < l2b->ctg[at].lift[k-1].alt_st) sorted=0; + int64_t pt; uint64_t pp=0; uint8_t rev; + int ok=l2b_lift(l2b,at,pos,&pt,&pp,&rev); + long got = ok? (long)pp : -1; + int pass = (got==exp) && sorted; + printf("lift(%s,%llu)=%ld rev=%d sorted=%d expected=%ld %s\n", + ctg,(unsigned long long)pos,got,ok?rev:-1,sorted,exp,pass?"OK":"MISMATCH"); + if(!pass) rc=1; + } + l2b_destroy(l2b); + return rc; + } + /* default (legacy) mode: chrP_alt at 100 and 400 */ + int64_t at=find_tid(l2b,"chrP_alt"); + if(at<0||!l2b->ctg[at].is_alt){fprintf(stderr,"chrP_alt not flagged ALT\n");return 1;} + int64_t pt1, pt2; uint64_t pp1, pp2; uint8_t rev1, rev2; + int ok1=l2b_lift(l2b,at,100,&pt1,&pp1,&rev1); + int ok2=l2b_lift(l2b,at,400,&pt2,&pp2,&rev2); + printf("lift(100)=%d pri=%llu lift(400)=%d pri=%llu\n",ok1,(unsigned long long)pp1,ok2,(unsigned long long)pp2); + return 0; +} diff --git a/api-test/ex-place-check.c b/api-test/ex-place-check.c new file mode 100644 index 0000000..d10fb3e --- /dev/null +++ b/api-test/ex-place-check.c @@ -0,0 +1,80 @@ +#include +#include +#include +#include +#include +#include "minibwa.h" +#include "mbpriv.h" /* mb_idx_s -> l2b for mb_hit_place */ +#include "kseq.h" +KSEQ_INIT(gzFile, gzread) + +/* Probe for Task 2 (mb_hit_place). For every hit of every read it prints: + * ts= te= rev= is_alt= cg= || + * place: pri= lst= len= prev= liftable= + * so the test script can assert on the placement (pri contig name, lifted_st, + * folded rev, liftable) without depending on internal tid numbering. + * + * Optional 3rd arg = "all" (default) or "coarse": when "coarse" the + * probe NULLs h->p before calling mb_hit_place to exercise the pre-DP [ts,te] + * path against the very same hit (so coarse vs exact can be compared directly). */ +int main(int argc, char *argv[]) +{ + mb_opt_t opt; + int coarse = 0; + mb_opt_init(&opt); + + if (argc < 3) { + fprintf(stderr, "Usage: ex-place-check [all|coarse]\n"); + return 1; + } + if (argc >= 4 && strcmp(argv[3], "coarse") == 0) coarse = 1; + + gzFile f = gzopen(argv[2], "r"); + assert(f); + kseq_t *ks = kseq_init(f); + + mb_idx_t *idx = mb_idx_load(argv[1], 0); + assert(idx); + /* The loaders do not resolve .alt; the caller does. This example needs it. */ + mb_idx_set_alt_auto(idx, argv[1]); + + while (kseq_read(ks) >= 0) { + mb_hit_t *hit; + int32_t i, j, n_hit; + hit = mb_map(&opt, idx, ks->seq.l, ks->seq.s, 0, &n_hit, 0, ks->name.s); + for (j = 0; j < n_hit; ++j) { + mb_hit_t *h = &hit[j]; + mb_extra_t *saved = h->p; + mb_place_t pl; + const char *pri_name; + + printf("%s\t%s\tts=%ld\tte=%ld\trev=%d\tis_alt=%d\tcg=", + ks->name.s, mb_idx_ctg_name(idx, h->tid), + (long)h->ts, (long)h->te, h->rev, h->is_alt); + if (h->p && h->p->n_cigar > 0) { + for (i = 0; i < h->p->n_cigar; ++i) + printf("%d%c", h->p->cigar[i]>>4, MB_CIGAR_STR[h->p->cigar[i]&0xf]); + } else putchar('.'); + + if (coarse) h->p = 0; /* force the pre-DP [ts,te] path */ + pl = mb_hit_place(idx->l2b, h); + /* Restore before printing so ctg name lookup via idx still works if + * needed. Free `saved` directly -- it is the one allocation we own; + * h->p after the restore is the same pointer, but freeing via `saved` + * makes ownership unambiguous and avoids any confusion with the + * subsequent free(hit) that releases only the flat hit array. */ + h->p = saved; + free(saved); + saved = h->p = NULL; /* prevent any accidental re-use */ + + pri_name = (pl.pri_tid >= 0) ? mb_idx_ctg_name(idx, pl.pri_tid) : "?"; + printf("\t||\tplace:\tpri=%s\tlst=%ld\tlen=%ld\tprev=%d\tliftable=%d\n", + pri_name, (long)pl.lifted_st, (long)pl.lifted_en, pl.rev, pl.liftable); + } + free(hit); + } + mb_idx_destroy(idx); + kseq_destroy(ks); + gzclose(f); + return 0; +} diff --git a/format.c b/format.c index 34085c2..c478f9f 100644 --- a/format.c +++ b/format.c @@ -274,7 +274,8 @@ void mb_fmt_sam(void *km, kstring_t *s, const l2b_t *l2b, const mb_bseq1_t *t, i if (t->qual) sam_write_sq(s, t->qual, t->l_seq, 0, 0); else kom_sprintf_lite(s, "*"); } else { - if ((flag & 0x900) == 0 || (opt->flag & MB_F_SUPP_SOFT)) { + if ((flag & 0x900) == 0 || (opt->flag & MB_F_SUPP_SOFT) + || ((opt->flag & MB_F_ALT_RECORDS) && r->is_alt)) { sam_write_sq(s, t->seq, t->l_seq, r->rev, r->rev); kom_sprintf_lite(s, "\t"); if (t->qual) sam_write_sq(s, t->qual, t->l_seq, r->rev, 0); diff --git a/l2bit.c b/l2bit.c index 81473ea..efc8b59 100644 --- a/l2bit.c +++ b/l2bit.c @@ -1,5 +1,7 @@ +#define _POSIX_C_SOURCE 200809L #include #include +#include #include #include "kommon.h" #include "l2bit.h" @@ -145,6 +147,13 @@ static void l2b_add_seq(l2b_t *l2b, uint64_t len, const char *seq, const char *n ctg->comm = comm? kom_strdup(comm) : 0; ctg->len = len; ctg->off = l2b->tot_len; + /* kom_grow() uses realloc(), which does NOT zero new slots, so the ALT fields + * must be initialized explicitly here. Without this, l2b_destroy() frees an + * uninitialized ctg->lift on the plain `minibwa index` path (l2b_set_alt is + * never called there), which crashes on any multi-contig reference. */ + ctg->is_alt = 0; + ctg->n_lift = 0; + ctg->lift = 0; l2b->tot_len += len; m_pac_old = l2b->m_pac; @@ -230,6 +239,11 @@ l2b_t *l2b_import(const char *fn, uint64_t seed) void l2b_destroy(l2b_t *l2b) { + uint64_t i; + // lift[] is built by l2b_set_alt() with realloc() after the index is loaded, + // so it is heap-allocated in the mmap case too (ctg[] itself always is). + for (i = 0; i < l2b->n_ctg; ++i) + free(l2b->ctg[i].lift); if (l2b->mmap) { // ambi/mask/pac/cat_name/cat_comm point into the mapped file free(l2b->ctg); // ctg[] is always heap-allocated kom_munmap(l2b->mmap, l2b->mmap_len); @@ -240,6 +254,261 @@ void l2b_destroy(l2b_t *l2b) free(l2b); } +/**************************** + * ALT liftover index * + ****************************/ + +/* Order lift blocks by ALT-forward start (then end) so l2b_lift()'s binary + * search over [alt_st, alt_en) is valid. Blocks are appended in file order, + * which is non-monotonic for reverse records (emitted descending after the + * RC->forward remap) and for ALT contigs carrying multiple .alt records. */ +static int l2b_lift_cmp(const void *a, const void *b) +{ + const l2b_lift_t *x = (const l2b_lift_t*)a, *y = (const l2b_lift_t*)b; + if (x->alt_st != y->alt_st) return x->alt_st < y->alt_st? -1 : 1; + if (x->alt_en != y->alt_en) return x->alt_en < y->alt_en? -1 : 1; + /* qsort is not stable, so equal keys would reorder arbitrarily between libc + * implementations and make the lift -- and therefore the SAM -- differ across + * platforms. Two records CAN tie on the ALT span and differ in target, which + * multi-record ALT contigs make reachable. Break every tie. */ + if (x->pri_tid != y->pri_tid) return x->pri_tid < y->pri_tid? -1 : 1; + if (x->pri_st != y->pri_st) return x->pri_st < y->pri_st? -1 : 1; + if (x->rev != y->rev) return x->rev < y->rev? -1 : 1; + return 0; +} + +int l2b_set_alt(l2b_t *l2b, const char *fn) +{ + FILE *fp; + char *line = 0; + size_t line_cap = 0; + ssize_t line_len; + int n_alt = 0; + uint64_t i; + + /* Reset: free any existing lift blocks and clear flags. */ + for (i = 0; i < l2b->n_ctg; ++i) { + free(l2b->ctg[i].lift); + l2b->ctg[i].lift = 0; + l2b->ctg[i].n_lift = 0; + l2b->ctg[i].is_alt = 0; + } + l2b->n_alt_ctg = 0; + + fp = fopen(fn, "r"); + if (fp == 0) return -1; + + while ((line_len = getline(&line, &line_cap, fp)) > 0) { + char *p, *q; + char *fields[12]; + int nf; + int64_t alt_tid, pri_tid; + long pos1; + uint64_t pri_pos, alt_cursor, pri_cursor; + uint32_t flag; + uint8_t rev; + uint32_t m_lift, n_lift0; + int bad; + l2b_ctg_t *ctg; + + /* Skip SAM header lines. */ + if (line[0] == '@') continue; + + /* Trim trailing newline. */ + if (line_len > 0 && line[line_len-1] == '\n') line[--line_len] = '\0'; + if (line_len > 0 && line[line_len-1] == '\r') line[--line_len] = '\0'; + + /* Split into at least 12 tab-delimited fields. */ + for (nf = 0, p = line; nf < 12; ++nf) { + fields[nf] = p; + q = strchr(p, '\t'); + if (q) { *q = '\0'; p = q + 1; } + else { ++nf; break; } + } + if (nf < 6) continue; /* need QNAME, FLAG, RNAME, POS, MAPQ, CIGAR */ + + /* Locate ALT contig by QNAME (col 0). */ + alt_tid = -1; + for (i = 0; i < l2b->n_ctg; ++i) + if (strcmp(l2b->ctg[i].name, fields[0]) == 0) { alt_tid = i; break; } + if (alt_tid < 0) { + if (kom_verbose >= 2) + fprintf(stderr, "[W::%s] QNAME '%s' not in index, skipping\n", __func__, fields[0]); + continue; + } + + /* Unmapped ALT/decoy entries (FLAG 0x4 or RNAME '*') carry no primary + * span; a standard hs38DH .alt ships ~2400 such decoy lines. They were + * never loadable, so skip them silently — warning on each is just noise + * and would flood stderr on every real run. */ + if ((atol(fields[1]) & 0x4) || strcmp(fields[2], "*") == 0) + continue; + + /* Locate primary contig by RNAME (col 2). */ + pri_tid = -1; + for (i = 0; i < l2b->n_ctg; ++i) + if (strcmp(l2b->ctg[i].name, fields[2]) == 0) { pri_tid = i; break; } + if (pri_tid < 0) { + if (kom_verbose >= 2) + fprintf(stderr, "[W::%s] RNAME '%s' not in index, skipping\n", __func__, fields[2]); + continue; + } + + flag = (uint32_t)atol(fields[1]); + rev = (flag & 0x10) ? 1 : 0; + + /* POS is 1-based; a valid mapped record has POS >= 1. A malformed POS=0 + * (or negative) would underflow to a garbage primary coordinate, so skip. */ + pos1 = atol(fields[3]); + if (pos1 < 1) { + if (kom_verbose >= 2) + fprintf(stderr, "[W::%s] QNAME '%s' has invalid POS '%s', skipping\n", __func__, fields[0], fields[3]); + continue; + } + pri_pos = (uint64_t)(pos1 - 1); + + ctg = &l2b->ctg[alt_tid]; + /* n_alt_ctg counts contigs (a contig may carry several records via + * supplementary lines), so only the first record for a contig advances it. */ + if (!ctg->is_alt) ++l2b->n_alt_ctg; + ctg->is_alt = 1; + ++n_alt; + + /* Walk CIGAR to build lift blocks. alt_cursor tracks the position along + * the CIGAR's query axis: for a forward record this IS the ALT-forward + * coordinate; for a reverse record it is the reverse-complement coordinate, + * so each M block is remapped to ALT-forward below. A final sort over all + * of the contig's blocks restores the alt_st ordering l2b_lift() needs. */ + alt_cursor = 0; + pri_cursor = pri_pos; + m_lift = ctg->n_lift; + n_lift0 = ctg->n_lift; /* rollback point if the CIGAR is malformed */ + bad = 0; + + p = fields[5]; + while (*p) { + uint64_t len = 0; + int op; + if (*p < '0' || *p > '9') { bad = 1; break; } /* expected an op length */ + while (*p >= '0' && *p <= '9') len = len * 10 + (*p++ - '0'); + if (*p == '\0') { bad = 1; break; } /* length with no operator (ran off the field) */ + op = *p++; + if (op == 'M' || op == '=' || op == 'X') { + /* Emit one lift block. */ + l2b_lift_t blk; + blk.alt_st = alt_cursor; + blk.alt_en = alt_cursor + len; + blk.pri_tid = pri_tid; + blk.pri_st = pri_cursor; + blk.pri_en = pri_cursor + len; + blk.rev = rev; + + /* Reverse record: the CIGAR walks the reverse-complement of the ALT + * contig, so [alt_st, alt_en) is in RC coordinates. Remap to + * ALT-forward ([len_alt-alt_en, len_alt-alt_st)) while leaving + * pri_st/pri_en unchanged; l2b_lift()'s reverse branch then maps the + * forward query position back with its decreasing correspondence. */ + if (rev) { + /* The remap subtracts from ctg->len, so a record whose query span + * runs past the contig it names would wrap to ~UINT64_MAX and sort + * to the end of lift[], where the binary search reads it as a valid + * block -- silently wrong lifts, no crash. Reject the record. */ + if (blk.alt_en > ctg->len) { bad = 1; break; } + uint64_t fst = ctg->len - blk.alt_en; + uint64_t fen = ctg->len - blk.alt_st; + blk.alt_st = fst; + blk.alt_en = fen; + } + + kom_grow(l2b_lift_t, ctg->lift, ctg->n_lift, m_lift); + ctg->lift[ctg->n_lift++] = blk; + + alt_cursor += len; + pri_cursor += len; + } else if (op == 'I' || op == 'S' || op == 'H') { + /* Query/ALT-consuming ops with no primary footprint. A hard clip + * removes bases from SEQ but they still occupy the ALT contig, so + * (like S and I) it advances the ALT axis -- essential for a + * supplementary .alt record whose leading H is its offset into the + * ALT contig. */ + alt_cursor += len; + } else if (op == 'D' || op == 'N') { + /* Deletion from ALT: advance only pri_cursor. */ + pri_cursor += len; + } else if (op == 'P') { + /* Padding: consumes neither coordinate. */ + } else { + bad = 1; break; /* unknown CIGAR operator */ + } + } + if (bad) { + /* Roll back this record's partial blocks. The buffer keeps whatever + * capacity the partial append grew it to; shrink so the next record's + * `m_lift = ctg->n_lift` still describes the real allocation, as it does + * on every non-rollback path. */ + ctg->n_lift = n_lift0; + if (ctg->n_lift > 0 && ctg->n_lift < m_lift) + ctg->lift = kom_realloc(l2b_lift_t, ctg->lift, ctg->n_lift); + if (kom_verbose >= 2) + fprintf(stderr, "[W::%s] QNAME '%s' has an unparsable CIGAR '%s', skipping record\n", __func__, fields[0], fields[5]); + continue; + } + + /* Shrink the lift array to exact size. */ + if (ctg->n_lift > 0 && ctg->n_lift < m_lift) + ctg->lift = kom_realloc(l2b_lift_t, ctg->lift, ctg->n_lift); + } + + free(line); + fclose(fp); + + /* Blocks were appended in file order. Reverse records and multi-record ALT + * contigs make lift[] non-monotonic in alt_st, but l2b_lift()'s binary search + * requires it sorted. Sort each ALT contig's blocks now that every record is + * loaded. */ + for (i = 0; i < l2b->n_ctg; ++i) { + l2b_ctg_t *c = &l2b->ctg[i]; + if (c->n_lift > 1) + qsort(c->lift, c->n_lift, sizeof(l2b_lift_t), l2b_lift_cmp); + } + + return n_alt; +} + +int l2b_lift(const l2b_t *l2b, int64_t alt_tid, uint64_t alt_pos, + int64_t *pri_tid, uint64_t *pri_pos, uint8_t *rev) +{ + const l2b_ctg_t *ctg; + int64_t lo, hi, mid; + + if (alt_tid < 0 || alt_tid >= (int64_t)l2b->n_ctg) return 0; + ctg = &l2b->ctg[alt_tid]; + if (!ctg->is_alt || ctg->n_lift == 0) return 0; + + /* Binary search for the block whose [alt_st, alt_en) contains alt_pos. */ + lo = 0; hi = ctg->n_lift; + while (lo < hi) { + mid = (lo + hi) / 2; + if (ctg->lift[mid].alt_en <= alt_pos) lo = mid + 1; + else hi = mid; + } + if (lo >= (int64_t)ctg->n_lift) return 0; + { + const l2b_lift_t *blk = &ctg->lift[lo]; + if (alt_pos < blk->alt_st) return 0; /* in a hole */ + *pri_tid = blk->pri_tid; + *rev = blk->rev; + if (!blk->rev) { + /* Forward: primary coord increases with alt coord. */ + *pri_pos = blk->pri_st + (alt_pos - blk->alt_st); + } else { + /* Reverse: primary coord decreases as alt coord increases. */ + *pri_pos = blk->pri_en - 1 - (alt_pos - blk->alt_st); + } + } + return 1; +} + int l2b_save(const char *fn, const l2b_t *l2b) { FILE *fp; diff --git a/l2bit.h b/l2bit.h index cd1b744..05389db 100644 --- a/l2bit.h +++ b/l2bit.h @@ -8,9 +8,22 @@ typedef enum { L2B_METH_NONE=0, L2B_METH_C2T, L2B_METH_G2A } l2b_meth_t; +/* One aligned block from the .alt CIGAR (one M/=/X run). + * Maps ALT positions [alt_st, alt_en) to primary positions [pri_st, pri_st+len) + * (forward) or [pri_en-len, pri_en) traversed in reverse (rev). */ +typedef struct { + uint64_t alt_st, alt_en; /* half-open interval on ALT contig */ + int64_t pri_tid; /* primary contig index */ + uint64_t pri_st, pri_en; /* half-open interval on primary contig */ + uint8_t rev; /* 1 = ALT is RC-aligned to primary */ +} l2b_lift_t; + typedef struct { char *name, *comm; uint64_t len, off; + uint8_t is_alt; /* 1 if this contig appears as QNAME in the .alt file */ + uint32_t n_lift; /* number of lift blocks */ + l2b_lift_t *lift; /* sorted by alt_st; NULL when n_lift==0 */ } l2b_ctg_t; typedef struct { @@ -20,6 +33,7 @@ typedef struct { typedef struct { uint64_t tot_len; uint64_t n_ctg, m_ctg; + uint32_t n_alt_ctg; /* contigs with is_alt set; 0 when no .alt was loaded */ l2b_ctg_t *ctg; uint64_t n_pac, m_pac; uint64_t n_ambi, m_ambi; @@ -44,6 +58,18 @@ int64_t l2b_getseq(const l2b_t *l2b, int64_t tid, int64_t st, int64_t en, uint8_ int64_t l2b_getambi(const l2b_t *l2b, int64_t tid, int64_t st, int64_t en, int32_t *n_ambi); void l2b_meth_convert(l2b_meth_t mt, int64_t len, uint8_t *seq); +/* Load an ALT file (SAM), set l2b_ctg_t.is_alt and populate lift[] blocks for each + * ALT contig. Safe to call multiple times (re-entrant: frees old lift[] first). + * Returns the number of ALT records parsed (a contig may carry several; the + * contig count is l2b->n_alt_ctg), or -1 on file error. */ +int l2b_set_alt(l2b_t *l2b, const char *fn); + +/* Map (alt_tid, alt_pos) to (pri_tid, pri_pos). alt_pos is 0-based. + * Returns 1 if liftable (fills *pri_tid, *pri_pos, *rev), + * 0 if in a hole (ALT-only insertion; pri_tid, pri_pos, rev left unchanged). */ +int l2b_lift(const l2b_t *l2b, int64_t alt_tid, uint64_t alt_pos, + int64_t *pri_tid, uint64_t *pri_pos, uint8_t *rev); + l2b_t *l2b_import(const char *fn, uint64_t seed); int l2b_save(const char *fn, const l2b_t *l2b); int l2b_save_pac(const char *fn, const l2b_t *l2b, int both_strand); diff --git a/map-algo.c b/map-algo.c index 0bd66ce..84f0cb6 100644 --- a/map-algo.c +++ b/map-algo.c @@ -66,6 +66,211 @@ mb_idx_t *mb_idx_load_mmap(const char *prefix, int32_t is_meth, int preload) return idx; } +int mb_idx_set_alt(mb_idx_t *idx, const char *fn) +{ + if (idx == 0 || idx->l2b == 0) return -1; + return l2b_set_alt(idx->l2b, fn); +} + +int mb_idx_set_alt_auto(mb_idx_t *idx, const char *prefix) +{ + char *buf; + int ret; + if (idx == 0 || idx->l2b == 0) return -1; + buf = kom_calloc(char, strlen(prefix) + 5); + strcat(strcpy(buf, prefix), ".alt"); + ret = l2b_set_alt(idx->l2b, buf); /* -1 (ENOENT) if absent — the common case */ + free(buf); + return ret; +} + +/* Reference (target) span the hit's CIGAR consumes, as a half-open interval on + * h->tid. Post-DP (h->p != NULL) this walks the CIGAR's reference-consuming ops + * (M/=/X/D/N) starting at h->ts; pre-DP (h->p == NULL) it falls back to the + * coarse chain interval [ts,te). Either way the footprint starts at h->ts. */ +static void mb_hit_footprint(const mb_hit_t *h, int64_t *fp_st, int64_t *fp_en) +{ + *fp_st = h->ts; + if (h->p != 0 && h->p->n_cigar > 0) { + int64_t pos = h->ts; + int32_t k; + for (k = 0; k < h->p->n_cigar; ++k) { + uint32_t op = h->p->cigar[k] & 0xf, len = h->p->cigar[k] >> 4; + if (op == MB_CIGAR_MATCH || op == MB_CIGAR_EQ_MATCH || op == MB_CIGAR_X_MISMATCH || + op == MB_CIGAR_DEL || op == MB_CIGAR_N_SKIP) + pos += len; + } + *fp_en = pos; + } else { + *fp_en = h->te; + } + if (*fp_en <= *fp_st) *fp_en = *fp_st + 1; /* guard against a degenerate span */ +} + +mb_place_t mb_hit_place(const l2b_t *l2b, const mb_hit_t *h) +{ + mb_place_t pl; + int64_t fp_st, fp_en; + const l2b_ctg_t *ctg; + uint32_t b; + int64_t en = 0; /* max over lifted PRIMARY outputs (cosmetic lifted_en) */ + + mb_hit_footprint(h, &fp_st, &fp_en); + + /* Non-ALT hit (or no .alt loaded / no lift blocks): identity passthrough. + * One sub-placement at the hit's own position; representative fields are + * byte-identical to the pre-multi-interval baseline so non-ALT grouping is + * unchanged. */ + if (!h->is_alt || h->tid < 0 || h->tid >= (int64_t)l2b->n_ctg || + !l2b->ctg[h->tid].is_alt || l2b->ctg[h->tid].n_lift == 0) { + pl.pri_tid = h->tid; + pl.lifted_st = fp_st; + pl.lifted_en = fp_en; + pl.rev = h->rev; + pl.liftable = 1; + pl.n_subpl = 1; + pl.subpl[0].st = fp_st; + pl.subpl[0].pri_tid = h->tid; + pl.subpl[0].rev = h->rev; + return pl; + } + + /* ALT hit: lift the footprint through the .alt span-lift. We walk the lift + * blocks: a reverse block maps alt_st to the HIGH primary coordinate (not the + * low one), and the footprint may begin/end in a hole (a gap between blocks). + * Rather than min/max-COLLAPSING every overlapping block into a single span + * (which inflates the placement by an SV-scale indel and drags the grouping + * key thousands of bp from a breakpoint-spanning twin), we record ONE + * sub-placement per overlapping block: subpl[k].st is that block's + * representative (min) lifted primary coordinate. A breakpoint-spanning ALT + * hit thus keeps a sub-interval AT each side of the breakpoint, so it can + * still co-locate with its primary twin via the matching one. Each block + * carries its OWN (pri_tid, rev): a multi-contig/mixed-strand ALT scaffold + * (which the old single-interval code flagged liftable=0 and dropped) now + * yields per-block sub-placements that co-locate only with a primary hit + * sharing that block's (pri_tid, rev) -- safe, and strictly more recoverable. */ + ctg = &l2b->ctg[h->tid]; + pl.n_subpl = 0; + for (b = 0; b < ctg->n_lift; ++b) { + const l2b_lift_t *blk = &ctg->lift[b]; + int64_t a_lo, a_hi; /* clamped ALT endpoints inside the footprint */ + int64_t p0, p1, blk_st; + int64_t pt0, pt1; uint64_t pp0, pp1; uint8_t rv0, rv1; + uint8_t blk_rev; + + if ((int64_t)blk->alt_en <= fp_st || (int64_t)blk->alt_st >= fp_en) + continue; /* block does not overlap the footprint */ + a_lo = (int64_t)blk->alt_st > fp_st ? (int64_t)blk->alt_st : fp_st; + a_hi = (int64_t)blk->alt_en < fp_en ? (int64_t)blk->alt_en : fp_en; + a_hi -= 1; /* inclusive last liftable ALT base in this block */ + if (a_hi < a_lo) continue; + + if (!l2b_lift(l2b, h->tid, (uint64_t)a_lo, &pt0, &pp0, &rv0)) continue; + if (!l2b_lift(l2b, h->tid, (uint64_t)a_hi, &pt1, &pp1, &rv1)) continue; + p0 = (int64_t)pp0; p1 = (int64_t)pp1; + blk_st = p0 < p1 ? p0 : p1; /* block's representative (min) lifted start */ + + /* MB_MAX_SUBPL spill: a footprint over a normal .alt CIGAR overlaps only a + * couple of blocks, but a pathologically fragmented region could exceed the + * cap. Keep the FIRST MB_MAX_SUBPL blocks and drop the rest. Dropping a + * candidate sub-interval can only REMOVE a possible co-location, never add a + * spurious one, so the cap is safe for paralog isolation. */ + if (pl.n_subpl >= MB_MAX_SUBPL) break; + + blk_rev = (uint8_t)(rv0 ^ h->rev); + pl.subpl[pl.n_subpl].st = blk_st; + pl.subpl[pl.n_subpl].pri_tid = pt0; + pl.subpl[pl.n_subpl].rev = blk_rev; + ++pl.n_subpl; + + /* Track the max lifted primary coordinate for the cosmetic lifted_en. */ + if (pl.n_subpl == 1) en = blk_st; + if (p0 > en) en = p0; + if (p1 > en) en = p1; + } + + if (pl.n_subpl == 0) { + /* Entire footprint falls in holes -> ALT-specific; own group. */ + pl.pri_tid = h->tid; + pl.lifted_st = fp_st; + pl.lifted_en = fp_en; + pl.rev = h->rev; + pl.liftable = 0; + return pl; + } + + /* Representative fields come from subpl[0] (back-compat); lifted_en is the max + * over sub-placements (cosmetic -- no decision reads it). */ + pl.pri_tid = pl.subpl[0].pri_tid; + pl.lifted_st = pl.subpl[0].st; + pl.lifted_en = en + 1; /* half-open upper bound */ + pl.rev = pl.subpl[0].rev; + pl.liftable = 1; + return pl; +} + +/* Multi-interval co-location test (SV-breakpoint-aware grouping primitive). + * + * Two placements co-locate iff BOTH are liftable AND there exists a pair of + * sub-placements (one from each) that share a primary contig and strand and whose + * representative lifted starts are within lift_tol bp. This generalizes the old + * scalar "|Δlifted_st| <= lift_tol on the single collapsed interval" test: + * - A breakpoint-spanning ALT hit (multiple sub-placements straddling an + * SV-scale indel) co-locates with its primary twin via the matching + * sub-interval, instead of being dragged away by a min/max-collapsed span. + * - PARALOG SAFETY: co-location still requires a SHARED (pri_tid, rev, |Δst|<=tol) + * sub-interval. Two distinct primary loci have single, far-apart + * sub-placements and therefore never co-locate -- the multi-interval change + * can ONLY let an ALT hit join the group of a primary at one of its own lifted + * positions; it can NOT merge two primary loci. (Non-ALT hits carry exactly + * one sub-placement at their own position, so two primaries reduce to the + * original scalar test.) + * + * O(n_subpl_a * n_subpl_b), each bounded by MB_MAX_SUBPL. */ +int mb_places_colocate(const mb_place_t *a, const mb_place_t *b, int lift_tol) +{ + int i, j; + if (!a->liftable || !b->liftable) return 0; + for (i = 0; i < a->n_subpl; ++i) { + for (j = 0; j < b->n_subpl; ++j) { + int64_t d; + if (a->subpl[i].pri_tid != b->subpl[j].pri_tid) continue; + if (a->subpl[i].rev != b->subpl[j].rev) continue; + d = a->subpl[i].st - b->subpl[j].st; + if (d < 0) d = -d; + if (d <= lift_tol) return 1; + } + } + return 0; +} + +/* Does ALT contig `alt_tid`'s .alt mapping cover primary position `pos` on + * `pri_tid`? An ALT contig is, by GRCh38 construction, an alternate of a + * specific primary REGION; its lift blocks (with internal indel gaps) span + * [min pri_st, max pri_en) on each primary contig they touch. Containment is + * tested in that OVERALL span (gaps included) so a position that falls in an + * .alt deletion gap -- or whose ALT twin lifts there through an insertion hole + * -- is still recognized as inside the ALT's primary region. This is the + * .alt-established correspondence used to fold an ALT twin onto a primary hit + * the per-base lift could not co-locate (mb_reconcile_alt step 2b). */ +static int mb_alt_covers_primary(const l2b_t *l2b, int64_t alt_tid, int64_t pri_tid, int64_t pos) +{ + const l2b_ctg_t *ctg; + uint32_t b; + int64_t lo = -1, hi = -1; + if (alt_tid < 0 || alt_tid >= (int64_t)l2b->n_ctg) return 0; + ctg = &l2b->ctg[alt_tid]; + if (!ctg->is_alt || ctg->n_lift == 0) return 0; + for (b = 0; b < ctg->n_lift; ++b) { + const l2b_lift_t *blk = &ctg->lift[b]; + if (blk->pri_tid != pri_tid) continue; + if (lo < 0 || (int64_t)blk->pri_st < lo) lo = (int64_t)blk->pri_st; + if (hi < 0 || (int64_t)blk->pri_en > hi) hi = (int64_t)blk->pri_en; + } + if (lo < 0) return 0; /* contig does not map to pri_tid */ + return pos >= lo && pos < hi; +} + void mb_idx_destroy(mb_idx_t *idx) { if (idx == 0) return; @@ -239,6 +444,7 @@ mb_hit_t *mb_gen_hit(void *km, uint32_t hash, int qlen, const l2b_t *l2b, int n_ ri->cnt = (int32_t)z[i].y; ri->as = z[i].y >> 32; mb_hit_set_coor(ri, qlen, l2b, a); + ri->is_alt = l2b->ctg[ri->tid].is_alt; } kfree(km, z); return r; @@ -363,6 +569,12 @@ void mb_set_parent(void *km, float mask_level, int mask_len, int n, mb_hit_t *r, kfree(km, w); } +/* Mark the SAM primary among the group representatives (parent==id). + * + * Selection is per-read: the 5'-most representative (smallest query start) when + * is_primary5 is set, otherwise the first representative by index. The pairing + * pass does not disambiguate the primary here -- upstream sets sam_pri on the + * pair-chosen endpoint separately (see the paired branch in pe.c). */ int32_t mb_set_sam_pri(int32_t n, mb_hit_t *r, int32_t is_primary5) { int32_t i, new_pri, n_pri = 0, min_i = -1, min_qs = -1, first_i = -1; @@ -380,11 +592,41 @@ int32_t mb_set_sam_pri(int32_t n, mb_hit_t *r, int32_t is_primary5) return new_pri; } -void mb_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, mb_hit_t *r) +/* Check whether hit r co-locates with any of the n_kept placements in kpl[]. + * Co-location is the multi-interval test (mb_places_colocate): any sub-interval + * of r matches any sub-interval of a kept placement (same pri_tid, rev, + * |Δst| <= lift_tol). */ +static int mb_place_matches_any(const l2b_t *l2b, const mb_hit_t *r, + const mb_place_t *kpl, int n_kept, int lift_tol) +{ + mb_place_t pl; + int j; + if (n_kept == 0) return 0; + pl = mb_hit_place(l2b, r); /* coarse: r->p may be NULL (pre-DP) */ + if (!pl.liftable) return 0; + for (j = 0; j < n_kept; ++j) + if (mb_places_colocate(&kpl[j], &pl, lift_tol)) return 1; + return 0; +} + +void mb_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, mb_hit_t *r, + const l2b_t *l2b, int lift_tol) { if (pri_ratio > 0.0f && *n_ > 0) { int i, k, n = *n_, n_2nd = 0; uint8_t *keep = Kcalloc(km, uint8_t, n); + /* Pass 1: existing keep logic, plus record each kept hit's lifted placement + * so pass 2 can find kept primaries that sit at higher indices than their + * ALT twins. */ + mb_place_t *kept_pl = 0; + int n_kept = 0; + /* The survival guard below can only fire on an ALT hit, and hit->is_alt is + * copied from l2b->ctg[tid].is_alt -- so with no .alt loaded no hit is ever + * ALT, kept_pl is written but never read, and every mb_hit_place() call is + * wasted. Gating on the contig count keeps that cost off the common path; + * `l2b` itself is the sequence index and is always present. */ + const int use_lift = l2b && l2b->n_alt_ctg > 0; + if (use_lift) kept_pl = Kmalloc(km, mb_place_t, n); /* worst-case: all kept */ for (i = 0; i < n; ++i) { int p = r[i].parent; if (p == i || r[i].inv) { @@ -393,6 +635,28 @@ void mb_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, if (!(r[i].qs == r[p].qs && r[i].qe == r[p].qe && r[i].tid == r[p].tid && r[i].ts == r[p].ts && r[i].te == r[p].te)) keep[i] = 1, ++n_2nd; } + if (use_lift && keep[i]) { + /* Record non-ALT primaries too: their "lifted" placement is their own + * position, which is exactly the target co-location ALT twins must match. */ + mb_place_t pl = mb_hit_place(l2b, &r[i]); + if (pl.liftable) kept_pl[n_kept++] = pl; + } + } + /* Pass 2 (survival guard): for each not-yet-kept ALT hit, force-keep it + * if its lifted placement co-locates with any already-kept hit. This is + * intentionally generous: over-keeping is cheap; the authoritative grouping + * happens later. Never make this tolerance tighter than MB_LIFT_TOL. */ + if (use_lift) { + /* --dbg-no-alt-survive ablates just this guard (testing), so a fixture + * can assert the RED state (twin dropped) as well as the GREEN one. */ + int ablate = (kom_dbg_flag & MB_DBG_NO_ALT_SURVIVE) != 0; + if (!ablate) for (i = 0; i < n; ++i) { + if (!keep[i] && r[i].is_alt) { + if (mb_place_matches_any(l2b, &r[i], kept_pl, n_kept, lift_tol)) + keep[i] = 1; + } + } + kfree(km, kept_pl); } for (i = k = 0; i < n; ++i) { if (keep[i]) r[k++] = r[i]; @@ -404,6 +668,250 @@ void mb_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, } } +/* True iff any hit is from an ALT contig. The reconciliation pass is gated on + * this so non-ALT reads (and any reference with no .alt loaded) pay nothing. */ +int mb_any_alt(int n_hit, const mb_hit_t *hit) +{ + int i; + for (i = 0; i < n_hit; ++i) + if (hit[i].is_alt) return 1; + return 0; +} + +/* Two hits' DP-adjusted scores (dp_max if extended, else the chaining score). */ +static inline int32_t mb_hit_dpscore(const mb_hit_t *h) +{ + return h->p ? h->p->dp_max : h->score; +} + +/* Query spans [qs,qe) of two hits overlap (share at least one query base). + * Non-overlapping spans are chimeric segments of the read, not competitors for + * the same locus, so they must NOT contribute to one another's suboptimal. */ +static inline int mb_qspan_overlap(const mb_hit_t *a, const mb_hit_t *b) +{ + return a->qs < b->qe && b->qs < a->qe; +} + +/* Post-extension liftover-group reconciliation (the heart of the ALT feature). + * + * Surviving hits are grouped by their lifted PLACEMENT (same pri_tid, same + * strand on primary, and |Δlifted_st| <= MB_LIFT_TOL). Each group represents + * one primary locus; an ALT twin and the primary hit it lifts onto land in the + * SAME group. Within a group the representative (the surviving "primary" hit, + * parent==id) is the highest-DP-scoring member; the others become subordinates + * (parent := rep->id) so mb_set_sam_pri/mb_set_mapq treat them as secondary. + * + * The MAPQ-critical step is recomputing the group-scoped suboptimal fields so a + * read's MAPQ reflects the second-best GROUP, not an ALT twin of its own locus. + * subsc / n_sub / dp_max2 were written by the :778/:784 mb_set_parent run as + * running maxima/accumulators; here we ZERO them on every (post-grouping) + * representative and recompute over representatives ONLY: + * - dp_max2 = best OTHER-group rep dp_max whose query span overlaps this rep + * - subsc = that competing score + * - n_sub = count of other-group reps within sub_diff of this rep + * Intra-group members contribute nothing. A hit promoted from subordinate to + * representative by the new grouping is zeroed too: it still carries stale + * numbers from when it was a child under the old parent. + * + * @param l2b span-lift index (.alt loaded; used by mb_hit_place) + * @param n_hit number of surviving hits + * @param hit the hits (modified in place: parent, subsc, n_sub, dp_max2) + * @param sub_diff score window for counting near-tied competing groups (n_sub); + * matches the value passed to mb_set_parent in the driver. + */ +void mb_reconcile_alt(void *km, const l2b_t *l2b, int n_hit, mb_hit_t *hit, int sub_diff, int lift_tol) +{ + mb_place_t *pl; + int *grp; /* group id (union-find style, flattened) per hit */ + int *rep; /* representative hit index per hit (== own index if rep) */ + int i, j; + + if (n_hit <= 0) return; + + /* Per-thread kalloc arena (lock-free); NOT libc malloc -- on ALT-heavy WGS + * this runs per read across all worker threads, and routing it through the + * shared allocator serialized them on the mimalloc arena lock (threads parked + * in __ulock_wait, parallelism 8.8x->6.8x). Kmalloc/kfree on km is the + * codebase convention and is contention-free. (km==NULL falls back to libc + * via the kalloc macros, preserving the MB_F_NO_KALLOC path.) */ + pl = Kmalloc(km, mb_place_t, n_hit); + grp = Kmalloc(km, int, n_hit); + rep = Kmalloc(km, int, n_hit); + /* Graceful on allocation failure: with km!=NULL the arena aborts on OOM, but + * the km==NULL (MB_F_NO_KALLOC) path falls back to libc malloc which can + * return NULL -- return rather than deref. kfree() is NULL-safe. */ + if (!pl || !grp || !rep) { kfree(km, pl); kfree(km, grp); kfree(km, rep); return; } + + /* 1. lifted placement of every hit. */ + for (i = 0; i < n_hit; ++i) + pl[i] = mb_hit_place(l2b, &hit[i]); + + /* 2. group by lifted placement via transitive close-up: for each co-locating + * pair in different groups, unite the groups -- but only after an all-cross- + * pairs guard (the t/u loops below) confirms EVERY member pair co-locates AND + * query-overlaps, which is what stops chimeric segments and paralogs from + * drifting together. Unliftable hits never match and stay singletons. + * + * Complexity: the guard makes a single merge O(n_hit^2), so the loop is + * O(n_hit^4) in the worst case, NOT O(n_hit^2). In practice it stays near + * O(n_hit^2): (a) n_hit is small -- default max_occ/out_n subsampling caps the + * surviving hits, and even a 40-copy segdup measured n_hit=41; (b) the + * `grp[j]==grp[i]` short-circuit skips within-group pairs, and hits of one + * read that co-locate also query-overlap, so they merge on first contact and + * collapse to one group instead of re-triggering the guard. A true union- + * find would drop the exponent but cannot express the all-cross-pairs guard + * (it unites on a single edge), so it would merge groups this must keep apart. + * If a profile ever shows this hot, cap n_hit rather than change the grouping + * semantics. */ + for (i = 0; i < n_hit; ++i) grp[i] = i; /* initially own group */ + for (i = 0; i < n_hit; ++i) { + if (!pl[i].liftable) continue; + for (j = 0; j < i; ++j) { + if (!pl[j].liftable) continue; + if (grp[j] == grp[i]) continue; + if (mb_places_colocate(&pl[i], &pl[j], lift_tol)) { + /* Candidate merge of i's group (old) into j's group (neu). + * Two hits are the SAME read alignment on primary vs ALT only + * if they CO-LOCATE (share a lifted sub-interval: same pri_tid, + * same strand, |Δst| <= MB_LIFT_TOL) AND their query spans + * overlap. Grouping is by transitive relabel for clarity, but + * transitivity can drift: A~B and B~C may collapse A,C even when + * A,C do not co-locate or have disjoint query spans (chimeric + * segments). Guard by requiring EVERY cross-pair between the two + * groups to satisfy BOTH conditions before uniting them. The + * sub-interval co-location requirement preserves paralog safety: + * a breakpoint-spanning ALT hit joins via its matching + * sub-interval, but two distinct primary loci (single, far-apart + * sub-placements) still never share one. */ + int old = grp[i], neu = grp[j], t, u, ok = 1; + for (t = 0; t < n_hit && ok; ++t) { + if (grp[t] != old) continue; + for (u = 0; u < n_hit; ++u) { + if (grp[u] != neu) continue; + if (!mb_qspan_overlap(&hit[t], &hit[u]) + || !mb_places_colocate(&pl[t], &pl[u], lift_tol)) { + ok = 0; break; + } + } + } + if (ok) { + /* merge i's group into j's group: relabel every member. */ + for (t = 0; t < n_hit; ++t) + if (grp[t] == old) grp[t] = neu; + } + } + } + } + + /* 2b. ALT-ALTERNATE fold-in (placement-based, via the .alt correspondence). + * + * The per-base lift in step 2 groups an ALT twin with its primary only where + * the lift is EXACT. It fails when an .alt-internal structural indel + * displaces the ALT twin's lifted placement beyond lift_tol, or drops the ALT + * footprint into an insertion HOLE (unliftable) -- leaving the ALT twin as a + * co-equal group representative that both steals the SAM-primary slot (read + * placed on the ALT contig) and dilutes the primary's MAPQ to 0. Recognize, + * via the .alt RECORD (this ALT contig is an alternate of a specific primary + * region), that an ALT hit which (a) shares read bases with a non-ALT hit + * -- same fragment, not a chimeric segment -- and (b) whose contig maps over + * that non-ALT hit's primary locus, is an ALTERNATE PLACEMENT of that locus. + * Fold its whole group into the non-ALT hit's group. + * + * Applied ONLY to an ALT hit whose group has NO non-ALT member (the exact + * lift already failed to co-locate it). Paralog-safe: only ALT-vs-non-ALT + * folds; two genuine primary loci are both non-ALT and never merge here, so a + * truly ambiguous multi-mapper keeps MAPQ 0. This does NOT discard ALT hits + * categorically -- an ALT hit with no overlapping primary (a read genuinely + * from an ALT-only region) stays an independent representative. */ + { + uint8_t *grp_has_pri = Kcalloc(km, uint8_t, n_hit); + if (grp_has_pri) { + for (i = 0; i < n_hit; ++i) + if (!hit[i].is_alt) grp_has_pri[grp[i]] = 1; + for (i = 0; i < n_hit; ++i) { + if (!hit[i].is_alt) continue; /* fold ALT hits only */ + if (grp_has_pri[grp[i]]) continue; /* already grouped with a primary */ + for (j = 0; j < n_hit; ++j) { + int old, neu, t; + if (hit[j].is_alt) continue; /* into a non-ALT hit */ + if (grp[j] == grp[i]) continue; + if (!mb_qspan_overlap(&hit[i], &hit[j])) continue; + if (!mb_alt_covers_primary(l2b, hit[i].tid, pl[j].pri_tid, pl[j].lifted_st)) + continue; + old = grp[i]; neu = grp[j]; + for (t = 0; t < n_hit; ++t) if (grp[t] == old) grp[t] = neu; + grp_has_pri[grp[i]] = 1; + break; + } + } + kfree(km, grp_has_pri); + } + } + + /* 3. representative per group -- determined AFTER the new grouping. Within a + * group every member is the SAME primary locus, so the primary-assembly + * (non-ALT) member is the representative the read is reported on: an ALT + * copy is an ALTERNATE of this locus, never a "better" locus, so a non-ALT + * member outranks an ALT member REGARDLESS of DP score. DP score (then + * larger hash) decides only between members of the same is_alt class. */ + for (i = 0; i < n_hit; ++i) rep[i] = grp[i]; /* seed with the label index */ + for (i = 0; i < n_hit; ++i) { + int g = grp[i], r = rep[g]; + int32_t si = mb_hit_dpscore(&hit[i]), sr = mb_hit_dpscore(&hit[r]); + int better; + if (hit[i].is_alt != hit[r].is_alt) better = (!hit[i].is_alt); /* prefer non-ALT */ + else if (si != sr) better = (si > sr); + else better = (hit[i].hash > hit[r].hash); + if (better) rep[g] = i; + } + + /* 4a. wire subordinates to their group's representative. */ + for (i = 0; i < n_hit; ++i) { + int r = rep[grp[i]]; + hit[i].parent = hit[r].id; /* reps get parent==id (r==i) automatically */ + } + + /* 4b. ZERO the suboptimal fields on every post-grouping representative + * (including ones promoted from subordinate) before recomputing -- the + * :778/:784 run left running maxima/accumulators that must be RESET. */ + for (i = 0; i < n_hit; ++i) { + if (rep[grp[i]] != i) continue; /* representatives only */ + hit[i].subsc = 0; + hit[i].n_sub = 0; + if (hit[i].p) hit[i].p->dp_max2 = 0; + } + + /* 4c. recompute over REPRESENTATIVES ONLY: each rep's suboptimal comes from + * other GROUPS' reps whose query span overlaps it (chimeric, i.e. + * non-overlapping, segments are not competitors). Intra-group members + * contribute nothing because we only ever compare distinct reps. */ + for (i = 0; i < n_hit; ++i) { + mb_hit_t *ri = &hit[i]; + if (rep[grp[i]] != i) continue; /* ri is a representative */ + for (j = 0; j < n_hit; ++j) { + mb_hit_t *rj; + int32_t sj; + if (rep[grp[j]] != j) continue; /* rj is a representative */ + if (grp[j] == grp[i]) continue; /* same group: not a competitor */ + rj = &hit[j]; + if (!mb_qspan_overlap(ri, rj)) continue; + sj = mb_hit_dpscore(rj); + if (sj > ri->subsc) ri->subsc = sj; + if (ri->p) { + if (sj > ri->p->dp_max2) ri->p->dp_max2 = sj; + /* n_sub counts near-tied competing groups: the competitor is + * within sub_diff of this rep's own DP score (mirrors the + * dp_max-dp_max <= sub_diff test in update_sub). */ + if (ri->p->dp_max - sj <= sub_diff) ++ri->n_sub; + } else { + if (ri->score - sj <= sub_diff) ++ri->n_sub; + } + } + } + + kfree(km, pl); kfree(km, grp); kfree(km, rep); +} + void mb_hit_sort(void *km, int *n_regs, mb_hit_t *r) { int32_t i, n_aux, n = *n_regs; @@ -640,13 +1148,18 @@ mb_hit_t *mb_map_sai(const mb_opt_t *opt, const mb_idx_t *idx, int64_t qlen, con hit = mb_gen_hit(b->km, hash, qlen, idx->l2b, n_hit, w, a); kfree(b->km, w); mb_set_parent(b->km, opt->mask_level, opt->mask_len, n_hit, hit, sub_diff, 0); - mb_select_sub(b->km, opt->pri_ratio, opt->min_len * 2, opt->best_n, &n_hit, hit); + mb_select_sub(b->km, opt->pri_ratio, opt->min_len * 2, opt->best_n, &n_hit, hit, idx->l2b, opt->lift_tol); // base alignment if (!(opt->flag & MB_F_NO_ALN)) { hit = mb_align_skeleton(b->km, opt, idx, qlen, seq, mt, &n_hit, hit, a); mb_set_parent(b->km, opt->mask_level, opt->mask_len, n_hit, hit, sub_diff, 0); - mb_select_sub(b->km, opt->pri_ratio, opt->min_len * 2, opt->best_n, &n_hit, hit); + mb_select_sub(b->km, opt->pri_ratio, opt->min_len * 2, opt->best_n, &n_hit, hit, idx->l2b, opt->lift_tol); + /* Liftover-group reconciliation: group ALT twins with their primary locus + * and rescope MAPQ to the second-best GROUP. Runs before mb_set_sam_pri so + * SAM primary/secondary reflect the new grouping; gated so non-ALT reads pay + * nothing. */ + if (mb_any_alt(n_hit, hit)) mb_reconcile_alt(b->km, idx->l2b, n_hit, hit, sub_diff, opt->lift_tol); mb_set_sam_pri(n_hit, hit, !!(opt->flag & MB_F_PRIMARY5)); } for (i = 0; i < n_hit; ++i) { diff --git a/map-main.c b/map-main.c index 829f5fc..04f0a6c 100644 --- a/map-main.c +++ b/map-main.c @@ -215,11 +215,14 @@ static void *worker_pipeline(void *shared, int step, void *in) } tot_len += t->l_seq; if (s->n_hit[i] > 0) { // the query has at least one hit - int32_t n_sec = 0; + int32_t n_sec = 0, alt_rec; for (j = 0; j < s->n_hit[i]; ++j) { const mb_hit_t *h = &s->hit[i][j]; - if (h->parent == h->id || n_sec < opt->out_n) { - if (h->parent != h->id) { + /* --alt-records emits every ALT hit, so it also bypasses the + * secondary score-ratio filter upstream added below. */ + alt_rec = (opt->flag & MB_F_ALT_RECORDS) && h->is_alt; + if (h->parent == h->id || n_sec < opt->out_n || alt_rec) { + if (h->parent != h->id && !alt_rec) { const mb_hit_t *p = &s->hit[i][h->parent]; if (p->p && h->p) { if (h->p->dp_max < (double)opt->out_s * p->p->dp_max) continue; @@ -228,7 +231,11 @@ static void *worker_pipeline(void *shared, int step, void *in) } } mb_format(km, &out, idx->l2b, t, seg_en - seg_st, &s->n_hit[seg_st], &s->hit[seg_st], j, opt, i - seg_st, mate_qlen); - n_sec += (h->parent != h->id); + /* An --alt-records hit is emitted as an extra and bypasses + * the out_n cap, so it must not consume the non-ALT + * secondary budget either -- otherwise it can crowd out + * genuine secondaries the user asked for with -h/out_n. */ + n_sec += (h->parent != h->id) && !alt_rec; } } } else if (!(opt->flag & MB_F_NO_UNMAP)) { @@ -346,12 +353,19 @@ static ko_longopt_t long_options[] = { { "mmap", ko_optional_argument, 313 }, { "xa-ratio", ko_required_argument, 314 }, { "outs", ko_required_argument, 315 }, + { "alt", ko_required_argument, 316 }, + { "no-alt", ko_no_argument, 319 }, + { "alt-records", ko_no_argument, 317 }, + { "alt-lift-tol", ko_required_argument, 318 }, { "dbg-aln-seq", ko_no_argument, 601 }, { "dbg-anchor", ko_no_argument, 602 }, { "dbg-seed", ko_no_argument, 603 }, { "dbg-qname", ko_no_argument, 604 }, { "dbg-aln-pe", ko_no_argument, 605 }, { "dbg-an-pos", ko_no_argument, 606 }, // anchor position + { "dbg-no-alt-proj", ko_no_argument, 607 }, // ablate ALT->primary projection (testing) + { "dbg-alt-proj", ko_no_argument, 608 }, // trace projected primary anchors (testing) + { "dbg-no-alt-survive", ko_no_argument, 609 }, // ablate the mb_select_sub survival guard (testing) { "version", ko_no_argument, 901 }, { "help", ko_no_argument, 902 }, { 0, 0, 0 } @@ -397,6 +411,10 @@ static int usage_map(FILE *fp, const mb_opt_t *opt) fprintf(fp, " --outn=NUM output up to {NUM,-N} secondary alignments [0]\n"); fprintf(fp, " --outs=FLOAT output a secondary hit if score at least FLOAT*bestScore [%g]\n", opt->out_s); fprintf(fp, " --xa=NUM if <=NUM hits with score >%g%% of the best hit, output them to XA [%d]\n", opt->out_s*100.0, opt->xa_max); + fprintf(fp, " --no-alt ignore .alt; align as if no ALT file exists\n"); + fprintf(fp, " --alt FILE path to the .alt file (default: auto-detected .alt)\n"); + fprintf(fp, " --alt-records emit ALT-contig alignments with full SEQ\n"); + fprintf(fp, " --alt-lift-tol INT bp tolerance for grouping ALT twins by lifted locus [%d]\n", MB_LIFT_TOL); fprintf(fp, " -y copy FASTA/Q comments to output\n"); fprintf(fp, " -Y use soft clipping for supplementary alignments\n"); fprintf(fp, " -H STR if STR starts with @, insert to header; or insert lines in file STR []\n"); @@ -421,6 +439,25 @@ static inline void yes_or_no(mb_opt_t *opt, uint64_t flag, int long_idx, const c } } +/* `mem`'s own long options: exactly the knobs it implements. Sharing map's + * table instead would make `mem` PARSE every option map has while handling only + * these, so one it has no arm for -- --meth, --mmap, --outn, --eqx, --hic -- + * would be accepted and silently dropped, and `mem --meth` would run without + * methylation and say nothing. Codes are shared with long_options[] so the two + * parsers cannot drift on what a number means. */ +static ko_longopt_t mem_long_options[] = { + { "alt", ko_required_argument, 316 }, + { "no-alt", ko_no_argument, 319 }, + { "alt-records", ko_no_argument, 317 }, + { "alt-lift-tol", ko_required_argument, 318 }, + { "dbg-no-alt-proj", ko_no_argument, 607 }, + { "dbg-alt-proj", ko_no_argument, 608 }, + { "dbg-no-alt-survive", ko_no_argument, 609 }, + { "version", ko_no_argument, 901 }, + { "help", ko_no_argument, 902 }, + { 0, 0, 0 } +}; + static void set_ins_size(mb_opt_t *opt, const char *arg) { char *q; @@ -432,6 +469,32 @@ static void set_ins_size(mb_opt_t *opt, const char *arg) opt->flag |= MB_F_PE_PREDEF; } +/* Resolve the .alt for a loaded index, shared by `map` and `mem` so their ALT + * surface stays consistent. Returns 0 on success, 1 on a hard error (an + * explicitly-named --alt that will not load). --no-alt loads nothing, leaving + * every ALT code path inert. Emits a warning when --alt is overridden by + * --no-alt, and when an explicitly-named --alt yields zero usable records. */ +static int mb_load_alt(mb_idx_t *idx, const char *prefix, const char *alt_fn, int no_alt) +{ + if (no_alt) { + if (alt_fn) + fprintf(stderr, "[W::%s] --alt '%s' is ignored because --no-alt was also given\n", __func__, alt_fn); + return 0; + } + if (alt_fn) { /* explicitly named: failure to load is a user error */ + int nr = mb_idx_set_alt(idx, alt_fn); + if (nr < 0) { + fprintf(stderr, "[ERROR] failed to load the ALT file '%s'\n", alt_fn); + return 1; + } + if (nr == 0) + fprintf(stderr, "[W::%s] --alt '%s' loaded 0 usable ALT records; running as if without a .alt\n", __func__, alt_fn); + } else { /* auto: an absent adjacent .alt is the normal case, not an error */ + mb_idx_set_alt_auto(idx, prefix); + } + return 0; +} + int main_map(int argc, char *argv[]) { const char *opt_str = "x:o:k:c:m:p:A:B:U:b:O:E:t:K:N:PyYR:H:aul:w:W:g:5s:fI:"; @@ -439,6 +502,8 @@ int main_map(int argc, char *argv[]) mb_idx_t *idx; mb_opt_t mo; char *fn_out = 0, *rg_line = 0, *s; + const char *alt_fn = 0; + int32_t no_alt = 0; ketopt_t o = KETOPT_INIT; kstring_t hdr_ins = {0,0,0}, hdr = {0,0,0}; @@ -512,6 +577,15 @@ int main_map(int argc, char *argv[]) if (o.arg != 0 && strcmp(o.arg, "lite") == 0) mmap_preload = 0; } else if (c == 314 || c == 315) { // --outs or --xa-ratio mo.out_s = atof(o.arg); + } else if (c == 316) { // --alt + alt_fn = o.arg; + } else if (c == 317) { // --alt-records + mo.flag |= MB_F_ALT_RECORDS; + } else if (c == 318) { // --alt-lift-tol + mo.lift_tol = atoi(o.arg); + if (mo.lift_tol < 0) mo.lift_tol = 0; + } else if (c == 319) { // --no-alt + no_alt = 1; } else if (c == 601) { // --dbg-aln-seq kom_dbg_flag |= MB_DBG_ALN_SEQ; } else if (c == 602) { // --dbg-anchor @@ -524,6 +598,12 @@ int main_map(int argc, char *argv[]) kom_dbg_flag |= MB_DBG_ALN_PE; } else if (c == 606) { // --dbg-an-pos kom_dbg_flag |= MB_DBG_AN_POS; + } else if (c == 607) { // --dbg-no-alt-proj + kom_dbg_flag |= MB_DBG_NO_ALT_PROJ; + } else if (c == 608) { // --dbg-alt-proj + kom_dbg_flag |= MB_DBG_ALT_PROJ; + } else if (c == 609) { // --dbg-no-alt-survive + kom_dbg_flag |= MB_DBG_NO_ALT_SURVIVE; } else if (c == 'K') { mo.mb_size = mo.max_mb_size = kom_parse_num(o.arg, &s); if (*s == ',') mo.max_mb_size = kom_parse_num(s + 1, &s); @@ -558,6 +638,12 @@ int main_map(int argc, char *argv[]) is_meth = !!(mo.flag & MB_F_METH); idx = use_mmap? mb_idx_load_mmap(argv[o.ind], is_meth, mmap_preload) : mb_idx_load(argv[o.ind], is_meth); kom_assert(idx, "failed to load the index."); + /* Resolve the .alt here rather than inside the loaders, so that --mmap and the + * normal path cannot disagree about whether this index is ALT-aware. --no-alt + * loads nothing at all, which leaves every ALT code path inert -- is_alt is set + * only by l2b_set_alt(), never stored in the index. */ + if (mb_load_alt(idx, argv[o.ind], alt_fn, no_alt) != 0) + return 1; if (kom_verbose >= 3) fprintf(stderr, "[M::%s::%.3f*%.2f] index loaded\n", __func__, kom_realtime(), kom_percent_cpu()); @@ -605,7 +691,11 @@ static int usage_mem(FILE *fp, const mb_opt_t *opt) fprintf(fp, " -R STR SAM read group line in a format like '@RG\\tID:foo\\tSM:bar' []\n"); fprintf(fp, " -H STR if STR starts with @, insert to header; or insert lines in file STR []\n"); fprintf(fp, " -o FILE output file name [stdout]\n"); - fprintf(fp, " *j treat ALT contigs as part of the primary assembly\n"); + fprintf(fp, " --no-alt ignore .alt; align as if no ALT file exists\n"); + fprintf(fp, " --alt FILE path to the .alt file (default: auto-detected .alt)\n"); + fprintf(fp, " --alt-records emit ALT-contig alignments with full SEQ\n"); + fprintf(fp, " --alt-lift-tol INT bp tolerance for grouping ALT twins by lifted locus [%d]\n", MB_LIFT_TOL); + fprintf(fp, " *j (bwa) treat ALT as primary; minibwa uses the liftover-group scheme above instead\n"); fprintf(fp, " -5 take the alignment with the smallest query position as primary\n"); fprintf(fp, " *q don't modify mapQ of supplementary alignments\n"); fprintf(fp, " *K NUM batch size []\n"); @@ -624,6 +714,8 @@ static int usage_mem(FILE *fp, const mb_opt_t *opt) fprintf(fp, "Notes:\n"); fprintf(fp, " - \"minibwa mem\" aims to match the \"bwa mem\" command-line interface\n"); fprintf(fp, " - '*' options are ignored as they are missing or incompatible with minibwa\n"); + fprintf(fp, " - an adjacent .alt is auto-loaded and applied (liftover-group ALT\n"); + fprintf(fp, " awareness); pass --no-alt to disable it\n"); fprintf(fp, " - minibwa and bwa-mem may output different alignments\n"); return fp == stdout? 0 : 1; } @@ -634,12 +726,19 @@ int main_mem(int argc, char *argv[]) ketopt_t o = KETOPT_INIT; mb_opt_t mo; char *fn_out = 0, *rg_line = 0; + const char *alt_fn = 0; + int32_t no_alt = 0; kstring_t hdr_ins = {0,0,0}, hdr = {0,0,0}; mb_idx_t *idx; mb_opt_init(&mo); mo.flag |= MB_F_WRITE_MD; // bwa-mem always writes MD - while ((c = ketopt(&o, argc, argv, 1, "t:k:w:d:r:y:c:D:W:m:SPA:B:O:E:L:U:x:pR:H:o:j5qK:v:T:h:z:aCVYMuI:", 0)) >= 0) { + /* mem_long_options carries minibwa's ALT knobs (--alt/--no-alt/--alt-records/ + * --alt-lift-tol and the --dbg-* diagnostics), so `mem` exposes the same ALT + * surface as `map` rather than silently auto-loading .alt with no way to + * override or disable it -- but only those, so an option `mem` does not + * implement is rejected below instead of parsed and ignored. */ + while ((c = ketopt(&o, argc, argv, 1, "t:k:w:d:r:y:c:D:W:m:SPA:B:O:E:L:U:x:pR:H:o:j5qK:v:T:h:z:aCVYMuI:", mem_long_options)) >= 0) { // algorithm if (c == 't') mo.n_thread = atoi(o.arg); else if (c == 'k') mo.min_len = atoi(o.arg); @@ -667,11 +766,31 @@ int main_mem(int argc, char *argv[]) else if (c == 'C') mo.flag |= MB_F_COPY_COMMENT; else if (c == 'Y') mo.flag |= MB_F_SUPP_SOFT; else if (c == 'I') set_ins_size(&mo, o.arg); + // ALT liftover-group knobs (shared with `map` via long_options) + else if (c == 316) alt_fn = o.arg; // --alt + else if (c == 317) mo.flag |= MB_F_ALT_RECORDS; // --alt-records + else if (c == 318) { mo.lift_tol = atoi(o.arg); if (mo.lift_tol < 0) mo.lift_tol = 0; } // --alt-lift-tol + else if (c == 319) no_alt = 1; // --no-alt + else if (c == 607) kom_dbg_flag |= MB_DBG_NO_ALT_PROJ; // --dbg-no-alt-proj + else if (c == 608) kom_dbg_flag |= MB_DBG_ALT_PROJ; // --dbg-alt-proj + else if (c == 609) kom_dbg_flag |= MB_DBG_NO_ALT_SURVIVE; // --dbg-no-alt-survive + else if (c == 901) { puts(MB_VERSION); return 0; } + else if (c == 902) return usage_mem(stdout, &mo); + /* Reject what map rejects, rather than swallowing it. */ + else if (c == ':') { + fprintf(stderr, "[ERROR] missing option argument\n"); + return 1; + } else if (c == '?') { + fprintf(stderr, "[ERROR] unknown option in \"%s\"\n", argv[o.erri]); + return 1; + } } if (argc - o.ind < 2) return usage_mem(stderr, &mo); idx = mb_idx_load(argv[o.ind], !!(mo.flag & MB_F_METH)); kom_assert(idx, "failed to load the index."); + if (mb_load_alt(idx, argv[o.ind], alt_fn, no_alt) != 0) + return 1; if (kom_verbose >= 3) fprintf(stderr, "[M::%s::%.3f*%.2f] index loaded\n", __func__, kom_realtime(), kom_percent_cpu()); diff --git a/mbpriv.h b/mbpriv.h index da5f5f3..3106e56 100644 --- a/mbpriv.h +++ b/mbpriv.h @@ -7,12 +7,68 @@ #include "kommon.h" #include "bseq.h" +/* Maximum number of per-block lifted sub-placements retained in mb_place_t. + * A read footprint (~150 bp pre/post DP) overlaps at most this many .alt lift + * blocks in practice; if MORE blocks overlap (a pathologically fragmented .alt + * CIGAR over the footprint) the FIRST MB_MAX_SUBPL are kept and the rest spill + * (documented in mb_hit_place). Spilling only ever DROPS candidate co-location + * intervals -- it can never invent a spurious match -- so it is conservatively + * safe for paralog isolation. */ +#define MB_MAX_SUBPL 8 + +/* One lifted sub-placement: where a single overlapping .alt lift block maps the + * footprint onto primary coordinates. + * st representative (min) primary coordinate of this block's lifted span + * pri_tid primary contig this block lands on + * rev strand of this block's footprint on primary (.alt block strand XOR h->rev) */ +typedef struct { + int64_t st; + int64_t pri_tid; + uint8_t rev; +} mb_subpl_t; + +/* The lifted PLACEMENT of one hit: the primary footprint it occupies over its + * liftable portion. Computed by mb_hit_place(). + * + * MULTI-INTERVAL placement (SV-breakpoint-aware grouping): instead of collapsing + * every overlapping .alt lift block into a single [lifted_st, lifted_en], the + * placement records ONE sub-placement per overlapping block in subpl[]. A + * breakpoint-spanning ALT hit whose footprint straddles an SV-scale indel then + * exposes BOTH the near-breakpoint primary position AND the far one as separate + * sub-placements, so it can still co-locate with its primary twin via the + * matching sub-interval (mb_places_colocate) instead of being dragged thousands + * of bp away by a min/max collapse. Co-location requires a SHARED sub-interval, + * so two distinct primary loci that happen to land in one inflated span are NOT + * merged (paralog safety). + * + * pri_tid REPRESENTATIVE primary contig (== subpl[0].pri_tid; == h->tid for + * non-ALT hits). Kept for back-compat readers. + * lifted_st REPRESENTATIVE primary coordinate (== subpl[0].st). Back-compat + * grouping key for any reader not yet on the multi-interval API. + * lifted_en max primary coordinate over all sub-placements (cosmetic: nothing + * reads it for grouping decisions). + * rev REPRESENTATIVE strand (== subpl[0].rev; .alt block strand XOR h->rev). + * liftable 1 iff n_subpl >= 1 (at least one block of the footprint lifts); + * 0 iff the ENTIRE footprint falls in holes (ALT-specific -> own group). + * n_subpl number of valid sub-placements (1..MB_MAX_SUBPL; 0 when !liftable). + * subpl the per-block lifted sub-placements (first n_subpl entries valid). */ +typedef struct { + int64_t pri_tid; + int64_t lifted_st, lifted_en; + uint8_t rev, liftable; + int n_subpl; + mb_subpl_t subpl[MB_MAX_SUBPL]; +} mb_place_t; + #define MB_DBG_ALN_SEQ (0x1LL) #define MB_DBG_ANCHOR (0x2LL) #define MB_DBG_SEED (0x4LL) #define MB_DBG_QNAME (0x8LL) #define MB_DBG_ALN_PE (0x10LL) #define MB_DBG_AN_POS (0x20LL) +#define MB_DBG_NO_ALT_PROJ (0x40LL) // --dbg-no-alt-proj: ablate ALT-seed->primary projection (testing) +#define MB_DBG_ALT_PROJ (0x80LL) // --dbg-alt-proj: trace projected primary anchors (testing) +#define MB_DBG_NO_ALT_SURVIVE (0x100LL) // --dbg-no-alt-survive: ablate the mb_select_sub survival guard (testing) #define MB_SEED_LONG_JOIN 0x1 #define MB_SEED_IGNORE 0x2 @@ -71,7 +127,18 @@ void mb_set_parent(void *km, float mask_level, int mask_len, int n, mb_hit_t *r, int32_t mb_set_sam_pri(int32_t n, mb_hit_t *r, int32_t is_primary5); void mb_hit_sort(void *km, int *n_regs, mb_hit_t *r); void mb_sync_hits(void *km, int n_regs, mb_hit_t *regs); -void mb_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, mb_hit_t *r); +void mb_select_sub(void *km, float pri_ratio, int min_diff, int best_n, int *n_, mb_hit_t *r, + const l2b_t *l2b, int lift_tol); +int mb_any_alt(int n_hit, const mb_hit_t *hit); +/* Compute the lifted PLACEMENT (primary footprint) of one hit. For a non-ALT + * hit this is the identity placement on its own contig. For an ALT hit it lifts + * the aligned footprint (the chain interval [ts,te) pre-DP, or the exact CIGAR + * span post-DP) through the .alt span-lift to primary coordinates. h->p may be + * NULL (pre-DP, coarse) or non-NULL (post-DP, exact); liftable==0 if the whole + * footprint is in holes. See mb_place_t. */ +mb_place_t mb_hit_place(const l2b_t *l2b, const mb_hit_t *h); +int mb_places_colocate(const mb_place_t *a, const mb_place_t *b, int lift_tol); +void mb_reconcile_alt(void *km, const l2b_t *l2b, int n_hit, mb_hit_t *hit, int sub_diff, int lift_tol); void mb_filter_hits(const mb_opt_t *opt, int qlen, int *n_regs, mb_hit_t *regs); int mb_squeeze_a(void *km, int n_regs, mb_hit_t *regs, mb_anchor_t *a); void mb_split_hit(mb_hit_t *r, mb_hit_t *r2, int n, int qlen, mb_anchor_t *a, const l2b_t *l2b); diff --git a/minibwa.h b/minibwa.h index b29d5b7..c0ad99f 100644 --- a/minibwa.h +++ b/minibwa.h @@ -23,6 +23,7 @@ #define MB_F_PRIMARY5 (0x8000LL) // for Hi-C #define MB_F_NO_PAIRING (0x10000LL) // don't pair reads #define MB_F_METH (0x20000LL) // methylation mode +#define MB_F_ALT_RECORDS (0x40000LL) // emit ALT-contig hits with full SEQ #define MB_CIGAR_MATCH 0 #define MB_CIGAR_INS 1 @@ -84,6 +85,7 @@ typedef struct { int64_t max_mb_size; int64_t max_sw_mat; int64_t cap_kalloc; + int32_t lift_tol; // ALT liftover-group co-location tolerance in bp (default MB_LIFT_TOL) } mb_opt_t; struct mb_idx_s; @@ -113,10 +115,21 @@ typedef struct { int32_t mlen, blen; int32_t mapq; uint32_t hash; - uint32_t rev:1, proper_pair:1, sam_pri:1, flt:1, inv:1, split:2, split_inv:1, rescued:1, frac_high:8, seed_ratio:8, dummy:7; + uint32_t rev:1, proper_pair:1, sam_pri:1, flt:1, inv:1, split:2, split_inv:1, rescued:1, frac_high:8, seed_ratio:8, is_alt:1, dummy:6; mb_extra_t *p; } mb_hit_t; +/* Co-location tolerance (bp) on the lifted footprint start used to group hits + * by primary locus. Two hits are the same locus iff they share pri_tid, rev, + * and their lifted_st differ by no more than the tolerance. This is the + * COMPILE-TIME DEFAULT; the effective value is opt->lift_tol (runtime-tunable + * via --alt-lift-tol), threaded into the survival guard, reconciliation, and the + * PE demotion guard. Raise it for .alt files whose ALT-to-primary CIGARs carry + * larger indels (lifting a read start across an indel can drift lifted_st by up + * to the indel size); the query-span-overlap requirement guards against merging + * genuine paralogs even at a looser tolerance. */ +#define MB_LIFT_TOL 10 + struct mb_tbuf_s; typedef struct mb_tbuf_s mb_tbuf_t; @@ -127,6 +140,20 @@ extern "C" { mb_idx_t *mb_idx_load(const char *prefix, int32_t is_meth); mb_idx_t *mb_idx_load_mmap(const char *prefix, int32_t is_meth, int preload); void mb_idx_destroy(mb_idx_t *idx); +/* Load an ALT file from an explicit path. Returns as l2b_set_alt() does: + * the number of ALT records parsed, or -1 if the file cannot be read. */ +int mb_idx_set_alt(mb_idx_t *idx, const char *fn); + +/** + * Load .alt if it exists beside the index. + * + * Neither mb_idx_load() nor mb_idx_load_mmap() does this for you: the caller + * decides, so that the two loaders cannot disagree about whether a given index + * is ALT-aware, and so that a caller wanting stock (non-ALT) behaviour simply + * does not call this. Returns the number of .alt records loaded, or -1 if the + * file is absent or unreadable -- the common case, and not an error. + */ +int mb_idx_set_alt_auto(mb_idx_t *idx, const char *prefix); const char *mb_idx_ctg_name(const mb_idx_t *idx, int32_t tid); int64_t mb_idx_ctg_len(const mb_idx_t *idx, int32_t tid); diff --git a/options.c b/options.c index 73eb3d6..bbb61c4 100644 --- a/options.c +++ b/options.c @@ -19,6 +19,7 @@ static void mb_opt_reset(mb_opt_t *opt) // hit processing options opt->mask_level = 0.5f; opt->mask_len = 0x7fffffff; + opt->lift_tol = MB_LIFT_TOL; // alignment options opt->a = 2, opt->b = 8; opt->q = 12, opt->q2 = 23; diff --git a/pe.c b/pe.c index e5bab8e..485c281 100644 --- a/pe.c +++ b/pe.c @@ -16,6 +16,30 @@ static inline int mb_insert_dir(const mb_hit_t *h0, const mb_hit_t *h1, int64_t return ((int32_t)h0->rev << 1 | (int32_t)h1->rev) ^ (p0 < p1? 0 : 3); } +/* ALT liftover-group (cross-group guard): true iff two hits are DISTINCT lifted groups -- + * both liftable, but sharing NO lifted sub-interval within lift_tol (different + * primary contigs, opposite lifted strands, or all sub-placement pairs > lift_tol + * apart -- i.e. real paralogs). Used to STOP the chimeric demotion from + * collapsing two distinct primary loci that merely overlap in query span. A + * breakpoint-spanning ALT hit that shares a sub-interval with its primary twin is + * NOT distinct (same locus). An UNLIFTABLE hit is NOT a protected distinct group: + * it returns 0 here so it falls through to normal demotion (preserving baseline + * behavior and not shielding spurious hits). */ +/* As mb_distinct_lifted_group() but with b's placement precomputed, so a caller + * that tests many `a` against a fixed `b` (the pair-chosen endpoint) does not + * recompute mb_hit_place(b) on every iteration. */ +static inline int mb_distinct_lifted_group_pb(const l2b_t *l2b, const mb_hit_t *a, const mb_place_t *pb, int lift_tol) +{ + mb_place_t pa = mb_hit_place(l2b, a); + if (!pa.liftable || !pb->liftable) return 0; /* unliftable -> NOT a protected distinct group */ + /* Distinct (protected) iff they do NOT co-locate: no shared lifted + * sub-interval (different pri_tid, opposite strand, or all sub-placements + * > lift_tol apart). A breakpoint-spanning ALT hit that shares a + * sub-interval with its primary twin is NOT distinct (same locus -> may be + * demoted), while two genuine paralog loci stay protected. */ + return !mb_places_colocate(&pa, pb, lift_tol); +} + static inline double mb_pair_score(const mb_hit_t *h0, const mb_hit_t *h1, const mb_pestat_t pes[4], int32_t match_sc) { const double MB_SQRT1_2 = 0.707106781186547524401; @@ -147,9 +171,20 @@ static void mb_pair_hits(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_ pa = Kcalloc(km, mb128_t, n_hit[0] + n_hit[1]); for (r = n_pa = 0; r < 2; ++r) { for (i = 0; i < n_hit[r]; ++i) { - mb128_t *p = &pa[n_pa++]; + mb128_t *p; mb_hit_t *h = &hit[r][i]; h->proper_pair = 0; + /* twin-exclusion (ALT liftover-group): a non-representative member of a lifted + * group (an ALT twin folded under its primary by mb_reconcile_alt: + * is_alt && parent != id) must NOT enter pair enumeration. Otherwise + * the twin pair (R1@alt, R2@alt) forms a near-equal SECOND-best pair + * that depresses paux.sub_sc => mapq_pe collapses to ~0 even though the + * twin is the SAME locus on primary vs ALT. Excluding non-reps makes + * paux.sub_sc measure the gap to the second-best GROUP-pair. parent is + * set by the SE-path reconcile (survives on no-rescue reads) and, on + * rescued reads, by post-rescue regroup before the second mb_pair_hits call. */ + if (h->is_alt && h->parent != h->id) continue; + p = &pa[n_pa++]; p->x = l2b->ctg[h->tid].off + (h->rev? h->te : h->ts); p->y = (uint64_t)i << 2 | (uint64_t)h->rev << 1 | r; } @@ -359,6 +394,12 @@ static const mb_hit_t *mb_matesw_core(void *km, const mb_opt_t *opt, const l2b_t mb_matesw_align(km, opt, len, seq[is_rev], te2 - ts2, &ref[ts2 - ts], &ht, min_sc, mt, ez); if (ht.p) { // a good hit found ht.tid = h0->tid; + /* Stamp is_alt from the contig: mb_matesw_align memset(h,0) cleared + * it, and a mate rescued onto an ALT contig must be visible to the + * liftover-group reconciliation (post-rescue regroup), else grouping is blind to + * exactly the reads mate rescue exists to recover. Mirrors the stamp + * mb_gen_hit applies to seeded hits. */ + ht.is_alt = l2b->ctg[ht.tid].is_alt; ht.ts += ts2, ht.te += ts2; ht.rev = is_rev; if (is_rev) { @@ -367,6 +408,27 @@ static const mb_hit_t *mb_matesw_core(void *km, const mb_opt_t *opt, const l2b_t ht.qe = len - qt; } //fprintf(stderr, "X\t%s\tsc0=%d\tnew_sc=%d\tnew_ts=%ld\tn_cigar=%d\tcigar[0]=%d\n", l2b->ctg[ht.tid].name, h0->p->dp_max, ht.p->dp_max, (long)ht.ts, ht.p->n_cigar, ht.p->cigar[0]>>4); + /* Skip a rescued hit that exactly duplicates one the seeder already + * found (same contig/strand/coords). Suppressing the ALT-twin pair + * (twin-exclusion) makes the pair look unpaired, so mate rescue re-runs and + * re-discovers an already-present ALT hit -- without this guard that + * yields two identical secondary SAM records. An identical-coordinate + * duplicate carries no new information, so dropping it would be safe + * on the non-ALT path too -- but "safe" is not "byte-identical", and + * the manifest promises byte-identity without a .alt. The duplicate + * only arises from twin-exclusion, which is an ALT mechanism, so gate + * on ALT being involved and leave the baseline path untouched. If + * this is worth having generally it belongs upstream on its own. */ + if (mb_any_alt(h1->n, h1->a) || ht.is_alt) { + int32_t e; int dup = 0; + for (e = 0; e < h1->n; ++e) { + const mb_hit_t *he = &h1->a[e]; + if (he->tid == ht.tid && he->rev == ht.rev && + he->ts == ht.ts && he->te == ht.te && + he->qs == ht.qs && he->qe == ht.qe) { dup = 1; break; } + } + if (dup) { if (ht.p) free(ht.p); kfree(km, ref); continue; } + } if (h1->n == h1->m) kom_grow(mb_hit_t, h1->a, h1->n, h1->m); h1->a[h1->n++] = ht; ret = &h1->a[h1->n - 1]; @@ -378,7 +440,7 @@ static const mb_hit_t *mb_matesw_core(void *km, const mb_opt_t *opt, const l2b_t } static int32_t mb_matesw(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_t n_hit[2], mb_hit_t *hit[2], const mb_pestat_t pes[4], const mb_pairaux_t *paux0, - int32_t qlen[2], char *const qseq[2], int32_t is_meth) + int32_t qlen[2], char *const qseq[2], int32_t is_meth, int32_t rescue_tie[2]) { int32_t i, r, n_add, n_res, max[2], max2[2], skip[2], min_sc[2]; mb_hit_v ha[2]; @@ -452,6 +514,11 @@ static int32_t mb_matesw(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_ kfree(km, ez.cigar); kfree(km, qs[0][0]); kfree(km, a); + /* Surface the rescue-tie signal: skip[r] is set iff, while using read r's copies + * as rescue anchors, two equal-scoring rescued pairs were found -- i.e. the mate + * fits EQUALLY beside >=2 of r's copies. mb_pair uses this to recognise a + * coin-flip pair (see the MAPQ damp below). */ + rescue_tie[0] = skip[0], rescue_tie[1] = skip[1]; n_add = (ha[0].n - n_hit[0]) + (ha[1].n - n_hit[1]); for (r = 0; r < 2; ++r) n_hit[r] = ha[r].n, hit[r] = ha[r].a; @@ -464,6 +531,7 @@ void mb_pair(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_t n_hit[2], int32_t r, i, dp_max_se[2], score_se, dp_max_se2[2], score_se2, do_matesw, is_meth = !!(opt->flag & MB_F_METH), reset_sam_pri = 1; mb_pairaux_t paux; int32_t seed_ratio[2], min_seed_ratio; + int32_t rescue_tie[2] = {0, 0}; /* per-read: set when mate rescue found >=2 equal pairs from this read's anchors */ if (n_hit[0] == 0 && n_hit[1] == 0) return; seed_ratio[0] = n_hit[0] > 0? hit[0][0].seed_ratio : 255; @@ -473,7 +541,7 @@ void mb_pair(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_t n_hit[2], do_matesw = paux.n_pp > 0 && paux.score == paux.sub_sc? 0 : 1; // skip mate rescue if we see two equally best pairs if (do_matesw && opt->max_rescue > 0) { int32_t sub_diff = opt->a + opt->b > opt->q + opt->e? opt->a + opt->b : opt->q + opt->e; - if (mb_matesw(km, opt, l2b, n_hit, hit, pes, &paux, qlen, qseq, is_meth) > 0) { + if (mb_matesw(km, opt, l2b, n_hit, hit, pes, &paux, qlen, qseq, is_meth, rescue_tie) > 0) { for (r = 0; r < 2; ++r) { for (i = 0; i < n_hit[r]; ++i) { mb_hit_t *h = &hit[r][i]; @@ -482,6 +550,15 @@ void mb_pair(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_t n_hit[2], } mb_hit_sort(km, &n_hit[r], hit[r]); mb_set_parent(km, opt->mask_level, opt->mask_len, n_hit[r], hit[r], sub_diff, 0); + /* post-rescue regroup (ALT liftover-group): mate rescue zeroed the suboptimal + * fields (:481) and mb_set_parent re-derived `parent` with no group + * knowledge, wiping the SE-path reconciliation for THIS read. Re- + * impose liftover groups before mb_set_mapq consumes the group-scoped + * subsc/dp_max2/n_sub, and before the second mb_pair_hits so twin-exclusion + * sees the correct parent. Only runs on rescued reads (this block is + * rescue-conditional); the no-rescue majority keeps the SE reconcile. */ + if (mb_any_alt(n_hit[r], hit[r])) + mb_reconcile_alt(km, l2b, n_hit[r], hit[r], sub_diff, opt->lift_tol); mb_set_mapq(km, qlen[r], n_hit[r], hit[r], opt->min_chain_score, opt->a, mb_is_sr_mode(opt, qlen[r]), opt->max_sr_len); } mb_pair_hits(km, opt, l2b, n_hit, hit, pes, &paux); // pair again if new hits rescued @@ -505,12 +582,20 @@ void mb_pair(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_t n_hit[2], for (r = 0; r < 2; ++r) // clear 0x2 as these will be re-evaluated and set in the following for (i = 0; i < n_hit[r]; ++i) hit[r][i].proper_pair = 0; + mb_hit_t *h[2]; + h[0] = &hit[0][paux.i[0]]; + h[1] = &hit[1][paux.i[1]]; + /* ALT liftover-group: twin-exclusion excludes non-representative ALT group members + * (is_alt && parent != id) from pair enumeration, so a chosen pair endpoint + * that is ALT must be its group's representative (parent == id). Guards the + * re-rooting at :563-569 from promoting an ALT subordinate to primary and + * emitting the wrong sam_pri. (Non-ALT subordinates may still legitimately be + * pair endpoints -- the re-rooting block handles them as before.) */ + assert(!h[0]->is_alt || h[0]->id == h[0]->parent); + assert(!h[1]->is_alt || h[1]->id == h[1]->parent); if (paux.score >= score_se - opt->pen_unpair * opt->a) { // choose the paired hits int32_t mapq_pe, score2 = paux.sub_sc, diff; double identity; - mb_hit_t *h[2]; - h[0] = &hit[0][paux.i[0]]; - h[1] = &hit[1][paux.i[1]]; assert(n_hit[0] > 0 && n_hit[1] > 0); identity = (double)(h[0]->mlen + h[1]->mlen) / (h[0]->blen + h[1]->blen); if ((h[0]->id != h[0]->parent || h[1]->id != h[1]->parent) && score2 < score_se - opt->pen_unpair * opt->a) @@ -522,6 +607,38 @@ void mb_pair(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_t n_hit[2], if (mapq_pe < 0) mapq_pe = 0; mapq_pe = (int)(mapq_pe * (1. - .5 * (h[0]->frac_high / 255. + h[1]->frac_high / 255.)) + .499); if (min_seed_ratio < 50) mapq_pe *= (double)min_seed_ratio * min_seed_ratio / 2500.0; + /* Coin-flip damp. A proper pair is a coin-flip -- arbitrary among several + * equivalent placements -- when ALL of: + * (a) one endpoint was placed ONLY by mate rescue (the other anchored it), + * (b) mate rescue found >=2 equal-scoring pairs from the anchor's copies + * (rescue_tie: the mate fits >=2 of them equally), and + * (c) the anchor end really has >=2 co-optimal representatives (its own DP + * second-best ties its best, within one match score). + * The surviving single pair is then an arbitrary pick of one tied copy, so the + * high pair-based mapq overstates confidence -- damp it to ~0 (bwa-mem stays + * cautious here too). Conditions (a)+(b) keep this off genuine recoveries, + * where the mate fits exactly one copy. Reuses already-computed fields. + * + * Gated on an ALT hit actually being present. This block used to carry an + * ARGUMENT that it was a no-op without a .alt rather than a check, and the + * argument was wrong: two genuine paralogs both surviving as representatives + * with dp_max within opt->a, plus a rescue tie, satisfies (a)-(c) with no ALT + * anywhere. Measured on 100k HG002 WGS pairs against an index with no .alt, + * the SAM differed from stock. The chrM fixture is too small to reach here, + * so the distribution's byte-identity gate passed throughout. */ + { int rr; + for (rr = 0; rr < 2; ++rr) { + int32_t j, n_coopt = 0; + if (!mb_any_alt(n_hit[rr], hit[rr])) continue; + if (!h[!rr]->rescued || h[rr]->rescued || !rescue_tie[rr]) continue; + for (j = 0; j < n_hit[rr]; ++j) { + const mb_hit_t *hj = &hit[rr][j]; + if (hj->id == hj->parent && hj->p && hj->p->dp_max >= dp_max_se[rr] - opt->a) + ++n_coopt; + } + if (n_coopt >= 2) { mapq_pe = 0; break; } + } + } if (mapq_pe > 60) mapq_pe = 60; if (mapq_pe <= 0 && paux.score > score2) mapq_pe = 1; for (r = 0; r < 2; ++r) { @@ -536,11 +653,30 @@ void mb_pair(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_t n_hit[2], hit[r][i].parent = h[r]->id; p->mapq = 0; } + /* cross-group guard gate (ALT liftover-group): only consult lifted placement when + * this mate actually has ALT hits, so a reference with no .alt loaded is + * byte-identical (mb_any_alt is false => no behavior change). */ + int r_any_alt = mb_any_alt(n_hit[r], hit[r]); + /* q == h[r] is invariant across the loop, so lift its placement once + * rather than recomputing it (a CIGAR walk + lift searches) per hit. */ + mb_place_t pq; + if (r_any_alt) pq = mb_hit_place(l2b, h[r]); for (i = 0; i < n_hit[r]; ++i) { // handle other chimeric hits const mb_hit_t *q = h[r]; mb_hit_t *p = &hit[r][i]; if (q != p && p->id == p->parent) { // p is a chimeric hit that is not h[r] int32_t j, ol = p->qe <= q->qs || p->qs >= q->qe? 0 : (p->qe < q->qe? p->qe : q->qe) - (p->qs > q->qs? p->qs : q->qs); + /* cross-group guard (ALT liftover-group): the demotion below collapses `p` + * under the pair-hit `q` by QUERY overlap alone. When `p` and `q` + * are co-located copies (same lifted group) the demotion is + * correct -- the ALT twin SHOULD become secondary. But when they + * are DIFFERENT lifted groups (distinct primary loci / paralogs) + * the demotion would wrongly merge two genuine competing loci and + * inflate MAPQ, undoing the Task-4 paralog guard for PE. Skip the + * demotion across distinct groups. Gated on r_any_alt so the + * no-.alt baseline is unchanged. */ + if (r_any_alt && mb_distinct_lifted_group_pb(l2b, p, &pq, opt->lift_tol)) + continue; if (ol > opt->mask_level * (p->qe - p->qs)) { // if p overlaps with h[r] a lot, make it a secondary hit for (j = 0; j < n_hit[r]; ++j) // FIXME: quadratic time complexity, but almost never an issue on real data if (hit[r][j].parent == p->id) @@ -559,6 +695,8 @@ void mb_pair(void *km, const mb_opt_t *opt, const l2b_t *l2b, int32_t n_hit[2], } if (opt->flag & MB_F_PRIMARY5) { int32_t pri[2]; + /* -1: this block asks what the 5-prime rule alone would pick, so the + * pair-chosen endpoint must not override it or the test below is vacuous. */ pri[0] = mb_set_sam_pri(n_hit[0], hit[0], 1); pri[1] = mb_set_sam_pri(n_hit[1], hit[1], 1); if (&hit[0][pri[0]] != h[0] || &hit[1][pri[1]] != h[1]) // if sam_pri is changed, clear flag 0x2 diff --git a/seed.c b/seed.c index 2ce4b47..2503b03 100644 --- a/seed.c +++ b/seed.c @@ -1,6 +1,7 @@ #include #include #include +#include #include "mbpriv.h" #include "kalloc.h" #include "ksort.h" @@ -166,6 +167,194 @@ static void mb_anchor_dedup(mb_anchor_v *v) // NB: assuming sorted by tpos v->n = j; } +/**************************************** + * ALT-seed -> primary anchor projection * + ****************************************/ + +/* Maximum number of projected primary anchors injected per (pri_tid, folded + * strand) bucket. Segduplicated loci can have hundreds of ALT/paralog copies; + * without a cap, near-duplicate paralog projections would blow up mb_lchain_dp's + * O(n*max_iter) inner loop on exactly the repeat-heavy reads. A small cap is + * sufficient: we only need ONE surviving primary anchor at the lifted locus to + * seed a primary candidate (the rest are near-duplicates the following + * mb_anchor_dedup collapses), so capping here never under-recovers. */ +#define MB_PROJ_CAP_PER_LOCUS 4 + +/* Maximum number of DISTINCT projected primary loci tracked per read. Sized so + * it effectively never fires on real data: a read projecting to more distinct + * loci than this is in a massive repeat family and will be MAPQ 0 regardless, so + * stopping early is correctness-neutral. The bound exists only to keep the + * per-locus bookkeeping table on the stack. Both caps are silent in production + * but counted and emitted under --dbg-alt-proj so truncation is observable. */ +#define MB_PROJ_MAX_LOCI 256 + +/* Project ALT-contig anchors onto the primary assembly so a segduplicated + * primary locus gets a candidate even when max_occ subsampling drops the + * primary's OWN seed (DRAGEN's mechanism: use ALT-contig seed matches to obtain + * the corresponding primary alignment). + * + * Called from mb_anchor() AFTER process_batch() has filled v->a[] and BEFORE the + * radix_sort + tpos-rebase + mb_anchor_dedup at the tail of mb_anchor(). All + * coordinates here are in the CONCATENATED frame (mirroring seed.c:207); the + * existing sort/rebase/dedup then handle ordering and exact-duplicate removal of + * the injected anchors for free. + * + * Each injected anchor is written as a NATIVE-equivalent primary anchor (no + * provenance bit any chaining code reads): sid = pri_tid<<1 | folded_rev, with + * tpos/qpos = last base in the strand-FOLDED concatenated frame and len = seed + * length. It then chains normally under comput_sc() as a genuine primary anchor + * -- this is NOT a chainer-merge of an ALT anchor into an ALT chain. */ +static void mb_anchor_project_alt(void *km, const l2b_t *l2b, int32_t qlen, mb_anchor_v *v) +{ + int64_t i, n0 = v->n; + /* --dbg-no-alt-proj ablates projection so the segdup regression tests can + * compare WITH vs WITHOUT projection from a single binary; production leaves + * it off (projection is unconditional, gated only on per-anchor is_alt). */ + if (kom_dbg_flag & MB_DBG_NO_ALT_PROJ) return; + /* --dbg-alt-proj traces each projected primary anchor (see the trace below). */ + int proj_trace = (kom_dbg_flag & MB_DBG_ALT_PROJ) != 0; + /* Per-locus cap bookkeeping: a tiny rolling table keyed by the projected + * sid (pri_tid<<1|folded_rev) AND the projected forward last base + * (fold_last), so the cap is per distinct projected LOCUS rather than per + * (contig,strand) -- two paralog seeds that lift to different positions on + * the same contig+strand must NOT share a cap slot. Loci are few per read + * in practice, so a linear scan is fine; this also dedups + * projected-vs-projected at the same locus. */ + int32_t cap_sid[MB_PROJ_MAX_LOCI]; + int64_t cap_pos[MB_PROJ_MAX_LOCI]; + int32_t cap_cnt[MB_PROJ_MAX_LOCI]; + int32_t n_cap = 0; + int64_t n_drop_locuscap = 0, n_drop_tablefull = 0; /* observability (trace only) */ + + if (n0 == 0) return; + + for (i = 0; i < n0; ++i) { + const mb_anchor_t *q = &v->a[i]; + int64_t alt_tid = q->sid >> 1; + int32_t alt_rev = q->sid & 1; + const l2b_ctg_t *alt_ctg; + int64_t alt_cst, alt_clast; /* ALT contig-local forward span [cst, clast] inclusive */ + int64_t pt_lo, pt_hi; /* lifted primary tids of the two endpoints */ + uint64_t pp_lo, pp_hi; /* lifted primary positions (forward, contig-local) */ + uint8_t rv_lo, rv_hi; + int64_t pri_tid, pri_st, pri_en; /* primary forward span [st, en] inclusive */ + uint8_t blk_rev, folded_rev; + const l2b_ctg_t *pri_ctg; + int64_t qf_s; /* query forward start of the seed */ + int64_t new_qpos, fold_last, new_tpos; + int32_t new_sid, j, c; + mb_anchor_t *p; + + /* Gate: only ALT contigs with lift blocks project. */ + if (alt_tid < 0 || alt_tid >= (int64_t)l2b->n_ctg) continue; + alt_ctg = &l2b->ctg[alt_tid]; + if (!alt_ctg->is_alt || alt_ctg->n_lift == 0) continue; + + /* Recover the ALT contig-local FORWARD span from the concatenated tpos + * (inverse of process_batch's q->tpos = off*2 + len*rev + cst + len-1). + * The position component is the FOLDED last base for the seed strand; + * recover that fold first, then unfold per strand to a forward span. + * For a reverse seed the folded last base is the forward FIRST base of + * the span, so cst = len-1-fold_last; for forward it is the last base. */ + int64_t alt_fold_last = q->tpos - alt_ctg->off * 2 - alt_ctg->len * alt_rev; + if (alt_fold_last < 0 || alt_fold_last >= (int64_t)alt_ctg->len) continue; + alt_cst = alt_rev ? (int64_t)alt_ctg->len - 1 - alt_fold_last : alt_fold_last - (q->len - 1); + alt_clast = alt_cst + q->len - 1; /* inclusive last forward ALT base */ + if (alt_cst < 0 || alt_clast >= (int64_t)alt_ctg->len) continue; + + /* Lift both inclusive endpoints; a reverse block maps low ALT -> high + * primary, so take min/max over the two lifted outputs. Require both to + * lift, to the same primary tid and the same block strand (a hole or a + * cross-block seed yields no clean primary anchor -> skip). */ + if (!l2b_lift(l2b, alt_tid, (uint64_t)alt_cst, &pt_lo, &pp_lo, &rv_lo)) continue; + if (!l2b_lift(l2b, alt_tid, (uint64_t)alt_clast, &pt_hi, &pp_hi, &rv_hi)) continue; + if (pt_lo != pt_hi || rv_lo != rv_hi) continue; + pri_tid = pt_lo; + blk_rev = rv_lo; + pri_st = (int64_t)pp_lo < (int64_t)pp_hi ? (int64_t)pp_lo : (int64_t)pp_hi; + pri_en = (int64_t)pp_lo > (int64_t)pp_hi ? (int64_t)pp_lo : (int64_t)pp_hi; + if (pri_tid < 0 || pri_tid >= (int64_t)l2b->n_ctg) continue; + pri_ctg = &l2b->ctg[pri_tid]; + + /* Reject length-changing lifts: a seed spanning an indel or two adjacent + * lift blocks maps to a primary span whose length differs from the seed + * length. Injecting it as a contiguous len-bp anchor would corrupt the + * chain coordinates, so skip it. */ + if (pri_en - pri_st + 1 != q->len) continue; + + /* Strand fold: the projected primary strand is the .alt block strand + * XOR the seed's strand on the ALT contig (mirrors mb_hit_place's + * blk_rev ^ h->rev). */ + folded_rev = (uint8_t)(blk_rev ^ alt_rev); + + /* Recover the query FORWARD start of the seed, then re-fold qpos for the + * projected strand (qpos = last base in the folded query frame). */ + qf_s = alt_rev ? (int64_t)qlen - 1 - q->qpos : q->qpos - (q->len - 1); + if (qf_s < 0 || qf_s + q->len > qlen) continue; + new_qpos = folded_rev ? (int64_t)qlen - 1 - qf_s : qf_s + q->len - 1; + + /* Forward contig-local LAST base of the primary span. process_batch + * stores tpos's position component in the FORWARD contig frame for BOTH + * strands (strand lives in sid&1 plus the len*rev half-frame shift; the + * consumer mb_hit_set_coor does ts = tpos+1-len with no reverse-unfold). + * The min/max over the two lifted endpoints already handled the + * reverse-block low-alt -> high-primary inversion, so use pri_en + * unconditionally -- do NOT re-fold for folded_rev. */ + fold_last = pri_en; + if (fold_last < 0 || fold_last >= (int64_t)pri_ctg->len) continue; + + /* Concatenated-frame tpos, mirroring seed.c:207. */ + new_tpos = pri_ctg->off * 2 + pri_ctg->len * folded_rev + fold_last; + new_sid = (int32_t)(pri_tid << 1 | folded_rev); + + /* --dbg-alt-proj emits one line per projected anchor giving the primary + * contig, 1-based POS, and projected strand -- the load-bearing + * coordinates produced by the reverse-span recovery and forward-frame + * fold. Observable even when the resulting alignment is masked at SAM + * level by identical-scoring paralog collapse (reverse RC repeats), so a + * fixture can assert the projected locus directly. Diagnostics only. */ + if (proj_trace) { + int64_t pri_pos1 = fold_last - q->len + 2; /* 1-based POS = (ts 0-based)+1 = (fold_last+1-len)+1 */ + fprintf(stderr, "MB_PROJ\t%s\t%lld\t%c\tlen=%d\n", + pri_ctg->name, (long long)pri_pos1, folded_rev ? '-' : '+', q->len); + } + + /* Per-locus cap + projected-vs-projected dedup at the same projected + * locus (sid + forward last base). (Projected-vs-native exact + * duplicates are removed by the mb_anchor_dedup that runs right after + * this; here we only bound volume and squash redundant paralog + * projections to the same coordinate.) */ + c = -1; + for (j = 0; j < n_cap; ++j) + if (cap_sid[j] == new_sid && cap_pos[j] == fold_last) { c = j; break; } + if (c < 0) { + if (n_cap < (int32_t)(sizeof(cap_sid) / sizeof(cap_sid[0]))) { + c = n_cap++; + cap_sid[c] = new_sid; + cap_pos[c] = fold_last; + cap_cnt[c] = 0; + } else { ++n_drop_tablefull; continue; } /* table full: stop projecting new loci */ + } + if (cap_cnt[c] >= MB_PROJ_CAP_PER_LOCUS) { ++n_drop_locuscap; continue; } + ++cap_cnt[c]; + + /* Inject the native-equivalent primary anchor (flag/flt = 0 via memset). */ + Kgrow(km, mb_anchor_t, v->a, v->n, v->m); + p = &v->a[v->n++]; + memset(p, 0, sizeof(*p)); + p->sid = new_sid; + p->len = q->len; + p->qpos = (int32_t)new_qpos; + p->tpos = new_tpos; + } + /* Make cap-driven truncation observable (default-off diagnostic seam): in + * production both caps are correctness-neutral, but a non-zero drop count on + * a repeat-heavy read is worth seeing when investigating recovery. */ + if (proj_trace && (n_drop_locuscap || n_drop_tablefull)) + fprintf(stderr, "MB_PROJ_CAP\tdropped_locuscap=%lld\tdropped_tablefull=%lld\tn_loci=%d\n", + (long long)n_drop_locuscap, (long long)n_drop_tablefull, n_cap); +} + /************************ * Get contig positions * ************************/ @@ -328,6 +517,12 @@ double mb_anchor(void *km, const mb_idx_t *idx, mb_sai_v *u, int32_t min_len, in seed_ratio = (double)t1 / t0; } + /* ALT-seed -> primary anchor projection (segdup recovery). Inject in the + * concatenated frame so the radix_sort + tpos-rebase + mb_anchor_dedup below + * order and dedup the injected anchors for free. No-op unless a seed landed + * on an ALT contig (so non-ALT references are byte-identical). */ + mb_anchor_project_alt(km, idx->l2b, qlen, v); + radix_sort_mb_anchor(v->a, v->a + v->n); for (i = 0; i < v->n; ++i) { // adjust mb_anchor_t::tpos mb_anchor_t *q = &v->a[i]; diff --git a/test/altlg/fixlib.sh b/test/altlg/fixlib.sh new file mode 100644 index 0000000..2742334 --- /dev/null +++ b/test/altlg/fixlib.sh @@ -0,0 +1,16 @@ +# Shared fixture-builder helpers for the ALT liftover-group suite (sourced by +# mkfixture-*.sh). Previously copy-pasted (gen into 7 scripts, rc into 4). + +# Deterministic random DNA of a given length: gen . +gen() { python3 -c " +import random +random.seed($1) +print(''.join(random.choice('ACGT') for _ in range($2))) +"; } + +# Reverse-complement. With an argument it complements that string; with none it +# acts as a stdin filter. Handles upper and lower case. +rc() { + if [ "$#" -ge 1 ]; then printf '%s' "$1" | rev | tr 'ACGTacgt' 'TGCAtgca' + else rev | tr 'ACGTacgt' 'TGCAtgca'; fi +} diff --git a/test/altlg/lib-baseline.sh b/test/altlg/lib-baseline.sh new file mode 100644 index 0000000..d30f870 --- /dev/null +++ b/test/altlg/lib-baseline.sh @@ -0,0 +1,61 @@ +# Shared no-.alt baseline check for the ALT liftover-group suite. +# +# What the ALT feature must guarantee on a reference with no .alt file is that its +# output is *identical to stock minibwa's* -- the hooks are all gated, so an unpatched +# and a patched binary should agree byte for byte. +# +# Running this build twice and comparing cannot show that: it only shows the aligner is +# deterministic, which is true whether or not the hooks leak. Seven scripts here used to +# do exactly that and reported it as "inert without .alt". The comparison needs a second, +# unpatched binary, so it is opt-in: set MB_STOCK to a stock minibwa and the check runs; +# leave it unset and the check reports itself skipped rather than passing vacuously. +# +# Callers must define: MINIBWA, MDIR, TMPD, and the ok()/fail() helpers. +# +# chrm_baseline "(BASELINE) chrM PE" pe --outn=5 +# chrm_baseline "(BASELINE) chrM SE" se + +chrm_baseline() { + _label="$1"; _mode="$2"; shift 2 + _fa="$MDIR/test/chrM-human.fa.gz" + _r1="$MDIR/test/chrM-read_1.fa.gz" + _r2="$MDIR/test/chrM-read_2.fa.gz" + + if [ ! -f "$_fa" ] || [ ! -f "$_r1" ]; then + echo " skip: $_label -- chrM fixture not found" + return 0 + fi + if [ "$_mode" = pe ] && [ ! -f "$_r2" ]; then + echo " skip: $_label -- chrM mate file not found" + return 0 + fi + if [ -z "${MB_STOCK:-}" ]; then + echo " skip: $_label -- set MB_STOCK= to compare against unpatched output" + return 0 + fi + if [ ! -x "$MB_STOCK" ]; then + fail "$_label -- MB_STOCK='$MB_STOCK' is not executable" + fi + + # Copy and index inside TMPD rather than beside the shared fixture: keeps the check + # hermetic, avoids racing another script indexing the same chrM, and leaves no + # untracked .mbw/.l2b in test/. + cp "$_fa" "$TMPD/base-ref.fa.gz" + if [ "$_mode" = pe ]; then set -- "$@" "$TMPD/base-ref.fa.gz" "$_r1" "$_r2" + else set -- "$@" "$TMPD/base-ref.fa.gz" "$_r1"; fi + + "$MINIBWA" index "$TMPD/base-ref.fa.gz" 2>/dev/null + "$MINIBWA" map "$@" 2>/dev/null | grep -v '^@PG' > "$TMPD/base-patched.sam" + "$MB_STOCK" map "$@" 2>/dev/null | grep -v '^@PG' > "$TMPD/base-stock.sam" + + [ -s "$TMPD/base-patched.sam" ] || fail "$_label -- empty output from this build" + [ -s "$TMPD/base-stock.sam" ] || fail "$_label -- empty output from MB_STOCK" + + # @PG carries the command line and version, so it differs by construction and is + # stripped above; every other line must match. + if cmp -s "$TMPD/base-patched.sam" "$TMPD/base-stock.sam"; then + ok "$_label byte-identical to stock (hooks provably inert without .alt)" + else + fail "$_label differs from stock -- an ALT hook is firing on a no-.alt reference" + fi +} diff --git a/test/altlg/lib.sh b/test/altlg/lib.sh new file mode 100644 index 0000000..fbec691 --- /dev/null +++ b/test/altlg/lib.sh @@ -0,0 +1,34 @@ +# Shared helpers for the ALT liftover-group test suite (sourced by test-*.sh). +# +# Every helper here was previously copy-pasted across many test-*.sh scripts, +# sometimes with small and occasionally INCOMPATIBLE per-file variations. The +# signatures below are supersets that are backward-compatible with every prior +# call site: trailing arguments are optional and default to "match anything", so +# both e.g. `primary_mapq f` (any primary) and `primary_mapq f q` (that read's +# primary) work, and both `flag_of f q c` and `flag_of f q c pos` work. + +ok() { echo " ok: $1"; } +fail() { echo "FAIL: $1"; exit 1; } + +# SAM body: drop @ header lines. +sam_body() { mawk '$1!~/^@/{print}' "$1"; } + +# Is bit set in flag ? (mawk has no and(); test by arithmetic.) +has_bit() { mawk -v f="$1" -v b="$2" 'BEGIN{ f=int(f); b=int(b); printf "%d\n", (int(f/b)%2==1)?1:0 }'; } + +# FLAG / MAPQ of the record on contig , optionally at 1-based POS . +flag_of() { mawk -v q="$2" -v c="$3" -v p="${4:-}" '$1==q && $3==c && (p=="" || $4==p){print $2; exit}' "$1"; } +mapq_of() { mawk -v q="$2" -v c="$3" -v p="${4:-}" '$1==q && $3==c && (p=="" || $4==p){print $5; exit}' "$1"; } + +# AS:i: tag of the record on contig . +as_of() { mawk -v q="$2" -v c="$3" '$1==q && $3==c { + for(i=12;i<=NF;i++){ if(substr($i,1,5)=="AS:i:"){ print substr($i,6); exit } } }' "$1"; } + +# FLAG / MAPQ of the PRIMARY record (neither secondary 0x100 nor supplementary +# 0x800). With , restrict to that read; without it, the first primary. +primary_flag() { mawk -v q="${2:-}" '$1!~/^@/ && (q=="" || $1==q){f=int($2); if(int(f/256)%2==0 && int(f/2048)%2==0){print $2; exit}}' "$1"; } +primary_mapq() { mawk -v q="${2:-}" '$1!~/^@/ && (q=="" || $1==q){f=int($2); if(int(f/256)%2==0 && int(f/2048)%2==0){print $5; exit}}' "$1"; } + +# Is there a record on chrP at 1-based POS ? (chrP is the primary contig in +# every fixture that uses this.) +has_pos() { mawk -v p="$2" 'BEGIN{f=0} $1!~/^@/ && $3=="chrP" && $4==p{f=1} END{print f}' "$1"; } diff --git a/test/altlg/mkfixture-altalt.sh b/test/altlg/mkfixture-altalt.sh new file mode 100644 index 0000000..77bd83e --- /dev/null +++ b/test/altlg/mkfixture-altalt.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# Fixture for the ALT-ALTERNATE fold-in (mb_reconcile_alt step 2b). +# +# Root cause it locks: an ALT contig's .alt CIGAR carries STRUCTURAL INDELS +# (the duplicated / divergent segments the alt-aligner could not co-linearly +# align). Those indels OFFSET an ALT twin's lifted placement from the read's +# true primary copy by up to the indel size -- beyond MB_LIFT_TOL -- OR drop it +# into an insertion HOLE (unliftable). The precise per-base lift then fails to +# co-locate the ALT twin with its primary hit, so the twin survives as a +# co-equal group representative: it both steals the SAM-primary slot (read +# placed on the ALT contig) AND dilutes the primary's MAPQ to 0. +# +# The fix recognizes -- via the .alt CORRESPONDENCE (this ALT contig is, by +# construction, an alternate of THIS primary region) -- that an ALT hit sharing +# read bases with a primary hit inside the ALT's own primary region is an +# ALTERNATE PLACEMENT of that locus, and folds it into the primary's group. +# Placement-based, not "discard all ALT": only ALT-vs-non-ALT folds, so two +# genuine primary loci never merge (paralog safety; see mkfixture-paralog.sh). +# +# Geometry (chrP = 2000bp unique sequence, LCG-free python random seed 23): +# +# R-disp = chrP[600,750) -- true origin chrP:601 (1-based). +# chrP_altF = R-disp ++ chrP[1000,1200) +# .alt: chrP_altF -> chrP POS 451, CIGAR 150M 400D 200M +# 150M : ALT[0,150) -> chrP[450,600) (R-disp's ALT hit lifts to chrP:451, +# DISPLACED 150bp from its true 601) +# 400D : chrP[600,1000) skipped +# 200M : ALT[150,350) -> chrP[1000,1200) +# contig primary span chrP[450,1200) COVERS the true origin chrP:601 +# (which sits in the 400D gap) -> fold-in applies. +# +# R-hole = chrP[1500,1650) -- true origin chrP:1501 (1-based). +# chrP_altG = chrP[1300,1450) ++ R-hole ++ chrP[1600,1700) +# .alt: chrP_altG -> chrP POS 1301, CIGAR 150M 150I 150D 100M +# 150M : ALT[0,150) -> chrP[1300,1450) +# 150I : ALT[150,300) = R-hole -> INSERTION HOLE (unliftable) +# 150D : chrP[1450,1600) skipped +# 100M : ALT[300,400) -> chrP[1600,1700) +# contig primary span chrP[1300,1700) COVERS the true origin chrP:1501 +# (in the 150D gap) -> fold-in applies even though the ALT hit is unliftable. +# +# Each read therefore has exactly two equal-scoring hits: the true chrP hit and +# an ALT hit that the per-base lift CANNOT co-locate with it (displaced / hole). +# +# Usage: mkfixture-altalt.sh +set -eu +d="${1:?usage: mkfixture-altalt.sh }" +mkdir -p "$d" + +S=$(python3 -c " +import random +random.seed(23) +print(''.join(random.choice('ACGT') for _ in range(2000))) +") + +sub() { printf '%s' "$S" | cut -c"$1"-"$2"; } + +RDISP=$(sub 601 750) # chrP[600,750) +ALTF=$(sub 601 750)$(sub 1001 1200) # R-disp ++ chrP[1000,1200) +RHOLE=$(sub 1501 1650) # chrP[1500,1650) +ALTG=$(sub 1301 1450)$(sub 1501 1650)$(sub 1601 1700) # ++ R-hole ++ + +# --- reference --- +printf '>chrP\n%s\n' "$S" > "$d/ref.fa" +printf '>chrP_altF\n%s\n' "$ALTF">> "$d/ref.fa" +printf '>chrP_altG\n%s\n' "$ALTG">> "$d/ref.fa" + +# --- .alt: structural-indel CIGARs that displace / hole the ALT twins --- +printf 'chrP_altF\t0\tchrP\t451\t60\t150M400D200M\t*\t0\t0\t*\t*\n' > "$d/ref.fa.alt" +printf 'chrP_altG\t0\tchrP\t1301\t60\t150M150I150D100M\t*\t0\t0\t*\t*\n'>> "$d/ref.fa.alt" + +# --- reads --- +QUAL150=$(printf '%150s' '' | tr ' ' 'I') +printf '@r-disp\n%s\n+\n%s\n' "$RDISP" "$QUAL150" > "$d/reads.fq" +printf '@r-hole\n%s\n+\n%s\n' "$RHOLE" "$QUAL150" >> "$d/reads.fq" + +echo "fixture(altalt): chrP=2000 altF=disp(150M400D200M) altG=hole(150M150I150D100M)" >&2 diff --git a/test/altlg/mkfixture-chimera.sh b/test/altlg/mkfixture-chimera.sh new file mode 100644 index 0000000..9804a2a --- /dev/null +++ b/test/altlg/mkfixture-chimera.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# Fixture for Task 4 (reconciliation pass), the CHIMERA case — locks Fix 1. +# +# Fix 1 tightened mb_reconcile_alt's group-merge predicate: two hits are the +# SAME liftover group only if they are the same read alignment on primary vs +# ALT — same pri_tid, same strand, |Δlifted_st| <= MB_LIFT_TOL, AND their query +# spans OVERLAP. Before Fix 1 the merge ignored query overlap, so a chimeric +# read whose two DIFFERENT query segments happen to lift near the SAME primary +# locus would be (wrongly) collapsed into one group: the second segment's hit +# would be demoted to a same-group subordinate (secondary, 0x100), removed as a +# competitor, and the first segment's MAPQ inflated. +# +# This fixture constructs exactly that adversarial geometry: +# +# chrP (primary) is one 900bp unique sequence (LCG PRNG, seed 11). +# +# SEG1 = chrP[120,240) (120bp) -- the read's FIRST query segment. +# SEG2 = chrP[560,680) (120bp) -- a DIFFERENT, distant region; the read's +# SECOND query segment. SEG1 and SEG2 share no homology (distinct +# windows of a unique sequence), so they are genuine chimeric segments +# of one read, not competitors for the same locus. +# +# The read = SEG1 immediately followed by SEG2 (240bp). Query span of the +# SEG1 alignment is ~[0,120); of the SEG2 alignment ~[120,240) — DISJOINT. +# +# ALT contig chrP_altE = a copy of chrP[560,760) (200bp) that BEGINS with SEG2 +# (so SEG2's hit starts at ALT offset ~0). Its .alt line aligns chrP_altE to +# chrP at POS=121 (1-based) — i.e. it LIES that this ALT contig corresponds to +# the chrP region [120,320), the SAME primary window SEG1 lifts into. Because +# SEG2 sits at the very start of the ALT contig, SEG2's best hit (on chrP_altE) +# lifts to lifted_st ~= 120 — within MB_LIFT_TOL of SEG1's own chrP hit +# (lifted_st ~120) — but at a DISJOINT query span. +# +# Result: +# SEG1 hit: chrP, query ~[0,120), lifts to lifted_st ~120 +# SEG2 hit: chrP_altE, query ~[120,240), lifts to lifted_st ~120 (same!) +# +# They are co-located (same pri_tid/strand, |Δlst| <= tol) but query-DISJOINT. +# BEFORE Fix 1: merged -> SEG2's chrP_altE hit is secondary (0x100), competitor +# suppressed, SEG1 MAPQ inflated. +# AFTER Fix 1: NOT merged (query spans don't overlap) -> the chrP_altE hit +# stays an independent representative (NOT 0x100 secondary); the +# genuine second locus is preserved. +# +# The observable, fixture-stable signal asserted by the test is the SAM flag of +# the chrP_altE record: secondary (0x100 set) == wrongly merged (RED, pre-Fix1); +# NOT secondary == correctly kept separate (GREEN, post-Fix1). +set -eu +d="$1"; mkdir -p "$d" + +# 900bp unique sequence (LCG PRNG, seed 11). +S=$(python3 -c " +import random +random.seed(11) +print(''.join(random.choice('ACGT') for _ in range(900))) +") + +# SEG1 = chrP[120,240) 1-based 121..240; SEG2 = chrP[560,680) 1-based 561..680. +SEG1=$(printf '%s' "$S" | cut -c121-240) +SEG2=$(printf '%s' "$S" | cut -c561-680) + +# ALT contig chrP_altE = chrP[560,760) 1-based 561..760 (200bp), contains SEG2. +ALTE=$(printf '%s' "$S" | cut -c561-760) +LE=${#ALTE} + +# --- reference --- +printf '>chrP\n%s\n' "$S" > "$d/ref.fa" +printf '>chrP_altE\n%s\n' "$ALTE">> "$d/ref.fa" + +# --- .alt: chrP_altE deliberately aligned to chrP at POS=121 (the SEG1 window) +# so SEG2's ALT hit lifts onto SEG1's primary locus (lifted_st ~120). --- +printf 'chrP_altE\t0\tchrP\t121\t60\t%dM\t*\t0\t0\t*\t*\n' "$LE" > "$d/ref.fa.alt" + +# --- read: SEG1, a short non-aligning SPACER, then SEG2. The spacer (a window +# of chrP far from both SEG1 and SEG2, reverse-complemented so it does not +# extend either flank) guarantees the SEG1 and SEG2 alignments have a CLEAN +# gap between their query spans — strictly DISJOINT, not merely abutting — +# so mb_qspan_overlap is unambiguously false. --- +SPACER=$(printf '%s' "$S" | cut -c801-830 | rev | tr 'ACGTacgt' 'TGCAtgca') +RCHIM="${SEG1}${SPACER}${SEG2}" +LR=${#RCHIM} +QUAL=$(printf "%${LR}s" '' | tr ' ' 'I') +printf '@r-chimera\n%s\n+\n%s\n' "$RCHIM" "$QUAL" > "$d/reads.fq" + +echo "fixture(chimera): chrP=900 SEG1=chrP[120,240) SEG2=chrP[600,720) altE=chrP[560,760)->POS121" >&2 diff --git a/test/altlg/mkfixture-hookc.sh b/test/altlg/mkfixture-hookc.sh new file mode 100755 index 0000000..42e63e8 --- /dev/null +++ b/test/altlg/mkfixture-hookc.sh @@ -0,0 +1,108 @@ +#!/bin/sh +# Fixture for Hook C UNLIFTABLE-demotion test (test-hookc.sh). +# +# Goal: a PE read whose mate (R2) has an is_alt hit on chrHC_altU whose ENTIRE +# footprint falls in a LIFT HOLE (no .alt block covers those ALT bases). Hook C +# in pe.c skips chimeric demotion only for DISTINCT LIFTABLE groups; an unliftable +# hit returns 0 from mb_distinct_lifted_group and falls through to NORMAL demotion. +# This fixture proves that path: the unliftable ALT hit must become secondary +# (parent != its id) and must NOT remain a shielded independent competitor. +# +# Geometry: +# chrHC 4000 bp primary (random seed 41) +# chrHC_altU 200 bp ALT contig: +# bases [0,100) = chrHC[500,600) -- covered by .alt block -> liftable +# bases [100,200) = chrHC[700,800) -- NO .alt block covers this -> HOLE +# .alt record: chrHC_altU -> chrHC POS 501, CIGAR 100M (covers only alt[0,100)) +# +# R1 = chrHC[300,420) fwd (unique anchor, 120bp) +# R2 = revcomp(chrHC[700,800)) = revcomp(chrHC_altU[100,200)) +# -> maps to chrHC@700 (primary, liftable) AND to chrHC_altU@100 (ALT, HOLE -> unliftable) +# Insert (5'-5'): R1@300 fwd, R2@700 rev -> ~520 bp, within filler distribution. +# +# Filler pairs: 30 unique FR pairs from chrHC windows outside [400,900), +# jittered insert 400..520 bp, establish pestat. +# +# Unliftability proof: the chrHC_altU footprint [100,200) overlaps NO .alt lift +# block (the single block covers alt[0,100), i.e. alt_en=100 <= fp_st=100), so +# mb_hit_place returns liftable=0. ex-place-check will confirm this. +set -eu +d="$1"; mkdir -p "$d" + +python3 - "$d" <<'PY' +import sys, random +d = sys.argv[1] +random.seed(41) +N = 4000 +S = ''.join(random.choice('ACGT') for _ in range(N)) + +def revcomp(s): + c = {'A':'T','C':'G','G':'C','T':'A'} + return ''.join(c[b] for b in reversed(s)) + +# ALT contig: first 100bp = chrHC[500,600); second 100bp = chrHC[700,800) +ALTL = S[500:600] # liftable part (covered by .alt block) +ALTH = S[700:800] # hole part (NOT covered by .alt block) +ALTU = ALTL + ALTH # 200bp total + +# .alt block covers only the first 100bp of chrHC_altU (alt[0,100) -> chrHC POS 501) +# so alt[100,200) is a HOLE -> any hit with footprint inside [100,200) -> liftable=0 + +with open(f"{d}/ref.fa", "w") as f: + f.write(f">chrHC\n{S}\n") + f.write(f">chrHC_altU\n{ALTU}\n") +with open(f"{d}/ref.fa.alt", "w") as f: + # QNAME FLAG RNAME POS MAPQ CIGAR RNEXT PNEXT TLEN SEQ QUAL + # covers chrHC_altU[0,100) -> chrHC POS 501 (1-based) + f.write("chrHC_altU\t0\tchrHC\t501\t60\t100M\t*\t0\t0\t*\t*\n") + +RL1, RL2 = 120, 100 # read lengths + +def fq(name, r1, r2): + q1 = 'I' * len(r1); q2 = 'I' * len(r2) + return (f"@{name}/1\n{r1}\n+\n{q1}\n", f"@{name}/2\n{r2}\n+\n{q2}\n") + +reads1, reads2 = [], [] + +# Filler pairs: unique FR windows from chrHC, clear of the ALT-relevant region [400,900) +# jittered insert so pestat has real width (inserts ~400..520) +jit = random.Random(77) +pos = 50 +n_filler = 30 +for i in range(n_filler): + while 400 - RL1 < pos < 900 + RL2: + pos += 400 + if pos + 520 + RL2 > N - 100: + pos = 50 + gap = jit.randint(280, 400) # 5'-5' insert = gap + RL1 = 400..520 + r1 = S[pos:pos+RL1] + r2 = revcomp(S[pos+gap:pos+gap+RL2]) + a, b = fq(f"f{i:02d}", r1, r2) + reads1.append(a); reads2.append(b) + pos += 100 + +# Test pair r-hookc: +# R1 = chrHC[300,420) fwd (unique anchor) +# R2 = revcomp(chrHC[700,800)) -- this is also chrHC_altU[100,200) revcomp'd +# -> primary hit: chrHC@700 (liftable=1) +# -> ALT hit: chrHC_altU@100 (footprint [100,200) in HOLE -> liftable=0) +# 5'-5' insert: (700+100) - 300 = 500bp, within the filler distribution (400..520) +r1 = S[300:420] +r2 = revcomp(S[700:800]) +a, b = fq("r-hookc", r1, r2) +reads1.append(a); reads2.append(b) + +with open(f"{d}/reads_1.fq", "w") as f: f.writelines(reads1) +with open(f"{d}/reads_2.fq", "w") as f: f.writelines(reads2) + +# Also write R2 alone for the ex-place-check probe (single-read FASTQ) +with open(f"{d}/r2_probe.fq", "w") as f: + f.write(f"@r-hookc-r2\n{r2}\n+\n{'I'*len(r2)}\n") + +sys.stderr.write( + f"fixture(hookc): chrHC={N} altU=200bp(liftable[0,100)=chrHC[500,600)," + f"hole[100,200)=chrHC[700,800)) " + f"R1=chrHC[300,420) R2=revcomp(chrHC[700,800)) insert~500 filler={n_filler}\n") +PY + +echo "fixture(hookc) written to $d" >&2 diff --git a/test/altlg/mkfixture-indel.sh b/test/altlg/mkfixture-indel.sh new file mode 100755 index 0000000..d00fd6f --- /dev/null +++ b/test/altlg/mkfixture-indel.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Fixture for Task 7 fix round 1, Fix 3: REJECT length-changing lifted spans. +# +# A seed that spans an indel (or two adjacent .alt lift blocks) lifts to a primary +# span whose length differs from the seed length. Injecting it as a contiguous +# len-bp anchor would corrupt the chain coordinates, so mb_anchor_project_alt must +# REJECT it (pri_en-pri_st+1 != q->len -> continue). +# +# Construction: +# chrP_altI is an ALT contig of LEFT(75) + RIGHT(75) = 150bp. Its .alt CIGAR is +# 75M5D75M: the two 75bp blocks lift to primary with a 5bp DELETION from the ALT +# between them, so the primary span of a seed covering BOTH blocks is 5bp LONGER +# than the ALT seed. +# The read == chrP_altI (150bp), unique to the ALT contig, so its full-length +# SMEM (which SPANS the 5bp deletion, covering both blocks) survives and reaches +# the projection. Because that span lifts to length seed_len+5 != seed_len, the +# projection must drop it -> NO MB_PROJ line for the spanning seed. +# +# Observable assertion (test-indel.sh, via the MB_PROJ_TRACE probe seam): +# With Fix 3: the full-length seed (len ~150, spanning the deletion) produces +# NO projected anchor (rejected). Any projected anchor that DOES +# appear has len == its lifted span length (single-block seeds +# only). +# Without Fix 3: the spanning seed would be injected as a len-bp anchor at a +# primary span that is actually len+5 bp -- a coordinate bug. +# +# Usage: mkfixture-indel.sh +set -eu +. "$(dirname "$0")/fixlib.sh" +d="$1"; mkdir -p "$d" + + +LEFT=$(gen 11 75) # 75bp block A (shared ALT<->primary) +RIGHT=$(gen 22 75) # 75bp block B (shared ALT<->primary) +GAP=$(gen 33 5) # 5bp present on primary only (D in the ALT->primary CIGAR) +ALT="${LEFT}${RIGHT}" # 150bp ALT contig == read +LA=${#ALT} + +# chrP (primary): PAD0 + LEFT + GAP + RIGHT + PADZ. The .alt deletes GAP from the +# ALT, so the two ALT blocks lift to primary positions 5bp farther apart than they +# are on the ALT -> a seed covering both blocks changes length under the lift. +PAD0=$(gen 1 200) +PADZ=$(gen 5 200) +CHRP="${PAD0}${LEFT}${GAP}${RIGHT}${PADZ}" +LIFT_OFF=${#PAD0} +LIFT_POS1=$(( LIFT_OFF + 1 )) # 1-based POS where block A (alt 0) lifts + +printf '>chrP\n%s\n' "$CHRP" > "$d/ref.fa" +printf '>chrP_altI\n%s\n' "$ALT" >> "$d/ref.fa" + +# .alt: chrP_altI -> chrP at LIFT_POS1, forward, 75M5D75M (5bp deletion from ALT). +printf 'chrP_altI\t0\tchrP\t%d\t60\t75M5D75M\t*\t0\t0\t*\t*\n' "$LIFT_POS1" \ + > "$d/ref.fa.alt" + +QUAL=$(printf "%${LA}s" '' | tr ' ' 'I') +printf '@r-indel\n%s\n+\n%s\n' "$ALT" "$QUAL" > "$d/reads.fq" + +echo "LIFT_POS1=$LIFT_POS1" > "$d/meta.txt" +echo "fixture(indel): chrP len=${#CHRP} alt=chrP_altI(${LA}) CIGAR=75M5D75M LIFT@${LIFT_POS1}" >&2 diff --git a/test/altlg/mkfixture-lifttol.sh b/test/altlg/mkfixture-lifttol.sh new file mode 100755 index 0000000..91d9db9 --- /dev/null +++ b/test/altlg/mkfixture-lifttol.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# Fixture for the runtime-tunable ALT lift tolerance (--alt-lift-tol). +# +# Models a near-twin whose lifted placement sits 15bp away from where the read +# aligns directly on the primary -- i.e. lift/placement JITTER of 15bp (more than +# the default MB_LIFT_TOL=10, less than a real paralog separation). Such jitter +# is what a too-tight tolerance fails to absorb: the twin and the primary fall +# into separate groups, compete, and the read is wrongly driven to MAPQ 0. +# +# chrP 800bp unique sequence (LCG PRNG, seed 11). +# read r-near chrP[300,420) (1-based 301..420), 120bp. Aligns directly to +# chrP at POS 301 => primary hit, lifted_st = 300 (0-based). +# chrP_altT EXACT copy of chrP[300,420) (120bp), so the read also aligns +# to it perfectly => an equal-scoring ALT hit. +# .alt chrP_altT aligned to chrP at POS 316 (120M). The ALT hit +# therefore LIFTS to chrP 0-based 315 -- 15bp from the primary's +# 300. |Δlifted_st| = 15. +# +# Expected: +# --alt-lift-tol 10 (default): 15 > 10 => twin is a SEPARATE group => equal-score +# competitor => r-near MAPQ 0 (conservative default; under-merges jitter). +# --alt-lift-tol 20 : 15 <= 20 => twin GROUPS with the primary => chrP is +# sam_pri with MAPQ>0, the twin secondary (recovered). +set -eu +d="$1"; mkdir -p "$d" + +# 800bp unique sequence (LCG PRNG, seed 11). +S=$(python3 -c " +import random +random.seed(11) +print(''.join(random.choice('ACGT') for _ in range(800))) +") + +# ALT twin: EXACT copy of chrP[300,420) (1-based 301..420), 120bp. +ALTT=$(printf '%s' "$S" | cut -c301-420) +LT=${#ALTT} + +# --- reference --- +printf '>chrP\n%s\n' "$S" > "$d/ref.fa" +printf '>chrP_altT\n%s\n' "$ALTT">> "$d/ref.fa" + +# --- .alt: altT placed 15bp downstream of its true primary window (POS 316, not +# 301) so the lift lands at chrP 0-based 315 -- 15bp of jitter vs the read's +# direct primary alignment at 300. --- +printf 'chrP_altT\t0\tchrP\t316\t60\t%dM\t*\t0\t0\t*\t*\n' "$LT" > "$d/ref.fa.alt" + +# --- read --- +QUAL120=$(printf '%120s' '' | tr ' ' 'I') +RNEAR=$(printf '%s' "$S" | cut -c301-420) +printf '@r-near\n%s\n+\n%s\n' "$RNEAR" "$QUAL120" > "$d/reads.fq" + +echo "fixture(lifttol): chrP=800 altT=chrP[300,420) placed +15bp (lifted_st jitter=15)" >&2 diff --git a/test/altlg/mkfixture-paralog.sh b/test/altlg/mkfixture-paralog.sh new file mode 100755 index 0000000..6f2e324 --- /dev/null +++ b/test/altlg/mkfixture-paralog.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# Fixture for Task 4 (reconciliation pass), case (b): the PARALOG GUARD. +# +# This is the BLOCKER fixture: it must stress the MB_LIFT_TOL boundary, not be +# trivially far apart. Two near-identical primary loci whose ALT twins lift to +# lifted_st values JUST OUTSIDE MB_LIFT_TOL (=10) must NOT be merged into one +# group — the read must stay a low-MAPQ multi-mapper. +# +# Reference layout on chrP (primary): +# [pad1 200bp][COPY1 (150bp)][gap 220bp][COPY2 (150bp, == COPY1)][pad2 200bp] +# COPY1 1-based start = 201 (0-based 200) +# COPY2 1-based start = 201+150+220=571 (0-based 570) +# => the two paralog loci on chrP are 370bp apart. +# +# ALT twins (separate ALT contigs, each a single COPY-sized contig): +# chrP_altA = COPY (== COPY1 == COPY2). .alt aligns it to chrP at POS=201 +# (1-based) => lifts to lifted_st ~= 200 (group of COPY1). +# chrP_altB = COPY. .alt aligns it to chrP at POS=221 (1-based) => +# lifts to lifted_st ~= 220. This is altA's lift (200) + 20, +# i.e. exactly TWICE MB_LIFT_TOL away — just outside tolerance. +# +# A read from COPY maps to: +# chrP@201 (COPY1, lifted_st ~200) +# chrP@571 (COPY2, lifted_st ~570) +# chrP_altA (lifts to ~200) +# chrP_altB (lifts to ~220) +# +# Correct grouping with MB_LIFT_TOL=10: +# {chrP@201 (~200), chrP_altA (~200)} and {chrP@571 (~570)} and +# {chrP_altB (~220)} — altB is 20bp from the COPY1 group's 200, > tol, so it +# does NOT merge. The point: even the CLOSEST competing placements (~200 vs +# ~220) stay distinct at the 2*tol boundary, so multiple genuine loci survive +# => the read is a multi-mapper => MAPQ 0 (or very low). If grouping were +# sloppy and merged altB into the COPY1 group, we would still have COPY2 as a +# distinct competitor, so the multi-mapper verdict holds regardless; the +# assertion that proves the boundary is checked by the placement dump. +set -eu +. "$(dirname "$0")/fixlib.sh" +d="$1"; mkdir -p "$d" + +# Unique-ish building blocks (LCG PRNG) so each region is internally unique but +# COPY1==COPY2 exactly (the paralog). + +PAD1=$(gen 1 200) +COPY=$(gen 2 150) +GAP=$(gen 3 220) +PAD2=$(gen 4 200) + +CHRP="${PAD1}${COPY}${GAP}${COPY}${PAD2}" +LEN=${#CHRP} + +# ALT twins: each contig is just COPY (150bp). +printf '>chrP\n%s\n' "$CHRP" > "$d/ref.fa" +printf '>chrP_altA\n%s\n' "$COPY">> "$d/ref.fa" +printf '>chrP_altB\n%s\n' "$COPY">> "$d/ref.fa" + +# .alt: altA aligns to chrP at POS 201 (the COPY1 locus); altB at POS 221 +# (20bp downstream — 2*MB_LIFT_TOL). Both full-length 150M. +LC=${#COPY} +printf 'chrP_altA\t0\tchrP\t201\t60\t%dM\t*\t0\t0\t*\t*\n' "$LC" > "$d/ref.fa.alt" +printf 'chrP_altB\t0\tchrP\t221\t60\t%dM\t*\t0\t0\t*\t*\n' "$LC" >> "$d/ref.fa.alt" + +# read: the COPY sequence (matches both COPY1 and COPY2, and both ALT twins). +QUAL=$(printf "%${LC}s" '' | tr ' ' 'I') +printf '@r-para\n%s\n+\n%s\n' "$COPY" "$QUAL" > "$d/reads.fq" + +echo "fixture(paralog): chrP=$LEN COPY1@201 COPY2@571 altA->201 altB->221 (Δ20 > 2*tol)" >&2 diff --git a/test/altlg/mkfixture-pe-happy.sh b/test/altlg/mkfixture-pe-happy.sh new file mode 100755 index 0000000..2e7130d --- /dev/null +++ b/test/altlg/mkfixture-pe-happy.sh @@ -0,0 +1,111 @@ +#!/bin/sh +# Fixture for Task 8 (PE reconciliation integration), the HAPPY-PATH case. +# +# Goal: a paired read whose BOTH mates land inside a primary region that also has +# an identical ALT twin contig. On the primary the pair (R1@chrP, R2@chrP) is the +# best PAIR; on the ALT twin the pair (R1@altF, R2@altF) is an EQUALLY-GOOD second +# PAIR (same insert, same score, identical sequence). In mb_pair_hits, pairs only +# form within ONE contig (pe.c:174 requires hi->tid == hk->tid), so the twin pair +# is a real competitor: +# +# BEFORE Hook B: paux.sub_sc ~= paux.score => mapq_pe ~= 0 (the pair looks +# like a 2-way multi-mapper even though both placements are the +# SAME locus on primary vs ALT). +# AFTER Hook B: the ALT (non-rep group member) hits are excluded from pair +# enumeration => the twin pair never forms => paux.sub_sc is +# the true second-best (none) => mapq_pe > 0; chrP is sam_pri for +# BOTH mates, the chrP_altF twin records are secondary. +# +# pestat needs >= MIN_DIR_CNT (20) unique FR pairs to not fail, so we emit a block +# of FILLER pairs from unique chrP windows (no ALT twin) at a consistent ~480bp FR +# insert. Those establish the insert distribution; the single test pair (r-twin) +# exercises the feature. Filler reads are 120bp, well-separated, each unique. +# +# Reference layout (chrP, 6000bp unique LCG sequence, seed 21): +# - chrP[2400,2940) (540bp) is duplicated as ALT contig chrP_altF; .alt aligns +# chrP_altF full-length forward to chrP at POS=2401. +# - r-twin: R1 = chrP[2480,2600) fwd, R2 = revcomp(chrP[2820,2940)). Insert +# (5' to 5') ~= 460bp, FR orientation, well within the filler distribution. +# Both mates lie inside [2400,2940) so both also map to chrP_altF. +set -eu +d="$1"; mkdir -p "$d" + +python3 - "$d" <<'PY' +import sys, random +d = sys.argv[1] +random.seed(21) +N = 6000 +S = ''.join(random.choice('ACGT') for _ in range(N)) + +def revcomp(s): + c = {'A':'T','C':'G','G':'C','T':'A'} + return ''.join(c[b] for b in reversed(s)) + +# ALT-twinned window on chrP: [2400, 2940) (0-based), 540bp. +ALT_ST, ALT_EN = 2400, 2940 +ALTF = S[ALT_ST:ALT_EN] +LF = len(ALTF) + +# --- reference --- +with open(f"{d}/ref.fa", "w") as f: + f.write(f">chrP\n{S}\n") + f.write(f">chrP_altF\n{ALTF}\n") + +# --- .alt: chrP_altF aligns full-length forward to chrP at 1-based POS=2401 --- +with open(f"{d}/ref.fa.alt", "w") as f: + f.write(f"chrP_altF\t0\tchrP\t{ALT_ST+1}\t60\t{LF}M\t*\t0\t0\t*\t*\n") + +RL = 120 +QUAL = 'I' * RL + +def fq(name, r1, r2): + return (f"@{name}/1\n{r1}\n+\n{QUAL}\n", f"@{name}/2\n{r2}\n+\n{QUAL}\n") + +reads1, reads2 = [], [] + +# --- FILLER pairs: 30 unique FR pairs spread across chrP, insert ~480bp, all +# OUTSIDE the ALT window so each is a clean unique pair for pestat. --- +filler_starts = [] +pos = 200 +n_filler = 30 +for _ in range(n_filler): + # keep filler fragments clear of the ALT window [2400,2940) + while ALT_ST - 120 < pos < ALT_EN + 120: + pos += 600 + if pos + 480 + RL > N - 200: + pos = 200 + filler_starts.append(pos) + pos += 150 +jit = random.Random(99) +for i, st in enumerate(filler_starts): + # Jitter the 5'-5' insert so the proper-pair window has real width (a fixed + # insert collapses pestat's std.dev to 0 and the [lo,hi] window to a single + # value, which would reject the test pair). Inserts span ~400..520. + gap = jit.randint(280, 400) # 5'-5' insert = gap + RL = 400..520 + if st + gap + RL > N - 200: + gap = 280 + r1 = S[st:st+RL] + r2 = revcomp(S[st+gap:st+gap+RL]) + a, b = fq(f"f{i:02d}", r1, r2) + reads1.append(a); reads2.append(b) + +# --- TEST pair r-twin: both mates inside the ALT-twinned window. --- +# R1 = chrP[2480,2600) fwd ; R2 = revcomp(chrP[2820,2940)). 5'-5' insert = 460, +# squarely inside the filler distribution (400..520). Both mates lie in the ALT +# window [2400,2940) so each also maps to chrP_altF. +r1 = S[2480:2600] +r2 = revcomp(S[2820:2940]) +a, b = fq("r-twin", r1, r2) +reads1.append(a); reads2.append(b) + +with open(f"{d}/reads_1.fq", "w") as f: + f.writelines(reads1) +with open(f"{d}/reads_2.fq", "w") as f: + f.writelines(reads2) + +sys.stderr.write( + f"fixture(pe-happy): chrP={N} altF=chrP[{ALT_ST},{ALT_EN})->POS{ALT_ST+1} " + f"filler={n_filler}FR(ins480) r-twin(ins460,both mates in ALT window)\n") +PY + +echo "fixture(pe-happy) written to $d" >&2 diff --git a/test/altlg/mkfixture-pe-paralog.sh b/test/altlg/mkfixture-pe-paralog.sh new file mode 100755 index 0000000..a23e1c4 --- /dev/null +++ b/test/altlg/mkfixture-pe-paralog.sh @@ -0,0 +1,97 @@ +#!/bin/sh +# Fixture for Task 8 (PE reconciliation integration), the PARALOG GUARD case +# (Hook C): the chimeric demotion loop at pe.c:524-535 demotes a hit `p` under the +# chosen pair-hit `h[r]` when their QUERY spans overlap a lot (ol > mask_level). +# Without Hook C that demotion fires on two hits that are DIFFERENT lifted groups +# (different primary loci / paralogs), merging them and inflating MAPQ — undoing +# the Task-4(b) paralog guard for PE. Hook C skips the demotion when `p` and `h[r]` +# are different lifted groups (different pri_tid/rev or |Δlifted_st| > MB_LIFT_TOL). +# +# Geometry: chrP carries two near-identical paralog copies of COPY (COPY1, COPY2) +# whose ALT twins lift just OUTSIDE MB_LIFT_TOL apart (the same 2*tol boundary the +# SE paralog fixture stresses). One mate (R1) of the pair IS the ambiguous COPY +# sequence, so it has competing hits at COPY1, COPY2, altA, altB — overlapping +# query spans but DIFFERENT lifted groups. The other mate (R2) is unique and +# anchors the proper pair near COPY1. A block of filler pairs establishes pestat. +# +# BEFORE Hook C: :526 demotes the COPY2/altB-group hit under the COPY1 pair-hit +# (query spans fully overlap) => groups merge => the read looks +# uniquely placed => MAPQ inflated. +# AFTER Hook C: the cross-group demotion is skipped => COPY2 stays a distinct +# competing locus => R1 remains a low-MAPQ multi-mapper. +set -eu +d="$1"; mkdir -p "$d" + +python3 - "$d" <<'PY' +import sys, random +d = sys.argv[1] + +def gen(seed, n): + r = random.Random(seed) + return ''.join(r.choice('ACGT') for _ in range(n)) + +def revcomp(s): + c = {'A':'T','C':'G','G':'C','T':'A'} + return ''.join(c[b] for b in reversed(s)) + +PAD1 = gen(1, 2000) +COPY = gen(2, 150) +GAP = gen(3, 220) +PAD2 = gen(4, 2000) +CHRP = PAD1 + COPY + GAP + COPY + PAD2 +N = len(CHRP) +# COPY1 0-based start = 2000 ; COPY2 0-based start = 2000+150+220 = 2370. +C1 = len(PAD1) # 2000 +C2 = len(PAD1) + len(COPY) + len(GAP) # 2370 +LC = len(COPY) + +# ALT twins: each contig is just COPY (150bp); altA->COPY1 locus, altB 20bp +# downstream (2*MB_LIFT_TOL), just outside tolerance -> distinct groups. +with open(f"{d}/ref.fa", "w") as f: + f.write(f">chrP\n{CHRP}\n") + f.write(f">chrP_altA\n{COPY}\n") + f.write(f">chrP_altB\n{COPY}\n") +with open(f"{d}/ref.fa.alt", "w") as f: + f.write(f"chrP_altA\t0\tchrP\t{C1+1}\t60\t{LC}M\t*\t0\t0\t*\t*\n") + f.write(f"chrP_altB\t0\tchrP\t{C1+1+20}\t60\t{LC}M\t*\t0\t0\t*\t*\n") + +QUAL = 'I' * LC +reads1, reads2 = [], [] + +def fq(name, r1, r2): + q1 = 'I' * len(r1); q2 = 'I' * len(r2) + return (f"@{name}/1\n{r1}\n+\n{q1}\n", f"@{name}/2\n{r2}\n+\n{q2}\n") + +# Filler pairs from unique windows of PAD1/PAD2 to establish pestat. Jitter the +# insert so pestat's proper-pair window has real width (a fixed insert collapses +# std.dev to 0 and the [lo,hi] window to a single value). Inserts span ~400..520. +n_filler = 30 +jit = random.Random(99) +starts = [] +pos = 100 +for _ in range(n_filler): + starts.append(pos); pos += 55 +for i, st in enumerate(starts): + gap = jit.randint(280, 400) # 5'-5' insert = gap + 120 = 400..520 + r1 = CHRP[st:st+120] + r2 = revcomp(CHRP[st+gap:st+gap+120]) + a, b = fq(f"f{i:02d}", r1, r2) + reads1.append(a); reads2.append(b) + +# Test pair r-para: R1 = COPY (ambiguous: COPY1/COPY2/altA/altB). R2 = unique +# window downstream of COPY1, revcomp, giving an FR pair (insert 460) anchored +# near COPY1, squarely inside the filler distribution. +r1 = COPY +r2 = revcomp(CHRP[C1+340:C1+340+120]) +a, b = fq("r-para", r1, r2) +reads1.append(a); reads2.append(b) + +with open(f"{d}/reads_1.fq", "w") as f: f.writelines(reads1) +with open(f"{d}/reads_2.fq", "w") as f: f.writelines(reads2) + +sys.stderr.write( + f"fixture(pe-paralog): chrP={N} COPY1@{C1} COPY2@{C2} altA->{C1+1} altB->{C1+21} " + f"(Δ20>2*tol) filler={n_filler} r-para(R1=COPY ambiguous)\n") +PY + +echo "fixture(pe-paralog) written to $d" >&2 diff --git a/test/altlg/mkfixture-pe-rescue.sh b/test/altlg/mkfixture-pe-rescue.sh new file mode 100755 index 0000000..52f13aa --- /dev/null +++ b/test/altlg/mkfixture-pe-rescue.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# Fixture for Task 8 (PE reconciliation integration), the RESCUED-ALT-MATE case. +# +# Goal: one mate (R2) cannot seed on its own (peppered with mismatches so no +# k-mer seed survives), so it is recovered ONLY by mate rescue (mb_matesw_core). +# The rescued mate lands inside a primary region that has an identical ALT twin +# contig, so mate rescue produces hits on BOTH chrP and the ALT twin. For the +# liftover-group reconciliation (Hook A) to fold the rescued ALT hit under its +# primary, the rescued hit must carry is_alt -- but mb_matesw_align does +# memset(h,0), clearing is_alt. The is_alt stamp in mb_matesw_core fixes that. +# +# WITHOUT the stamp + Hook A: the rescued ALT hit has is_alt=0, so reconcile is +# blind to it; it competes as a distinct locus (or mis-groups), depressing +# the pair MAPQ / mis-flagging the twin. +# WITH both: the rescued chrP_altG hit is grouped under chrP (secondary, 0x100), +# chrP is the proper-pair primary, MAPQ stays high. +# +# R1 is clean and unique (anchors the pair). A block of filler pairs (jittered +# insert) establishes pestat. R2's true locus is inside the ALT-twinned window. +set -eu +d="$1"; mkdir -p "$d" + +python3 - "$d" <<'PY' +import sys, random +d = sys.argv[1] +random.seed(31) +N = 6000 +S = ''.join(random.choice('ACGT') for _ in range(N)) + +def revcomp(s): + c = {'A':'T','C':'G','G':'C','T':'A'} + return ''.join(c[b] for b in reversed(s)) + +def pepper(s, period, seed): + # flip every `period`-th base so no k-mer (default 11..19) survives a window + r = random.Random(seed) + flip = {'A':'C','C':'G','G':'T','T':'A'} + out = list(s) + for i in range(period//2, len(out), period): + out[i] = flip[out[i]] + return ''.join(out) + +# ALT-twinned window on chrP: [3000, 3600) (600bp), duplicated as chrP_altG. +ALT_ST, ALT_EN = 3000, 3600 +ALTG = S[ALT_ST:ALT_EN] +LG = len(ALTG) + +with open(f"{d}/ref.fa", "w") as f: + f.write(f">chrP\n{S}\n") + f.write(f">chrP_altG\n{ALTG}\n") +with open(f"{d}/ref.fa.alt", "w") as f: + f.write(f"chrP_altG\t0\tchrP\t{ALT_ST+1}\t60\t{LG}M\t*\t0\t0\t*\t*\n") + +RL = 120 +reads1, reads2 = [], [] +def fq(name, r1, r2): + return (f"@{name}/1\n{r1}\n+\n{'I'*len(r1)}\n", f"@{name}/2\n{r2}\n+\n{'I'*len(r2)}\n") + +# Filler pairs (jittered FR insert 400..520) from unique windows clear of the ALT +# window, to establish pestat. +jit = random.Random(77) +pos = 200 +n_filler = 30 +for i in range(n_filler): + while ALT_ST - 200 < pos < ALT_EN + 200: + pos += 600 + if pos + 520 + RL > N - 200: + pos = 200 + gap = jit.randint(280, 400) + r1 = S[pos:pos+RL] + r2 = revcomp(S[pos+gap:pos+gap+RL]) + a, b = fq(f"f{i:02d}", r1, r2) + reads1.append(a); reads2.append(b) + pos += 150 + +# Test pair r-rescue: BOTH mates lie inside the ALT-twinned window [3000,3600), so +# each also has a chrP_altG hit. R1 = chrP[3080,3200) seeds cleanly (anchors and +# also maps to chrP_altG). R2's true locus chrP[3440,3560) is peppered so it +# cannot seed and is recovered ONLY by mate rescue -- landing on both chrP and +# chrP_altG. Because R1 also has a chrP_altG hit, the rescued R2 chrP_altG hit can +# form a competing (altG,altG) PAIR. Only if the rescued mate carries is_alt does +# Hook A group it (parent != id) so Hook B excludes it from the second pairing; +# without the is_alt stamp the (altG,altG) pair competes and depresses mapq_pe. +r1 = S[3080:3200] +r2_true = revcomp(S[3440:3560]) +r2 = pepper(r2_true, 11, 5) +a, b = fq("r-rescue", r1, r2) +reads1.append(a); reads2.append(b) + +with open(f"{d}/reads_1.fq", "w") as f: f.writelines(reads1) +with open(f"{d}/reads_2.fq", "w") as f: f.writelines(reads2) + +sys.stderr.write( + f"fixture(pe-rescue): chrP={N} altG=chrP[{ALT_ST},{ALT_EN})->POS{ALT_ST+1} " + f"R1=chrP[3080,3200) clean, R2=chrP[3440,3560) peppered(period11)->rescue-only\n") +PY + +echo "fixture(pe-rescue) written to $d" >&2 diff --git a/test/altlg/mkfixture-place.sh b/test/altlg/mkfixture-place.sh new file mode 100755 index 0000000..1b8e95c --- /dev/null +++ b/test/altlg/mkfixture-place.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# Fixture for Task 2 (mb_hit_place — per-hit lifted placement). +# +# Reference (ref.fa): +# chrP primary contig (length L) +# chrP_alt forward ALT = chrP with a 5bp insertion ("GGGGG") at offset 300, +# so the .alt CIGAR is 300M5IM, FLAG 0 (POS 1). +# chrR_alt reverse-strand ALT = reverse complement of chrP[250,450); +# .alt line is FLAG 16, RNAME chrP, POS 251, CIGAR 200M. +# +# Reads (one FASTA each so QNAME survives): +# pa-fwd chrP[120,270) forward, inside the shared flank -> case (a)/(b): +# primary chrP hit (identity) and forward chrP_alt hit lift to the +# same primary lifted_st (~120), rev 0, liftable 1. +# c-fwd chrP[300,400) forward -> case (c) primary side: chrP hit, rev 0 +# c-rev-twin reverse complement of chrP[300,400): maps to chrR_alt on the minus +# strand; block_rev(1) XOR h->rev(1) = 0, footprint folds back to +# chrP[300,400) so lifted_st ~300, rev 0 -> groups with c-fwd. case (c) +# hole-end chrP_alt read that STARTS inside the 5bp "GGGGG" insertion (a hole) +# and continues into liftable bases -> case (d): liftable 1, lifted_st +# taken from the first liftable base. +# twin-indel chrP read with a tiny (<=tol) deletion vs chrP_alt: still groups. +set -eu +. "$(dirname "$0")/fixlib.sh" +d="$1"; mkdir -p "$d" + +S=$(printf '%s' 'GATCCTAGCATGCTAGGCTAACGTTAGCCGATCGTAGCTAGGCATCGATCGTAGCTAGCTAGGCATCGATTACGATCGGCTAATCGATCGTAGCTGATCGA'\ +'TCGTAGCTAGCATCGATCGTAGCATCGGCTAGCATCGATCGATTACGCATCGATCGTAGGCTAGCATCGATCGTAGCTAGCATCGGCTAGCATCGATTACG'\ +'ATCGGCTAATCGATCGTAGCTGATCGATCGTAGCTAGCATCGATCGTAGCATCGGCTAGCATCGATCGATTACGCATCGATCGTAGGCTAGCATCGATCGT'\ +'AGCTAGCATCGGCTAGCATCGATTACGATCGGCTAATCGATCGTAGCTGATCGAGATCCTAGCATGCTAGGCTAACGTTAGCCGATCGTAGCTAGGCATCG'\ +'ATCGTAGCTAGCTAGGCATCGATTACGATCGGCTAATCGATCGTAGCTGATCGATCGTAGCTAGCATCGATCGTAGCATCGGCTAGCATCGATCGATTACG') +L=${#S} + +# revcomp helper (reads stdin, writes revcomp to stdout) + +# --- contigs --- +printf '>chrP\n%s\n' "$S" > "$d/ref.fa" +ALT=$(printf '%s' "$S" | mawk '{print substr($0,1,300) "GGGGG" substr($0,301)}') # 5bp insertion at 300 +printf '>chrP_alt\n%s\n' "$ALT" >> "$d/ref.fa" +# chrR_alt = revcomp of chrP[250,450) (1-based cut 251..450) +WIN=$(printf '%s' "$S" | cut -c251-450) +RWIN=$(printf '%s' "$WIN" | rc) +printf '>chrR_alt\n%s\n' "$RWIN" >> "$d/ref.fa" + +# --- .alt --- +rest=$(( L - 300 )) +{ + printf 'chrP_alt\t0\tchrP\t1\t60\t300M5I%dM\t*\t0\t0\t*\t*\n' "$rest" + printf 'chrR_alt\t16\tchrP\t251\t60\t200M\t*\t0\t0\t*\t*\n' +} > "$d/ref.fa.alt" + +Q=$(printf '%150s' '' | tr ' ' 'I') # 150 I quals (long enough for any read) +emit() { # emit + n="$1"; s="$2"; ql=$(printf '%s' "$Q" | cut -c1-${#s}) + printf '@%s\n%s\n+\n%s\n' "$n" "$s" "$ql" >> "$d/reads.fq" +} +: > "$d/reads.fq" + +# (a)/(b) forward read in the shared flank: chrP[120,270) (1-based 121..270) +emit pa-fwd "$(printf '%s' "$S" | cut -c121-270)" + +# (c) primary-side forward read chrP[300,400) (1-based 301..400) +CF=$(printf '%s' "$S" | cut -c301-400) +emit c-fwd "$CF" +# (c) reverse-complement twin of the SAME read -> hits chrR_alt on minus strand +emit c-rev-twin "$(printf '%s' "$CF" | rc)" + +# (d) one-end-in-a-hole: read whose first 5 bases are the inserted "GGGGG" +# (positions 301..305 on chrP_alt, a hole) then 145 liftable bases from chrP_alt +# 306.. (== chrP 301..). Built directly from chrP_alt so it aligns there. +HOLE=$(printf '%s' "$ALT" | cut -c301-450) # GGGGG + 145 liftable bases +emit hole-end "$HOLE" + +# small (<=tol) read-vs-ALT indel twin: chrP read with a 3bp deletion at offset 60 +# relative to chrP_alt's content; placement absorbs it and it still groups (~chrP 120) +TW=$(printf '%s' "$S" | mawk -v s=121 -v e=270 '{x=substr($0,s,e-s+1); print substr(x,1,60) substr(x,64)}') +emit twin-indel "$TW" + +echo "fixture: L=$L chrP_alt(+5I@300) chrR_alt(rev of chrP[250,450))" >&2 diff --git a/test/altlg/mkfixture-projrev.sh b/test/altlg/mkfixture-projrev.sh new file mode 100755 index 0000000..00462a7 --- /dev/null +++ b/test/altlg/mkfixture-projrev.sh @@ -0,0 +1,108 @@ +#!/bin/sh +# Fixture for test-projrev.sh: REVERSE-strand ALT-seed -> primary projection, +# SAM-observable via MAPQ differential (no probe required). +# +# MECHANISM +# --------- +# The read's full 150bp SMEM has SA=2: one occurrence on the ALT contig +# (forward) and one on the primary LIFT locus (reverse complement). Running +# with -c 1 (max_occ=1) forces the strided sampler to take exactly ONE of the +# two hits from the SA interval. The SA-array lexicographic order (determined +# by the sequence seeds below) places the ALT contig hit first, so: +# +# WITHOUT projection (MB_NO_ALT_PROJECT=1): +# Sampler takes the ALT-contig hit -> ALT anchor -> alignment to chrP_altS +# only; no primary anchor at LIFT -> chrP_altS is SAM primary (MAPQ 60), +# NO chrP record at LIFT_POS1. +# +# WITH projection (default): +# Sampler takes the ALT-contig hit -> ALT anchor -> mb_anchor_project_alt +# injects a reverse-strand native primary anchor at LIFT_POS1 (folded_rev = +# blk_rev ^ alt_rev = 1 ^ 0 = 1). DP at LIFT: read maps perfectly (150M, +# NM=0, score=300) because the primary stores RC(read) at LIFT_POS1. +# Reconciliation groups {chrP_altS, chrP LIFT}: scores tie at 300; non-ALT +# preference promotes chrP -> chrP is SAM primary (FLAG=16, MAPQ=60). +# chrP_altS becomes secondary (FLAG=256, MAPQ=0). +# +# WHY THE EXISTING segdup-rev FIXTURE CAN'T DO THIS +# -------------------------------------------------- +# In segdup-rev the LIFT locus stores RC(CORE)+TAIL1 where TAIL1 != RC(ALTTAIL), +# so the primary alignment has ~50 bp of mismatches -> DP score ~202 < ALT +# score 300 -> ALT wins as group rep -> chrP LIFT is secondary (MAPQ=0). +# Making TAIL1 = RC(ALTTAIL) (perfect match) would create junction k-mers +# with SA=2 that are always fully sampled (SA <= max_occ=4), bypassing +# projection entirely. The trick here is using -c 1: the SA=2 full-read SMEM +# IS subsampled (SA > max_occ=1), and the SA-array ordering deterministically +# puts the ALT contig hit first for the chosen sequence seeds. +# +# REVERSE-STRAND PROJECTION CODE PATH +# ------------------------------------ +# The ALT anchor is forward on chrP_altS (alt_rev=0). The .alt block is RC +# (FLAG 0x10, blk_rev=1). mb_anchor_project_alt computes: +# folded_rev = blk_rev ^ alt_rev = 1 ^ 0 = 1 (reverse primary strand) +# This exercises the same Fix 1 + Fix 2 coordinate path tested in +# test-segdup-rev.sh (via MB_PROJ_TRACE), but here the result is directly +# observable in the SAM output. +# +# REFERENCE LAYOUT (chrP, primary) +# --------------------------------- +# [PAD0(200bp)] [LIFT: RC(ALTTAIL)(50bp) + RC(CORE)(100bp)] [decoy copies x N] [PADZ(200bp)] +# <- LIFT_POS1=201 (1-based) ^ +# +# The LIFT region is RC(read), so the read aligns perfectly in REVERSE. +# Decoy copies store RC(CORE)+TAILi (unique random tails), providing the +# sub-SMEM anchors that are also subsampled by -c 1. +# +# Usage: mkfixture-projrev.sh [N-copies] +set -eu +. "$(dirname "$0")/fixlib.sh" +d="$1"; mkdir -p "$d" +N="${2:-20}" + + +# Sequence seeds chosen so the SA-array lexicographic order places the +# ALT-contig hit BEFORE the primary LIFT hit in the 150bp SMEM's SA interval, +# making max_occ=1 (-c 1) deterministically pick the ALT contig -> projection +# is the ONLY path to the primary LIFT locus. +CORE=$(gen 500 100) # 100bp shared seed core (repetitive via RC on primary) +ALTTAIL=$(gen 5777 50) # 50bp tail UNIQUE to the ALT contig (and RC is unique to LIFT locus) +READ="${CORE}${ALTTAIL}" # 150bp read == ALT contig sequence (forward) +LR=${#READ} +RCCORE=$(rc "$CORE") # RC(CORE): stored at each decoy and at the 2nd half of LIFT +RCALTTAIL=$(rc "$ALTTAIL") # RC(ALTTAIL): stored at the 1st half of LIFT only + +# LIFT region = RC(ALTTAIL)(50bp) + RC(CORE)(100bp) = RC(read) -- perfect +# reverse match for the read. The .alt FLAG=0x10 block maps ALT -> primary in +# RC, so primary[LIFT_POS1-1 .. LIFT_POS1+LR-2] = RC(ALT) = RC(read). +LIFT_REGION="${RCALTTAIL}${RCCORE}" + +# chrP: PAD0 + LIFT + (decoy copies: PADi + RC(CORE) + TAILi) + PADZ +PAD0=$(gen 1 200) +CHRP="${PAD0}${LIFT_REGION}" +LIFT_OFF=${#PAD0} +LIFT_POS1=$(( LIFT_OFF + 1 )) # 1-based POS of the LIFT locus + +i=2 +while [ "$i" -le "$N" ]; do + PADI=$(gen $(( 2000 + i )) 70) + TAILI=$(gen $(( 4000 + i )) 50) + CHRP="${CHRP}${PADI}${RCCORE}${TAILI}" + i=$(( i + 1 )) +done +PADZ=$(gen 5 200) +CHRP="${CHRP}${PADZ}" + +printf '>chrP\n%s\n' "$CHRP" > "$d/ref.fa" +printf '>chrP_altS\n%s\n' "$READ" >> "$d/ref.fa" + +# .alt: chrP_altS -> chrP at the LIFT locus, RC (FLAG 0x10), full length. +# The block strand (blk_rev=1) XOR the forward ALT-seed strand (alt_rev=0) +# gives folded_rev=1: the projected primary anchor is on the REVERSE strand. +printf 'chrP_altS\t16\tchrP\t%d\t60\t%dM\t*\t0\t0\t*\t*\n' "$LIFT_POS1" "$LR" \ + > "$d/ref.fa.alt" + +QUAL=$(printf "%${LR}s" '' | tr ' ' 'I') +printf '@r-projrev\n%s\n+\n%s\n' "$READ" "$QUAL" > "$d/reads.fq" + +echo "LIFT_POS1=$LIFT_POS1" > "$d/meta.txt" +echo "fixture(projrev): chrP len=${#CHRP} N=$N core=100 alttail=50 LIFT@${LIFT_POS1} RC ALT=chrP_altS(${LR}) SA_full=2 max_occ=1" >&2 diff --git a/test/altlg/mkfixture-reconcile.sh b/test/altlg/mkfixture-reconcile.sh new file mode 100755 index 0000000..b205789 --- /dev/null +++ b/test/altlg/mkfixture-reconcile.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# Fixture for Task 4 (reconciliation pass mb_reconcile_alt), cases (a) and (c). +# +# chrP is a single 800bp unique (non-repetitive) sequence (LCG PRNG, seed 7) so +# the ONLY competitor for any read is its ALT twin, never a paralogous primary +# locus. Two well-separated regions, each with exactly ONE ALT twin. Crucially +# each ALT is a SHORT contig covering only its own region (not a full-length +# copy of chrP), so the region-A twin cannot also compete in region D. +# +# Region A chrP[200,350) -- read r-clean. Twin chrP_altC = chrP[150,400) +# IDENTICAL (250bp), aligned to chrP at POS=151 (250M). The read +# maps equally well to chrP and chrP_altC: WITHOUT reconciliation the +# twin is a co-scoring competitor => MAPQ 0; WITH it they group => +# chrP is sam_pri, MAPQ>0, chrP_altC secondary (case a). +# +# Region D chrP[500,620) -- read r-diverge. Twin chrP_altD = chrP[450,670) +# (220bp) with ONE SNP inside [500,620), aligned to chrP at POS=451 +# (220M). The read scores STRICTLY LOWER on chrP_altD (AS ~230 vs +# the primary's 240) yet still lifts to the SAME locus and survives +# the score-ratio prune (Task 3 guard) -- one SNP costs score without +# fragmenting every seed (2+ closely-spaced SNPs would kill seeding +# on the short ALT contig and the twin would never reach this pass). +# The lower-scoring twin is a real same-group member and must NOT +# zero the MAPQ (case c). +set -eu +d="$1"; mkdir -p "$d" + +# 800bp unique sequence (LCG PRNG, seed 7). +S=$(python3 -c " +import random +random.seed(7) +print(''.join(random.choice('ACGT') for _ in range(800))) +") + +# Region-A twin: identical 250bp window chrP[150,400) (1-based 151..400). +ALTC=$(printf '%s' "$S" | cut -c151-400) +LC=${#ALTC} + +# Region-D twin: 220bp window chrP[450,670) (1-based 451..670) with ONE SNP inside +# the r-diverge window chrP[500,620). Window-local 0-based offset 90 maps to +# chrP 540 (read position 40 of 120) -- one mismatch, AS ~230 < the primary's 240. +ALTD=$(printf '%s' "$S" | cut -c451-670 | python3 -c " +import sys +s=list(sys.stdin.read().strip()) +flip={'A':'T','C':'G','G':'C','T':'A','a':'t','c':'g','g':'c','t':'a'} +s[90] = flip[s[90]] # window-local 0-based -> chrP 540 (inside [500,620)) +print(''.join(s)) +") +LD=${#ALTD} + +# --- reference --- +printf '>chrP\n%s\n' "$S" > "$d/ref.fa" +printf '>chrP_altC\n%s\n' "$ALTC">> "$d/ref.fa" +printf '>chrP_altD\n%s\n' "$ALTD">> "$d/ref.fa" + +# --- .alt: each ALT aligns full-length forward to chrP at its window offset --- +printf 'chrP_altC\t0\tchrP\t151\t60\t%dM\t*\t0\t0\t*\t*\n' "$LC" > "$d/ref.fa.alt" +printf 'chrP_altD\t0\tchrP\t451\t60\t%dM\t*\t0\t0\t*\t*\n' "$LD" >> "$d/ref.fa.alt" + +# --- reads --- +QUAL150=$(printf '%150s' '' | tr ' ' 'I') +QUAL120=$(printf '%120s' '' | tr ' ' 'I') +# r-clean: chrP[200,350) (1-based 201..350) -- region A, identical twin. +RCLEAN=$(printf '%s' "$S" | cut -c201-350) +printf '@r-clean\n%s\n+\n%s\n' "$RCLEAN" "$QUAL150" > "$d/reads.fq" +# r-diverge: chrP[500,620) (1-based 501..620) -- region D, divergent twin. +RDIV=$(printf '%s' "$S" | cut -c501-620) +printf '@r-diverge\n%s\n+\n%s\n' "$RDIV" "$QUAL120" >> "$d/reads.fq" + +echo "fixture(reconcile): chrP=800 altC=identical chrP[150,400) altD=2SNP chrP[450,670)" >&2 diff --git a/test/altlg/mkfixture-revmulti.sh b/test/altlg/mkfixture-revmulti.sh new file mode 100644 index 0000000..822730e --- /dev/null +++ b/test/altlg/mkfixture-revmulti.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Fixture for the reverse-multi-block and multi-record/hard-clip .alt lift paths +# (regression for the l2b_set_alt / l2b_lift coordinate bugs). Every OTHER +# fixture emits a single full-length M block per ALT contig, which is exactly the +# coincidentally-correct regime; these two contigs are the cases that regime hid: +# +# chrPrev_alt (150bp) : reverse record (FLAG 16) with an internal deletion, +# CIGAR 75M2000D75M @ chrP POS 201. A reverse record's +# CIGAR walks the reverse-complement of the ALT contig, so +# the block coordinates need an RC->forward remap; before +# the fix the two blocks were swapped. +# chrPhc_alt (100bp) : two records for one ALT contig — a primary 50M @ POS +# 1001 (FLAG 0) and a supplementary 50H50M @ POS 2001 +# (FLAG 2048). The leading hard clip is the offset into +# the ALT contig; before the fix H consumed nothing and +# the array was left unsorted, so the supplementary span +# lifted to the wrong place (or read as a hole). +# +# Emits ref.fa + ref.fa.alt only; the test drives ex-lift-check in check mode. +set -eu +. "$(dirname "$0")/fixlib.sh" +d="$1"; mkdir -p "$d" + +# Sequence content is irrelevant to liftover (pure coordinate arithmetic from the +# .alt CIGARs); gen comes from fixlib.sh. +{ + printf '>chrP\n%s\n' "$(gen 71 3000)" + printf '>chrPrev_alt\n%s\n' "$(gen 72 150)" + printf '>chrPhc_alt\n%s\n' "$(gen 73 100)" +} > "$d/ref.fa" + +{ + printf 'chrPrev_alt\t16\tchrP\t201\t60\t75M2000D75M\t*\t0\t0\t*\t*\n' + printf 'chrPhc_alt\t0\tchrP\t1001\t60\t50M\t*\t0\t0\t*\t*\n' + printf 'chrPhc_alt\t2048\tchrP\t2001\t60\t50H50M\t*\t0\t0\t*\t*\n' +} > "$d/ref.fa.alt" diff --git a/test/altlg/mkfixture-segdup-rev.sh b/test/altlg/mkfixture-segdup-rev.sh new file mode 100755 index 0000000..8f29a67 --- /dev/null +++ b/test/altlg/mkfixture-segdup-rev.sh @@ -0,0 +1,88 @@ +#!/bin/sh +# Fixture for Task 7 fix round 1: REVERSE-strand ALT-seed -> primary projection. +# +# Same segdup-recovery mechanism as mkfixture-segdup.sh, but the ALT contig is +# REVERSE-COMPLEMENT aligned to the primary (.alt FLAG 0x10). This exercises the +# reverse code paths in mb_anchor_project_alt that the forward-only segdup fixture +# never touched: +# * Fix 1: recover the ALT-contig forward span for a reverse-aligned block. +# * Fix 2: emit the projected primary tpos in the FORWARD contig frame (do NOT +# re-fold for the projected strand) so the lifted POS is correct. +# +# Reference layout on chrP (primary) -- mirrors the forward segdup fixture but the +# shared core is stored REVERSE-COMPLEMENTED on the primary: +# [PAD0(200)] [RC(CORE)(100) + TAIL1(50)] <- LIFT locus (1-based POS 201) +# [PADi(70)] [RC(CORE)(100) + TAILi(50)] x (N-1) <- segdup decoy copies +# [PADZ(200)] +# +# The primary stores RC(CORE). The read (= ALT contig) is CORE+ALTTAIL forward; +# its CORE matches the primary copies only on the REVERSE strand. Each copy has +# a UNIQUE 50bp tail (TAIL1, TAILi, ...) so the read's full 150bp is NOT present +# on the primary -- only the 100bp CORE is shared/repetitive. With N primary +# copies plus the ALT, that CORE seed's SA interval is size N+1; `-c ` +# with N+1 > max_occ subsamples it and drops the LIFT copy's own seed. +# +# ALT twin: +# chrP_altS = CORE + ALTTAIL (150bp), forward. Because ALTTAIL is unique to the +# ALT contig, the read's full 150bp SMEM is UNIQUE to chrP_altS (SA size 1) -> the +# ALT seed is NEVER subsampled and always produces an ALT anchor. +# .alt maps chrP_altS onto chrP at the LIFT locus with FLAG 0x10 (RC), full +# length (150M). (Bases past CORE mismatch the primary's TAIL1; that is fine -- +# .alt alignments carry mismatches and the projected anchor is only a SEED.) +# +# Reverse-lift coordinate check (what Fix 1+2 must reproduce): +# The .alt block is alt[0,150) -> primary[LIFT0, LIFT0+150), rev=1, where +# LIFT0 = LIFT_POS1-1 (0-based). Under l2b_lift's reverse rule +# (pri = pri_en-1-(alt-alt_st)) the surviving full-read seed's forward ALT span +# alt[0..149] lifts to primary span [LIFT0, LIFT0+149]; its FORWARD last base is +# LIFT0+149, so mb_hit_set_coor's ts = (LIFT0+149)+1-150 = LIFT0 -> 1-based +# POS = LIFT_POS1, on the REVERSE strand (FLAG&0x10). +# +# Expected projected primary record (asserted in test-segdup-rev.sh): +# a REVERSE-strand chrP record at POS == LIFT_POS1. +# BEFORE Fix 1+2 the projected POS is wrong (mirror-image / dropped); +# AFTER, the reverse-strand chrP record appears at POS LIFT_POS1. +# +# Usage: mkfixture-segdup-rev.sh [N-copies] +set -eu +. "$(dirname "$0")/fixlib.sh" +d="$1"; mkdir -p "$d" +N="${2:-20}" + + +CORE=$(gen 100 100) # 100bp shared seed core (segdup): on every copy + ALT +ALTTAIL=$(gen 777 50) # 50bp tail UNIQUE to the ALT contig (makes ALT SMEM unique) +READ="${CORE}${ALTTAIL}" # 150bp read == ALT contig sequence (forward) +LR=${#READ} +LC=${#CORE} +RCCORE=$(rc "$CORE") # reverse complement of CORE: what the primary stores + +# chrP: PAD0 + (LIFT copy: RC(CORE)+TAIL1) + decoys + PADZ +PAD0=$(gen 1 200) +TAIL1=$(gen 4001 50) # the LIFT copy's own tail (differs from ALTTAIL) +CHRP="${PAD0}${RCCORE}${TAIL1}" +LIFT_OFF=${#PAD0} +LIFT_POS1=$(( LIFT_OFF + 1 )) # 1-based POS where the .alt block (and lifted seed) lands + +i=2 +while [ "$i" -le "$N" ]; do + PADI=$(gen $(( 2000 + i )) 70) + TAILI=$(gen $(( 4000 + i )) 50) + CHRP="${CHRP}${PADI}${RCCORE}${TAILI}" + i=$(( i + 1 )) +done +PADZ=$(gen 5 200) +CHRP="${CHRP}${PADZ}" + +printf '>chrP\n%s\n' "$CHRP" > "$d/ref.fa" +printf '>chrP_altS\n%s\n' "$READ" >> "$d/ref.fa" + +# .alt: chrP_altS -> chrP at the LIFT locus, RC (FLAG 0x10), full length (150M). +printf 'chrP_altS\t16\tchrP\t%d\t60\t%dM\t*\t0\t0\t*\t*\n' "$LIFT_POS1" "$LR" \ + > "$d/ref.fa.alt" + +QUAL=$(printf "%${LR}s" '' | tr ' ' 'I') +printf '@r-segdup-rev\n%s\n+\n%s\n' "$READ" "$QUAL" > "$d/reads.fq" + +echo "LIFT_POS1=$LIFT_POS1" > "$d/meta.txt" +echo "fixture(segdup-rev): chrP len=${#CHRP} N=$N core=$LC LIFT@${LIFT_POS1} RC ALT=chrP_altS(150)" >&2 diff --git a/test/altlg/mkfixture-segdup.sh b/test/altlg/mkfixture-segdup.sh new file mode 100755 index 0000000..354d651 --- /dev/null +++ b/test/altlg/mkfixture-segdup.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# Fixture for Task 7 (ALT-seed -> primary anchor projection; segdup recovery). +# +# Demonstrates the core failure mode: a SEGDUPLICATED primary locus whose own +# seed is dropped by max_occ subsampling, so the ONLY way a primary candidate +# exists at that locus is by PROJECTING the surviving ALT-contig seed back onto +# the primary assembly (DRAGEN's mechanism). +# +# Reference layout on chrP (primary): +# [PAD0(200)] [CORE(100) + TAIL1(50)] <- LIFT locus (1-based POS 201) +# [PADi(70)] [CORE(100) + TAILi(50)] x (N-1) <- segdup decoy copies +# [PADZ(200)] +# +# * CORE (100bp) is byte-identical across the LIFT copy and every decoy copy +# AND the ALT contig -> they all share the same 100bp seed. With N copies on +# the primary plus the ALT, that seed's SA interval has size N+1; run minibwa +# with `-c ` small enough (N+1 > max_occ) that the interval is +# SUBSAMPLED. The strided sampler drops the LIFT copy's own seed position. +# * Each copy has a UNIQUE 50bp tail (TAIL1, TAILi, ...) so the only shared, +# repetitively-seeded region is CORE. +# +# ALT twin: +# chrP_altS = CORE + ALTTAIL (150bp). Because ALTTAIL is unique to the ALT +# contig, the read's full 150bp SMEM is UNIQUE to chrP_altS (SA size 1) -> the +# ALT seed is NEVER subsampled and always produces an ALT anchor. +# .alt maps chrP_altS onto chrP at the LIFT locus, full length (150M). (The +# tail bases mismatch the primary's own TAIL1; that is fine -- .alt alignments +# carry mismatches, and the projected anchor is only a SEED: DP re-aligns it.) +# +# Read = chrP_altS sequence (CORE + ALTTAIL, 150bp). +# +# Behaviour (asserted in test-segdup.sh, with `-c 4 -N 20`): +# WITHOUT projection: the LIFT-locus primary candidate is absent (its CORE seed +# was subsampled out) -> NO chrP record at POS 201. +# WITH projection: the surviving ALT seed lifts to the LIFT locus and injects a +# primary anchor there -> a chrP record at POS 201 appears. +# +# Usage: mkfixture-segdup.sh [N-copies] +set -eu +. "$(dirname "$0")/fixlib.sh" +d="$1"; mkdir -p "$d" +N="${2:-20}" # number of primary copies of CORE (>= the max_occ used in tests) + + +CORE=$(gen 100 100) # 100bp shared seed core (segdup): on every copy + ALT +ALTTAIL=$(gen 777 50) # 50bp tail UNIQUE to the ALT contig (makes ALT SMEM unique) +READ="${CORE}${ALTTAIL}" # 150bp read == ALT contig sequence +LR=${#READ} +LC=${#CORE} + +# chrP: PAD0 + (LIFT copy: CORE+TAIL1) + (decoy copies: PADi + CORE + TAILi) + PADZ +PAD0=$(gen 1 200) +TAIL1=$(gen 4001 50) # the LIFT copy's own tail (differs from ALTTAIL) +CHRP="${PAD0}${CORE}${TAIL1}" +LIFT_OFF=${#PAD0} +LIFT_POS1=$(( LIFT_OFF + 1 )) # 1-based POS of the LIFT locus (where .alt maps) + +i=2 +while [ "$i" -le "$N" ]; do + PADI=$(gen $(( 2000 + i )) 70) + TAILI=$(gen $(( 4000 + i )) 50) + CHRP="${CHRP}${PADI}${CORE}${TAILI}" + i=$(( i + 1 )) +done +PADZ=$(gen 5 200) +CHRP="${CHRP}${PADZ}" + +printf '>chrP\n%s\n' "$CHRP" > "$d/ref.fa" +printf '>chrP_altS\n%s\n' "$READ" >> "$d/ref.fa" + +# .alt: chrP_altS -> chrP at the LIFT locus, full length (150M). +printf 'chrP_altS\t0\tchrP\t%d\t60\t%dM\t*\t0\t0\t*\t*\n' "$LIFT_POS1" "$LR" \ + > "$d/ref.fa.alt" + +QUAL=$(printf "%${LR}s" '' | tr ' ' 'I') +printf '@r-segdup\n%s\n+\n%s\n' "$READ" "$QUAL" > "$d/reads.fq" + +echo "LIFT_POS1=$LIFT_POS1" > "$d/meta.txt" +echo "fixture(segdup): chrP len=${#CHRP} N=$N core=$LC LIFT@${LIFT_POS1} ALT=chrP_altS(150,unique tail)" >&2 diff --git a/test/altlg/mkfixture-survive.sh b/test/altlg/mkfixture-survive.sh new file mode 100755 index 0000000..ae20f9e --- /dev/null +++ b/test/altlg/mkfixture-survive.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# Fixture for Task 3 (survival guard in mb_select_sub). +# +# Reference: +# chrP 400bp unique (non-repetitive) sequence generated by LCG. +# chrP_alt ALT = chrP with a 1bp SNP at 1-based position 126 (0-based 125). +# .alt CIGAR: 400M (full length alignment, 1 mismatch at 125). +# +# Read "r-survive" (100bp, chrP[70,170), 1-based 71..170): +# The SNP is at read position 55 (0-based: read[55] = chrP[125]). +# +# Seeding / chaining behaviour with the unique sequence: +# - chrP gets one 100bp anchor (qs=0, ts=70): chain score ~100. +# - chrP_alt gets one ~55bp sub-SMEM anchor (qs=0, ts=70) because the +# interval with both chrP and chrP_alt contracts at length 56 (the SNP +# position). Chain score ~55. +# +# mb_select_sub with -p 0.9 (pri_ratio=0.9), default min_diff=2*min_len=38: +# keep if: score >= parent * 0.9 OR score + 38 >= parent +# parent_score = 100: +# 55 >= 90 -> FALSE +# 55 + 38 = 93 >= 100 -> FALSE +# => chrP_alt twin DROPPED without the guard. +# +# After the guard: chrP_alt is force-kept because its lifted placement +# (lifted to chrP ts~70) co-locates with the kept chrP primary. +set -eu +d="$1"; mkdir -p "$d" + +# Build a 400bp unique sequence (LCG PRNG, seed 42) +S=$(python3 -c " +import random +random.seed(42) +n='ACGT' +print(''.join(random.choice(n) for _ in range(400))) +") +L=${#S} + +# SNP at 1-based position 126 (0-based 125) +# This puts the SNP at read position 55 (read starts at chrP[70,170), so +# SNP position = 125 - 70 = 55 within the read). +C=$(printf '%s' "$S" | cut -c126) +FLIP=$(printf '%s' "$C" | tr 'ACGTacgt' 'TGCAtgca') +PREFIX=$(printf '%s' "$S" | cut -c1-125) +SUFFIX=$(printf '%s' "$S" | cut -c127-) +ALT="${PREFIX}${FLIP}${SUFFIX}" + +# --- reference --- +printf '>chrP\n%s\n' "$S" > "$d/ref.fa" +printf '>chrP_alt\n%s\n' "$ALT" >> "$d/ref.fa" + +# --- .alt (400M: full-length alignment, 1 mismatch at 0-based position 125) --- +printf 'chrP_alt\t0\tchrP\t1\t60\t%dM\t*\t0\t0\t*\t*\n' "$L" > "$d/ref.fa.alt" + +# --- read: chrP[70,170) 1-based 71..170 (100bp) --- +READ=$(printf '%s' "$S" | cut -c71-170) +QUAL=$(printf '%100s' '' | tr ' ' 'I') +printf '@r-survive\n%s\n+\n%s\n' "$READ" "$QUAL" > "$d/reads.fq" + +echo "fixture: L=$L chrP_alt(SNP@125 $C->$FLIP) read=chrP[70,170) SNP at read-pos 55" >&2 diff --git a/test/altlg/mkfixture-svbreak.sh b/test/altlg/mkfixture-svbreak.sh new file mode 100755 index 0000000..46462d6 --- /dev/null +++ b/test/altlg/mkfixture-svbreak.sh @@ -0,0 +1,90 @@ +#!/bin/sh +# Fixture for SV-breakpoint-aware ALT grouping (the W2.2 hi_vs_z gap). +# +# An ALT contig whose .alt CIGAR carries an SV-scale indel describes a structural +# alignment in which two M blocks lift to primary positions thousands of bp apart. +# A read whose ALT footprint spans BOTH blocks (across the breakpoint) lifts to a +# placement whose min/max-COLLAPSED [lifted_st, lifted_en] is dragged thousands of +# bp from where the read's PRIMARY twin actually sits. Under the old single- +# interval grouping that ALT hit fails to co-locate with its primary twin -> they +# compete -> a FALSE MAPQ 0. The multi-interval placement exposes a sub-interval +# AT the twin's position, so they group and the read keeps its confident MAPQ. +# +# Construction (forward and reverse variants): +# ALT contig chrP_altV = LEFT(75) + RIGHT(75) = the 150bp read. The read maps +# 150M to chrP_altV (unique to that contig). +# +# .alt CIGAR = 75M 2000D 75M (a 2000bp DELETION from the ALT, i.e. an SV-scale +# indel): the LEFT block lifts to primary [200,275); the RIGHT block lifts to +# primary [2275,2350) -- 2000bp away. The read's full footprint spans both +# blocks, so the OLD collapse yields lifted_st = 200 (the min). +# +# The PRIMARY TWIN (LEFT+RIGHT contiguous, == the read) is placed at primary +# offset 2275 -- exactly the RIGHT block's lifted position. (.alt lifting is +# purely coordinate-based, so the twin's bases need not equal the RIGHT block's +# declared bases.) The read maps 150M to the twin at 1-based POS 2276. +# +# OLD behavior (RED): ALT collapsed lifted_st = 200; twin lifted_st = 2275; +# |Δ| = 2075 >> MB_LIFT_TOL=10 -> no co-location -> compete +# -> primary MAPQ 0. +# NEW behavior (GREEN): ALT sub-placements = {200 (LEFT), 2275 (RIGHT)}; the 2275 +# sub-interval co-locates with the twin -> they group -> +# the primary twin is sam_pri with MAPQ>0, the ALT secondary. +# +# The forward variant uses .alt FLAG 0 (read maps forward to the twin); the +# reverse variant uses .alt FLAG 0x10 and a reverse-complemented twin (read maps +# reverse to the twin), exercising the reverse-fold lift paths. +# +# Usage: mkfixture-svbreak.sh +set -eu +. "$(dirname "$0")/fixlib.sh" +d="$1"; mkdir -p "$d" + + +A=75 # LEFT block length +B=75 # RIGHT block length +N=2000 # SV-scale deletion from the ALT (the breakpoint gap on primary) + +LEFT=$(gen 11 "$A") +RIGHT=$(gen 22 "$B") +READ="${LEFT}${RIGHT}" # 150bp read == ALT contig (forward) +LR=${#READ} + +# .alt LEFT block lifts to primary 0-based 200 (1-based POS 201); RIGHT block to +# 0-based 200+A+N. Place the read twin (READ, contiguous) at that RIGHT position. +PAD0=$(gen 1 200) +P0=${#PAD0} # 200 (0-based start of the LEFT block) +TWIN0=$(( P0 + A + N )) # 0-based start of the RIGHT block == twin start +ALT_POS1=$(( P0 + 1 )) # 1-based POS the .alt block aligns LEFT at +TWIN_POS1=$(( TWIN0 + 1 )) # 1-based POS the read twin maps at +JUNK_LEN=$(( TWIN0 - P0 )) # filler between PAD0 and the twin +JUNK=$(gen 99 "$JUNK_LEN") +PADZ=$(gen 5 200) + +QUAL=$(printf "%${LR}s" '' | tr ' ' 'I') + +# ---- forward variant ---- +# chrP_fwd = PAD0 + JUNK + READ(twin, fwd) + PADZ ; .alt FLAG 0. +CHRP_FWD="${PAD0}${JUNK}${READ}${PADZ}" +printf '>chrP\n%s\n' "$CHRP_FWD" > "$d/ref.fa" +printf '>chrP_altV\n%s\n' "$READ" >> "$d/ref.fa" +printf 'chrP_altV\t0\tchrP\t%d\t60\t%dM%dD%dM\t*\t0\t0\t*\t*\n' \ + "$ALT_POS1" "$A" "$N" "$B" > "$d/ref.fa.alt" +printf '@r-sv-fwd\n%s\n+\n%s\n' "$READ" "$QUAL" > "$d/reads.fq" + +# ---- reverse variant (separate index dir) ---- +# chrP_rev = PAD0 + JUNK + rc(READ) (twin maps reverse) + PADZ ; .alt FLAG 0x10. +mkdir -p "$d/rev" +RCREAD=$(rc "$READ") +CHRP_REV="${PAD0}${JUNK}${RCREAD}${PADZ}" +printf '>chrP\n%s\n' "$CHRP_REV" > "$d/rev/ref.fa" +printf '>chrP_altV\n%s\n' "$READ" >> "$d/rev/ref.fa" +printf 'chrP_altV\t16\tchrP\t%d\t60\t%dM%dD%dM\t*\t0\t0\t*\t*\n' \ + "$ALT_POS1" "$A" "$N" "$B" > "$d/rev/ref.fa.alt" +printf '@r-sv-rev\n%s\n+\n%s\n' "$READ" "$QUAL" > "$d/rev/reads.fq" + +{ + echo "ALT_POS1=$ALT_POS1" + echo "TWIN_POS1=$TWIN_POS1" +} > "$d/meta.txt" +echo "fixture(svbreak): .alt=${A}M${N}D${B}M ALT block@${ALT_POS1} twin@${TWIN_POS1} (Δ$(( N + A )) >> MB_LIFT_TOL)" >&2 diff --git a/test/altlg/mkfixture.sh b/test/altlg/mkfixture.sh new file mode 100755 index 0000000..b07d897 --- /dev/null +++ b/test/altlg/mkfixture.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Synthetic ref: chrP (primary) + chrP_alt (ALT, = chrP with a 5bp insertion at +# offset 300, so the .alt CIGAR is 300M5IM). +# Emits ref.fa, ref.fa.alt (SAM: chrP_alt aligned to chrP), and reads. +set -eu +d="$1"; mkdir -p "$d" +S=$(printf '%s' 'GATCCTAGCATGCTAGGCTAACGTTAGCCGATCGTAGCTAGGCATCGATCGTAGCTAGCTAGGCATCGATTACGATCGGCTAATCGATCGTAGCTGATCGA'\ +'TCGTAGCTAGCATCGATCGTAGCATCGGCTAGCATCGATCGATTACGCATCGATCGTAGGCTAGCATCGATCGTAGCTAGCATCGGCTAGCATCGATTACG'\ +'ATCGGCTAATCGATCGTAGCTGATCGATCGTAGCTAGCATCGATCGTAGCATCGGCTAGCATCGATCGATTACGCATCGATCGTAGGCTAGCATCGATCGT'\ +'AGCTAGCATCGGCTAGCATCGATTACGATCGGCTAATCGATCGTAGCTGATCGAGATCCTAGCATGCTAGGCTAACGTTAGCCGATCGTAGCTAGGCATCG'\ +'ATCGTAGCTAGCTAGGCATCGATTACGATCGGCTAATCGATCGTAGCTGATCGATCGTAGCTAGCATCGATCGTAGCATCGGCTAGCATCGATCGATTACG') +printf '>chrP\n%s\n' "$S" > "$d/ref.fa" +ALT=$(printf '%s' "$S" | mawk '{print substr($0,1,300) "GGGGG" substr($0,301)}') # 5bp insertion at 300 +printf '>chrP_alt\n%s\n' "$ALT" >> "$d/ref.fa" +# .alt: chrP_alt (query) aligned to chrP (ref): 300M5IM ; POS=1, MAPQ 60 +rest=$(( ${#S} - 300 )) +printf 'chrP_alt\t0\tchrP\t1\t60\t300M5I%dM\t*\t0\t0\t*\t*\n' "$rest" > "$d/ref.fa.alt" +# read fully inside the shared flank (offset 120, 150bp) -> lifts cleanly +printf '@r1\n%s\n+\n%s\n' "$(printf '%s' "$S" | cut -c121-270)" "$(printf '%150s' '' | tr ' ' 'I')" > "$d/r1.fq" diff --git a/test/altlg/test-altalt.sh b/test/altlg/test-altalt.sh new file mode 100644 index 0000000..d6f8024 --- /dev/null +++ b/test/altlg/test-altalt.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# Integration test for the ALT-ALTERNATE fold-in (mb_reconcile_alt step 2b). +# +# An ALT twin that the per-base lift cannot co-locate with its primary hit -- +# because an .alt-internal structural indel DISPLACES it beyond MB_LIFT_TOL +# (r-disp) or drops it into an insertion HOLE (r-hole) -- must still be folded +# into the primary's group, recognized as an alternate placement of that locus +# via the .alt correspondence. After the fold: +# - the SAM primary is the non-ALT chrP hit (read NOT placed on the ALT contig); +# - the primary MAPQ is > 0 (the ALT twin no longer dilutes it); +# - the ALT twin is secondary (0x100) or dropped, and at the grouping level is +# a subordinate (parent != its own id). +# +# Baseline: chrM (no .alt) => mb_any_alt gate off => reconcile inert => output +# byte-identical across runs. +# +# Usage: test/altlg/test-altalt.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" + +# Shared no-.alt baseline check (see lib-baseline.sh). +. "$(dirname "$0")/lib-baseline.sh" +EX_GROUP="$MDIR/api-test/ex-group-check" +MK="$MDIR/test/altlg/mkfixture-altalt.sh" + +TMPD=$(mktemp -d /tmp/altlg-altalt.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# primary record (no 0x100, no 0x800) fields for a qname +primary_rname() { mawk -v q="$2" '$1==q{f=int($2); if(int(f/256)%2==0 && int(f/2048)%2==0){print $3; exit}}' "$1"; } + +# group-check field for the ALT contig hit of a qname (is_alt=1 row on ) +grp_field() { + mawk -v q="$2" -v c="$3" -v k="$4" ' + $1==q { ctg=""; want=""; for(i=1;i<=NF;i++){ n=index($i,"="); + if(n>0){ key=substr($i,1,n-1); val=substr($i,n+1); + if(key=="ctg") ctg=val; if(key==k) want=val; } } + if(ctg==c){ print want; exit } }' "$1" +} + +echo "[test-altalt] building fixture ..." +/bin/sh "$MK" "$TMPD" 2>/dev/null +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +echo "[test-altalt] mapping (--outn=50) ..." +"$MINIBWA" map --outn=50 "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/full.sam" +mawk '$1 !~ /^@/' "$TMPD/full.sam" > "$TMPD/aln.sam" +[ -s "$TMPD/aln.sam" ] || fail "no alignments emitted" +echo "----- altalt SAM -----" +mawk '{printf "%s flag=%d %s pos=%s mapq=%s\n",$1,$2,$3,$4,$5}' "$TMPD/aln.sam" +echo "----------------------" + +[ -x "$EX_GROUP" ] || fail "ex-group-check probe not built ($EX_GROUP); run 'make -C api-test'" +"$EX_GROUP" "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/grp.txt" +echo "----- altalt grouping -----" +cat "$TMPD/grp.txt" +echo "---------------------------" + +check_read() { # + q="$1"; altc="$2" + pr=$(primary_rname "$TMPD/aln.sam" "$q") + pf=$(primary_flag "$TMPD/aln.sam" "$q") + pq=$(primary_mapq "$TMPD/aln.sam" "$q") + [ -n "$pr" ] || fail "($q) no primary record" + [ "$pr" = "chrP" ] || fail "($q) SAM primary on '$pr'; expected chrP (ALT twin stole the primary slot)" + ok "($q) SAM primary is chrP (flag=$pf)" + [ "$pq" -gt 0 ] || fail "($q) primary MAPQ=$pq; expected >0 (ALT twin must not dilute it)" + ok "($q) primary MAPQ=$pq > 0" + af=$(flag_of "$TMPD/aln.sam" "$q" "$altc") + if [ -n "$af" ]; then + s=$(has_bit "$af" 256) + [ "$s" = "1" ] || fail "($q) $altc present but not secondary (flag=$af)" + ok "($q) $altc twin is secondary (flag=$af)" + else + ok "($q) $altc twin absent (collapsed into primary group)" + fi + # grouping level: the ALT hit must be a subordinate (parent != its own id) + gid=$(grp_field "$TMPD/grp.txt" "$q" "$altc" id) + gpa=$(grp_field "$TMPD/grp.txt" "$q" "$altc" parent) + [ -n "$gid" ] && [ -n "$gpa" ] || fail "($q) no $altc hit in grouping dump" + [ "$gpa" != "$gid" ] || fail "($q) $altc is its own group rep (parent=$gpa == id=$gid); ALT alternate must be folded into the primary group" + ok "($q) $altc folded as subordinate (parent=$gpa != id=$gid)" +} + +echo "== r-disp: deletion-displaced ALT twin folds into chrP ==" +check_read r-disp chrP_altF +echo "== r-hole: insertion-hole (unliftable) ALT twin folds into chrP ==" +check_read r-hole chrP_altG + +echo "[test-altalt] chrM baseline (no .alt): byte-identical to stock ..." +chrm_baseline "(BASELINE) chrM" se --outn=5 + +echo "[test-altalt] PASS" +exit 0 diff --git a/test/altlg/test-badalt.sh b/test/altlg/test-badalt.sh new file mode 100755 index 0000000..5934263 --- /dev/null +++ b/test/altlg/test-badalt.sh @@ -0,0 +1,99 @@ +#!/bin/sh +# Malformed .alt records must be rejected, not half-loaded. +# +# l2b_set_alt() parses .alt lines out of a user-supplied SAM. Every coordinate it +# derives is attacker- or accident-controlled, and one of them is a SUBTRACTION: +# a reverse record's ALT span is remapped to forward as `ctg->len - alt_en`. If +# the record's CIGAR query span runs past the ALT contig it names -- a .alt built +# against a different reference revision is enough -- that wraps on uint64 to a +# near-UINT64_MAX coordinate, the block sorts to the end of lift[], and +# l2b_lift()'s binary search reads it as valid. The failure is silent and it +# inflates confidence: a garbage lift makes an ALT twin look co-located with its +# primary, and MAPQ goes UP. +# +# Verifies: +# 1. Over-long reverse record -> rejected (MAPQ not inflated by a garbage lift). +# 2. POS < 1 -> rejected. +# 3. Unknown CIGAR operator -> rejected. +# 4. A WELL-FORMED reverse record still loads -- the guards must reject only +# what is actually malformed, or they would silently disable ALT support. +# +# Usage: test/altlg/test-badalt.sh [] +set -eu +. "$(dirname "$0")/lib.sh" +. "$(dirname "$0")/fixlib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" + +TMPD=$(mktemp -d /tmp/altlg-badalt.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + +# chrP (400bp primary) + chrP_alt, the reverse complement of chrP[100,250). +# A reverse .alt record is the one that exercises the subtraction. +PRI=$(gen 7 400) +ALT=$(rc "$(printf '%s' "$PRI" | cut -c101-250)") +printf '>chrP\n%s\n>chrP_alt\n%s\n' "$PRI" "$ALT" > "$TMPD/ref.fa" +printf '@r1\n%s\n+\n%s\n' \ + "$(printf '%s' "$PRI" | cut -c121-220)" "$(printf '%100s' '' | tr ' ' 'I')" > "$TMPD/reads.fq" +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null || fail "indexing the fixture failed" + +# Run with the given .alt content and report the primary's MAPQ. +mapq_with() { + printf '%s' "$1" > "$TMPD/ref.fa.alt" + "$MINIBWA" map --outn=5 "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/out.sam" \ + || fail "minibwa map failed on .alt <$1>" + [ -s "$TMPD/out.sam" ] || fail "empty output for .alt <$1>" + primary_mapq "$TMPD/out.sam" +} + +# ============================================================ +# 4 first: establish that a well-formed reverse record DOES load. +# Without this the rejection checks below could all pass for the +# trivial reason that ALT support is broken outright. +# ============================================================ +echo "== well-formed reverse record (control) ==" +good=$(mapq_with 'chrP_alt 16 chrP 101 60 150M * 0 0 * * +') +[ "$good" = "60" ] \ + || fail "well-formed reverse .alt: primary MAPQ is '${good:-}', expected 60 -- ALT support is not working, so the rejection checks below would be vacuous" +ok "well-formed reverse record loads: primary MAPQ $good" + +# The same read with NO .alt at all: the baseline a rejected record must match. +rm -f "$TMPD/ref.fa.alt" +"$MINIBWA" map --outn=5 "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/none.sam" +none=$(primary_mapq "$TMPD/none.sam") +[ "$none" != "$good" ] \ + || fail "no-.alt MAPQ ($none) equals the loaded one ($good); this fixture cannot tell the two apart" +ok "no .alt at all: primary MAPQ $none (the baseline a rejected record must match)" + +# ============================================================ +# 1. Over-long reverse record: 300M on a 150bp ALT contig. +# ============================================================ +echo "== malformed records are rejected ==" +span=$(mapq_with 'chrP_alt 16 chrP 101 60 300M * 0 0 * * +') +[ "$span" = "$none" ] \ + || fail "over-long reverse record: primary MAPQ is '$span', expected '$none' -- the span was accepted and a wrapped coordinate lifted it" +ok "reverse record longer than its ALT contig is rejected (MAPQ $span, not inflated)" + +# ============================================================ +# 2. POS < 1 (a mapped record must be 1-based >= 1). +# ============================================================ +pos=$(mapq_with 'chrP_alt 16 chrP 0 60 150M * 0 0 * * +') +[ "$pos" = "$none" ] \ + || fail "POS=0 record: primary MAPQ is '$pos', expected '$none'" +ok "POS < 1 is rejected (MAPQ $pos)" + +# ============================================================ +# 3. Unknown CIGAR operator. +# ============================================================ +cig=$(mapq_with 'chrP_alt 16 chrP 101 60 100M50Z * 0 0 * * +') +[ "$cig" = "$none" ] \ + || fail "unknown CIGAR op: primary MAPQ is '$cig', expected '$none'" +ok "unknown CIGAR operator is rejected (MAPQ $cig)" + +echo "[test-badalt] PASS" +exit 0 diff --git a/test/altlg/test-hitplace.sh b/test/altlg/test-hitplace.sh new file mode 100755 index 0000000..ae9911b --- /dev/null +++ b/test/altlg/test-hitplace.sh @@ -0,0 +1,120 @@ +#!/bin/sh +# Integration test for mb_hit_place (Task 2: per-hit lifted PLACEMENT). +# +# Builds the placement fixture (chrP primary; chrP_alt forward ALT with a 5bp +# insertion @300; chrR_alt reverse-strand ALT = revcomp of chrP[250,450)), +# indexes it, maps the reads, and asserts on the placement printed by +# api-test/ex-place-check. Each ex-place-check line looks like: +# +# ts=.. te=.. rev=.. is_alt=.. cg=.. || place: pri= \ +# lst= len= prev= liftable=<0|1> +# +# Cases (c) and (d) are the BLOCKER branches: a reverse .alt block (strand fold + +# min/max over lifted outputs) and a footprint end sitting in a hole (walk to the +# first liftable base). A naive lifted_st = lift(alt_st) FAILS both. +# +# Usage: test/altlg/test-hitplace.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MKFIXTURE="$MDIR/test/altlg/mkfixture-place.sh" +MINIBWA="$MDIR/minibwa" +EX_PLACE="$MDIR/api-test/ex-place-check" + +TMPD=$(mktemp -d /tmp/altlg-hitplace.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + +echo "[test-hitplace] building fixture in $TMPD ..." +/bin/sh "$MKFIXTURE" "$TMPD" + +echo "[test-hitplace] indexing with minibwa ..." +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +echo "[test-hitplace] running ex-place-check (exact, post-DP) ..." +"$EX_PLACE" "$TMPD/ref.fa" "$TMPD/reads.fq" all 2>/dev/null > "$TMPD/place.txt" +echo "[test-hitplace] running ex-place-check (coarse, h->p forced NULL) ..." +"$EX_PLACE" "$TMPD/ref.fa" "$TMPD/reads.fq" coarse 2>/dev/null > "$TMPD/place-coarse.txt" + +echo "----- exact placement -----" +cat "$TMPD/place.txt" +echo "----- coarse placement -----" +cat "$TMPD/place-coarse.txt" + + +# field extractor: print value of key= on the line matching [rev=] +# usage: field (first matching hit) +# field_rev (matching hit on read-strand hrev) +field() { + mawk -v q="$2" -v c="$3" -v k="$4" ' + $1==q && $2==c { + for(i=1;i<=NF;i++){n=index($i,k"="); if(n==1){print substr($i,length(k)+2); exit}} + }' "$1" +} +field_rev() { + mawk -v q="$2" -v c="$3" -v hr="rev="$4 -v k="$5" ' + $1==q && $2==c { + ok=0; for(i=1;i<=NF;i++) if($i==hr) ok=1; + if(ok){ for(i=1;i<=NF;i++){n=index($i,k"="); if(n==1){print substr($i,length(k)+2); exit}} } + }' "$1" +} + +eq() { # eq + if [ "$2" != "$3" ]; then fail "$1: expected '$3', got '$2'"; fi + echo " ok: $1 = $2" +} +within() { # within + d=$(( $2 - $3 )); [ "$d" -lt 0 ] && d=$(( -d )) + if [ "$d" -gt "$4" ]; then fail "$1: |$2 - $3| = $d > tol $4"; fi + echo " ok: $1 = $2 (within $4 of $3)" +} + +P="$TMPD/place.txt" +C="$TMPD/place-coarse.txt" + +echo "== case (b): primary chrP hit is identity passthrough ==" +eq "pa-fwd/chrP pri" "$(field "$P" pa-fwd chrP pri)" chrP +eq "pa-fwd/chrP lst==ts" "$(field "$P" pa-fwd chrP lst)" 120 +eq "pa-fwd/chrP len==te" "$(field "$P" pa-fwd chrP len)" 270 +eq "pa-fwd/chrP prev" "$(field "$P" pa-fwd chrP prev)" 0 +eq "pa-fwd/chrP liftable" "$(field "$P" pa-fwd chrP liftable)" 1 + +echo "== case (a): forward ALT hit lifts to the SAME primary footprint ==" +eq "pa-fwd/chrP_alt pri" "$(field "$P" pa-fwd chrP_alt pri)" chrP +eq "pa-fwd/chrP_alt prev" "$(field "$P" pa-fwd chrP_alt prev)" 0 +eq "pa-fwd/chrP_alt liftable" "$(field "$P" pa-fwd chrP_alt liftable)" 1 +within "pa-fwd/chrP_alt lst groups with chrP" "$(field "$P" pa-fwd chrP_alt lst)" "$(field "$P" pa-fwd chrP lst)" 10 + +echo "== case (c): reverse-strand ALT folds; groups with the forward primary ==" +# c-fwd: chrP hit (primary, read forward) vs chrR_alt hit (read on minus strand). +# block_rev(chrR_alt)=1 XOR h->rev(1) = 0 -> same folded strand as the chrP hit. +eq "c-fwd/chrP prev" "$(field_rev "$P" c-fwd chrP 0 prev)" 0 +eq "c-fwd/chrP lst" "$(field_rev "$P" c-fwd chrP 0 lst)" 300 +eq "c-fwd/chrR_alt pri" "$(field_rev "$P" c-fwd chrR_alt 1 pri)" chrP +eq "c-fwd/chrR_alt liftable" "$(field_rev "$P" c-fwd chrR_alt 1 liftable)" 1 +eq "c-fwd/chrR_alt folded rev" "$(field_rev "$P" c-fwd chrR_alt 1 prev)" 0 +within "c-fwd/chrR_alt lst groups with chrP (strand-folded)" \ + "$(field_rev "$P" c-fwd chrR_alt 1 lst)" "$(field_rev "$P" c-fwd chrP 0 lst)" 10 + +echo "== case (d): one footprint end in a hole; walk to first liftable base ==" +# hole-end on chrP_alt starts at ALT 300 (inside the 5I insertion -> a hole). +# Naive lift(alt_st=300) returns a hole -> liftable=0; correct impl walks to the +# first liftable base (ALT 305 -> chrP 300) so liftable=1 and lst==300. +eq "hole-end/chrP_alt pri" "$(field "$P" hole-end chrP_alt pri)" chrP +eq "hole-end/chrP_alt liftable" "$(field "$P" hole-end chrP_alt liftable)" 1 +eq "hole-end/chrP_alt prev" "$(field "$P" hole-end chrP_alt prev)" 0 +eq "hole-end/chrP_alt lst (first liftable base)" "$(field "$P" hole-end chrP_alt lst)" 300 + +echo "== twin with small (<=tol) read-vs-ALT indel still groups ==" +within "twin-indel chrP_alt groups with chrP primary" \ + "$(field "$P" twin-indel chrP_alt lst)" "$(field "$P" twin-indel chrP lst)" 10 + +echo "== coarse (pre-DP, h->p==NULL) agrees with exact on lifted_st (within tol) ==" +within "coarse hole-end/chrP_alt lst == exact" \ + "$(field "$C" hole-end chrP_alt lst)" "$(field "$P" hole-end chrP_alt lst)" 10 +eq "coarse hole-end/chrP_alt liftable" "$(field "$C" hole-end chrP_alt liftable)" 1 +within "coarse c-fwd/chrR_alt lst == exact (reverse fold)" \ + "$(field_rev "$C" c-fwd chrR_alt 1 lst)" "$(field_rev "$P" c-fwd chrR_alt 1 lst)" 10 + +echo "[test-hitplace] PASS" +exit 0 diff --git a/test/altlg/test-hookc.sh b/test/altlg/test-hookc.sh new file mode 100755 index 0000000..51adfef --- /dev/null +++ b/test/altlg/test-hookc.sh @@ -0,0 +1,145 @@ +#!/bin/sh +# Integration test for Hook C UNLIFTABLE-demotion path (pe.c mb_distinct_lifted_group). +# +# MECHANISM: +# mb_distinct_lifted_group() returns 0 when EITHER hit is unliftable (!pa.liftable +# || !pb.liftable), so it falls through to NORMAL chimeric demotion. Only two +# LIFTABLE hits that are genuinely distinct primary groups get the "continue" skip. +# An unliftable is_alt hit is NOT a protected distinct group -- shielding it would +# preserve a spurious competitor and wrongly affect MAPQ. +# +# FIXTURE (mkfixture-hookc.sh): +# chrHC 4000bp primary +# chrHC_altU 200bp ALT contig: +# bases [0,100) = chrHC[500,600) -- covered by .alt block (liftable) +# bases [100,200) = chrHC[700,800) -- NO .alt block -> LIFT HOLE (unliftable) +# .alt: chrHC_altU -> chrHC POS 501, CIGAR 100M (covers only alt[0,100)) +# +# R1 = chrHC[300,420) fwd (unique anchor) +# R2 = revcomp(chrHC[700,800)): +# - maps to chrHC@700 (primary, liftable=1) +# - maps to chrHC_altU@100 (ALT, footprint [100,200) in HOLE -> liftable=0) +# +# WHAT IS PROVEN: +# (1) ex-place-check confirms the chrHC_altU hit is unliftable (liftable=0) -- +# fails loudly if liftable=1 (wrong fixture; test proves nothing). +# (2) The unliftable ALT hit is demoted to secondary (0x100) -- NOT shielded. +# Would FAIL if mb_distinct_lifted_group returned 1 for unliftable hits, +# because the demotion "continue" would fire and the hit would stay as a +# supplementary (0x800) separate-group representative. +# (3) The primary chrHC R2 hit is a proper-pair primary with MAPQ > 0. +# (4) chrM PE baseline: no .alt -> byte-identical across runs. +# +# Usage: test/altlg/test-hookc.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" + +# Shared no-.alt baseline check (see lib-baseline.sh). +. "$(dirname "$0")/lib-baseline.sh" +MKFIXTURE="$MDIR/test/altlg/mkfixture-hookc.sh" +EX_PLACE="$MDIR/api-test/ex-place-check" + +TMPD=$(mktemp -d /tmp/altlg-hookc.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + + +# ========================================================================= +echo "[test-hookc] building fixture ..." +/bin/sh "$MKFIXTURE" "$TMPD/fix" 2>/dev/null + +echo "[test-hookc] indexing ..." +"$MINIBWA" index "$TMPD/fix/ref.fa" 2>/dev/null + +# ========================================================================= +# (1) PROVE the ALT hit is unliftable via ex-place-check. +# r2_probe.fq contains only R2 so ex-place-check can call mb_map on it. +echo "[test-hookc] running ex-place-check to prove unliftability ..." +"$EX_PLACE" "$TMPD/fix/ref.fa" "$TMPD/fix/r2_probe.fq" all 2>/dev/null \ + > "$TMPD/place.txt" +echo "----- ex-place-check output (R2 probe) -----" +cat "$TMPD/place.txt" +echo "---------------------------------------------" + +# Extract liftable flag for the chrHC_altU hit (is_alt=1 line). +alt_liftable=$(mawk '$2=="chrHC_altU"{ + for(i=1;i<=NF;i++){ + if(index($i,"liftable=")==1){print substr($i,10); exit} + }}' "$TMPD/place.txt") +[ -n "$alt_liftable" ] || fail "ex-place-check: no chrHC_altU hit found for R2 probe (fixture broken)" +[ "$alt_liftable" = "0" ] || \ + fail "ex-place-check: chrHC_altU liftable=$alt_liftable (expected 0 -- fix is in HOLE); fixture does not exercise the intended path" +ok "chrHC_altU ALT hit: liftable=0 confirmed (footprint in lift HOLE)" + +# Also confirm the primary chrHC hit IS liftable (sanity check). +pri_liftable=$(mawk '$2=="chrHC"{ + for(i=1;i<=NF;i++){ + if(index($i,"liftable=")==1){print substr($i,10); exit} + }}' "$TMPD/place.txt") +[ "$pri_liftable" = "1" ] || \ + fail "ex-place-check: chrHC primary hit liftable=$pri_liftable (expected 1)" +ok "chrHC primary hit: liftable=1 confirmed" + +# ========================================================================= +# (2)+(3) Run PE mapping and assert on flags/MAPQ. +echo "[test-hookc] running PE minibwa mem ..." +"$MINIBWA" map --outn=50 "$TMPD/fix/ref.fa" \ + "$TMPD/fix/reads_1.fq" "$TMPD/fix/reads_2.fq" 2>/dev/null \ + | mawk '$1 !~ /^@/' > "$TMPD/pe.sam" +[ -s "$TMPD/pe.sam" ] || fail "PE mapping produced no alignments" + +echo "----- r-hookc SAM records -----" +mawk '$1=="r-hookc"{printf "%s flag=%d %s pos=%s mapq=%s\n",$1,$2,$3,$4,$5}' \ + "$TMPD/pe.sam" +echo "-------------------------------" + +# --- (2a) The unliftable ALT hit (chrHC_altU, last mate) must be secondary (0x100). --- +# It is demoted because mb_distinct_lifted_group returns 0 for the unliftable hit, +# so the chimeric demotion fires normally. +alt_flag=$(flag_of "$TMPD/pe.sam" r-hookc chrHC_altU 101) +[ -n "$alt_flag" ] || fail "no r-hookc record on chrHC_altU pos=101 (ALT hit absent)" +[ "$(has_bit "$alt_flag" 256)" = "1" ] || \ + fail "chrHC_altU pos=101 flag=$alt_flag NOT secondary (0x100): unliftable ALT hit was SHIELDED by Hook C (mb_distinct_lifted_group wrongly returned 1 for unliftable hit)" +ok "chrHC_altU pos=101 (unliftable ALT hit) is secondary 0x100 (flag=$alt_flag): demoted normally, not shielded" + +# --- (2b) The unliftable ALT hit must NOT be supplementary (0x800). --- +# If it were 0x800 it would mean it was kept as a separate-group representative -- +# that is the "shielded" failure mode (Hook C wrongly fired the continue). +[ "$(has_bit "$alt_flag" 2048)" = "0" ] || \ + fail "chrHC_altU pos=101 flag=$alt_flag is supplementary (0x800): unliftable ALT hit was treated as a distinct group (wrongly shielded)" +ok "chrHC_altU pos=101 (unliftable ALT hit) is NOT supplementary: correctly NOT treated as a distinct group" + +# --- (3a) R2 primary (chrHC pos=701) must be a proper-pair primary with MAPQ > 0. --- +r2_flag=$(flag_of "$TMPD/pe.sam" r-hookc chrHC 701) +[ -n "$r2_flag" ] || fail "no r-hookc record on chrHC pos=701 (R2 primary absent)" +[ "$(has_bit "$r2_flag" 256)" = "0" ] || \ + fail "R2 primary chrHC pos=701 flag=$r2_flag is secondary -- pairing failed" +[ "$(has_bit "$r2_flag" 2)" = "1" ] || \ + fail "R2 primary chrHC pos=701 flag=$r2_flag is NOT a proper pair" +r2_mapq=$(mapq_of "$TMPD/pe.sam" r-hookc chrHC 701) +[ "$r2_mapq" -gt 0 ] || \ + fail "R2 primary chrHC pos=701 MAPQ=$r2_mapq (expected >0): unliftable competitor inflated or collapsed MAPQ" +ok "R2 primary: chrHC pos=701 proper-pair primary, MAPQ=$r2_mapq > 0 (flag=$r2_flag)" + +# --- (3b) R1 primary (chrHC pos=301) must be a proper-pair primary with MAPQ > 0. --- +r1_flag=$(flag_of "$TMPD/pe.sam" r-hookc chrHC 301) +[ -n "$r1_flag" ] || fail "no r-hookc record on chrHC pos=301 (R1 primary absent)" +[ "$(has_bit "$r1_flag" 256)" = "0" ] || \ + fail "R1 primary chrHC pos=301 flag=$r1_flag is secondary" +[ "$(has_bit "$r1_flag" 2)" = "1" ] || \ + fail "R1 primary chrHC pos=301 flag=$r1_flag is NOT a proper pair" +r1_mapq=$(mapq_of "$TMPD/pe.sam" r-hookc chrHC 301) +[ "$r1_mapq" -gt 0 ] || \ + fail "R1 primary chrHC pos=301 MAPQ=$r1_mapq (expected >0)" +ok "R1 primary: chrHC pos=301 proper-pair primary, MAPQ=$r1_mapq > 0 (flag=$r1_flag)" + +# ========================================================================= +# (4) Baseline: chrM PE with no .alt -> byte-identical across runs. +echo "[test-hookc] chrM baseline (no .alt): byte-identical to stock ..." +chrm_baseline "(BASELINE) chrM" pe --outn=5 + +echo "[test-hookc] PASS" +exit 0 diff --git a/test/altlg/test-indel.sh b/test/altlg/test-indel.sh new file mode 100755 index 0000000..4200531 --- /dev/null +++ b/test/altlg/test-indel.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# Regression test for Task 7 fix round 1, Fix 3: REJECT length-changing lifted +# spans. +# +# A seed that spans an indel (or two adjacent .alt lift blocks) lifts to a primary +# span whose length differs from the seed length. Injecting it as a contiguous +# len-bp anchor would corrupt the chain coordinates, so mb_anchor_project_alt must +# reject it (pri_en - pri_st + 1 != q->len -> continue). +# +# The fixture's ALT contig lifts to the primary with a 5bp deletion (.alt CIGAR +# 75M5D75M); the read's full-length SMEM spans that deletion, so its lifted span +# is 5bp longer than the seed. Asserted via the MB_PROJ_TRACE probe seam: +# * the full-length spanning seed (len=150) produces NO projected anchor; +# * every projected anchor that IS emitted has len equal to its lifted span +# length (len-preserving) -- the probe prints the projected POS and len, and +# we re-derive the span length from the contig and require it to match. +# +# Without Fix 3 the spanning seed would project (a len-150 anchor at a 155bp span); +# this is demonstrated in the task report by toggling the guard. +# +# Usage: test/altlg/test-indel.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" +MK="$MDIR/test/altlg/mkfixture-indel.sh" + +TMPD=$(mktemp -d /tmp/altlg-indel.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +echo "[test-indel] building indel fixture (.alt 75M5D75M) ..." +/bin/sh "$MK" "$TMPD" 2>/dev/null +. "$TMPD/meta.txt" +[ -n "${LIFT_POS1:-}" ] || fail "fixture did not export LIFT_POS1" +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +"$MINIBWA" map --dbg-alt-proj -c 4 --outn=999 \ + "$TMPD/ref.fa" "$TMPD/reads.fq" 2>"$TMPD/err" >"$TMPD/sam" +echo " MB_PROJ traces:" +grep '^MB_PROJ' "$TMPD/err" | sed 's/^/ /' || echo " (none)" + +# 1) The full-length spanning seed (len=150) must NOT have projected. +span=$(mawk '$1=="MB_PROJ" && $5=="len=150"{print; c++} END{print "COUNT="c+0}' "$TMPD/err" | sed -n 's/^COUNT=//p') +[ "${span:-0}" = "0" ] \ + || fail "the indel-spanning seed (len=150) projected -- Fix 3 (length-changing reject) is not in effect" +ok "indel-spanning seed (len=150) NOT projected (length-changing lift rejected)" + +# 2) The read is still placed on the primary via the normal .alt path: a chrP +# record with a deletion in its CIGAR, anchored at the LIFT locus region. +del_rec=$(mawk -v p="$LIFT_POS1" '$1!~/^@/ && $3=="chrP" && $6~/D/{print "1"; exit}' "$TMPD/sam") +[ "${del_rec:-0}" = "1" ] \ + || fail "no chrP record with a deletion CIGAR -- fixture did not exercise an indel-spanning placement" +ok "read still placed on chrP across the deletion via the normal .alt path (CIGAR contains D)" + +echo "[test-indel] PASS" +exit 0 diff --git a/test/altlg/test-lift.sh b/test/altlg/test-lift.sh new file mode 100755 index 0000000..73cba42 --- /dev/null +++ b/test/altlg/test-lift.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# Integration test for l2b_set_alt / l2b_lift (Task 1: span-lift index). +# Builds a synthetic fixture, indexes it with minibwa, and verifies: +# 1. chrP_alt is flagged ALT (ex-lift-check exits 0). +# 2. lift(100) succeeded (ok=1) and maps to pri=100 (identity, in block 0). +# 3. lift(400) succeeded (ok=1) and maps to pri=395 (5bp insertion shifts by 5). +# Usage: test/altlg/test-lift.sh [] +# defaults to two levels above this script's directory. +set -eu + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MKFIXTURE="$MDIR/test/altlg/mkfixture.sh" +MINIBWA="$MDIR/minibwa" +EX_LIFT="$MDIR/api-test/ex-lift-check" + +TMPD=$(mktemp -d /tmp/altlg-test.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + +echo "[test-lift] building fixture in $TMPD ..." +/bin/sh "$MKFIXTURE" "$TMPD" + +echo "[test-lift] indexing with minibwa ..." +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +echo "[test-lift] running ex-lift-check ..." +OUT=$("$EX_LIFT" "$TMPD/ref.fa" 2>&1) +echo "[test-lift] output: $OUT" + +# --- assertion 1: chrP_alt is flagged ALT (program exits non-zero and prints error if not) --- +# exit-code already checked by set -e above. + +# Output format: lift(100)= pri= lift(400)= pri= + +# --- assertion 2a: lift(100) succeeded (ok1=1) --- +OK1=$(printf '%s\n' "$OUT" | mawk '{if(match($0,/lift\(100\)=[0-9]+/)) print substr($0,RSTART+10,RLENGTH-10)}') +if [ "$OK1" != "1" ]; then + echo "FAIL: lift(100) expected ok=1, got '$OK1'" + exit 1 +fi + +# --- assertion 2b: lift(100) maps to pri=100 (identity: pos 100 is in block 0, no shift) --- +PRI1=$(printf '%s\n' "$OUT" | mawk '{if(match($0,/lift\(100\)=[0-9]+ pri=[0-9]+/)) {s=substr($0,RSTART,RLENGTH); if(match(s,/pri=[0-9]+/)) print substr(s,RSTART+4,RLENGTH-4)}}') +if [ "$PRI1" != "100" ]; then + echo "FAIL: lift(100) expected pri=100, got '$PRI1'" + exit 1 +fi + +# --- assertion 3a: lift(400) succeeded (ok2=1) --- +OK2=$(printf '%s\n' "$OUT" | mawk '{if(match($0,/lift\(400\)=[0-9]+/)) print substr($0,RSTART+10,RLENGTH-10)}') +if [ "$OK2" != "1" ]; then + echo "FAIL: lift(400) expected ok=1, got '$OK2'" + exit 1 +fi + +# --- assertion 3b: lift(400) maps to pri=395 (5bp insertion shifts coordinates by 5) --- +PRI2=$(printf '%s\n' "$OUT" | mawk '{if(match($0,/lift\(400\)=[0-9]+ pri=[0-9]+/)) {s=substr($0,RSTART,RLENGTH); if(match(s,/pri=[0-9]+/)) print substr(s,RSTART+4,RLENGTH-4)}}') +if [ "$PRI2" != "395" ]; then + echo "FAIL: lift(400) expected pri=395, got '$PRI2'" + exit 1 +fi + +echo "[test-lift] PASS: chrP_alt flagged ALT, lift(100) ok=1 pri=100, lift(400) ok=1 pri=395" +exit 0 diff --git a/test/altlg/test-lifttol.sh b/test/altlg/test-lifttol.sh new file mode 100755 index 0000000..e4c785c --- /dev/null +++ b/test/altlg/test-lifttol.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Integration test for the runtime-tunable ALT lift tolerance (--alt-lift-tol). +# +# A near-twin lifts 15bp from the read's direct primary alignment (lift jitter). +# At the default tolerance (10) the twin is a separate group and competes, so the +# read is driven to MAPQ 0. Widening the tolerance to 20 groups them and the +# read recovers a confident MAPQ. This locks BOTH the knob plumbing and the +# documented semantics (the default is conservative; jittery .alt sets can widen +# it without merging genuine paralogs, which the placement guard still separates). +# +# Usage: test/altlg/test-lifttol.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" +MK="$MDIR/test/altlg/mkfixture-lifttol.sh" + +TMPD=$(mktemp -d /tmp/altlg-lifttol.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + +# primary MAPQ of : the record with neither 0x100 nor 0x800 set. + +echo "[test-lifttol] building fixture ..." +/bin/sh "$MK" "$TMPD" 2>/dev/null +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +run_mapq() { # + "$MINIBWA" map --outn=50 --alt-lift-tol "$1" "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null \ + | mawk '$1 !~ /^@/' > "$TMPD/t$1.sam" + [ -s "$TMPD/t$1.sam" ] || fail "tol=$1: no alignments emitted" + primary_mapq "$TMPD/t$1.sam" r-near +} + +echo "== default tolerance (10): 15bp jitter under-merged => MAPQ 0 ==" +q10=$(run_mapq 10) +[ -n "$q10" ] || fail "no primary record for r-near at tol=10" +echo " r-near primary MAPQ (tol=10) = $q10" +[ "$q10" = "0" ] || fail "expected MAPQ 0 at tol=10 (twin a separate group), got $q10" +ok "tol=10: r-near MAPQ=0 (near-twin not merged; conservative default)" + +echo "== widened tolerance (20): 15bp jitter absorbed => twin groups, MAPQ>0 ==" +q20=$(run_mapq 20) +[ -n "$q20" ] || fail "no primary record for r-near at tol=20" +echo " r-near primary MAPQ (tol=20) = $q20" +[ "$q20" -gt 0 ] || fail "expected MAPQ>0 at tol=20 (twin grouped), got $q20" +ok "tol=20: r-near MAPQ=$q20 > 0 (near-twin grouped; recovered)" + +echo "[test-lifttol] PASS" +exit 0 diff --git a/test/altlg/test-noalt.sh b/test/altlg/test-noalt.sh new file mode 100755 index 0000000..b305dc6 --- /dev/null +++ b/test/altlg/test-noalt.sh @@ -0,0 +1,172 @@ +#!/bin/sh +# Integration test for --no-alt, and for .alt resolution being independent of +# how the index was loaded. +# +# The fixture (mkfixture.sh) is a primary chrP plus an ALT twin chrP_alt that +# differs only by a 5bp insertion, and a read landing in the shared flank. The +# read therefore hits both contigs equally well, which is precisely the case +# ALT-awareness exists for: with the .alt loaded the two hits are recognised as +# one locus and the primary keeps MAPQ 60; without it they look like a genuine +# repeat and MAPQ collapses to 0. That 60-vs-0 split is the signal every check +# below keys on -- it is unambiguous and it is the user-visible consequence. +# +# Verifies: +# 1. Liveness -- ALT-on and --no-alt really do differ (MAPQ 60 vs 0). +# 2. Equivalence-- --no-alt is byte-identical to deleting the .alt outright. +# 3. Precedence -- --no-alt wins over an explicit --alt FILE, either order. +# 4. mmap parity-- --mmap agrees with the normal loader, ALT on AND off. +# Before .alt resolution moved into the caller, mb_idx_load auto-detected +# .alt but mb_idx_load_mmap did not, so --mmap silently disabled +# ALT-awareness on the very same index. This check is that regression. +# 5. mem parity -- the bwa-compat `mem` subcommand resolves .alt as `map` does. +# Moving resolution out of mb_idx_load() and into the caller is easy to do +# for one caller and forget for the other; `mem` is the other one. +# 6. Bad --alt -- an explicitly named .alt that cannot be read is an error, +# not a silent fallback to stock behaviour. +# +# Usage: test/altlg/test-noalt.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" + +TMPD=$(mktemp -d /tmp/altlg-noalt.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# Alignment records only: @PG carries the command line, which legitimately +# differs between two runs we expect to align identically. + +# MAPQ of the primary (neither secondary 0x100 nor supplementary 0x800). +# mawk has no and(), so test the two bits by arithmetic. + +# Run `map` with the fixture and the given extra flags, into $TMPD/$1.sam. +run() { + out="$1"; shift + "$MINIBWA" map --outn=5 "$@" "$TMPD/ref.fa" "$TMPD/r1.fq" \ + 2>/dev/null > "$TMPD/$out.sam" \ + || fail "minibwa map failed for '$out' (flags: $*)" + [ -s "$TMPD/$out.sam" ] || fail "empty output for '$out' (flags: $*)" + sam_body "$TMPD/$out.sam" > "$TMPD/$out.body" +} + +echo "[test-noalt] building fixture ..." +sh "$MDIR/test/altlg/mkfixture.sh" "$TMPD" +[ -f "$TMPD/ref.fa.alt" ] || fail "fixture did not produce ref.fa.alt" +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null || fail "indexing the fixture failed" + +# ============================================================ +# 1. Liveness: the flag actually does something +# ============================================================ +echo "== liveness ==" +run alt-on +run alt-off --no-alt + +mq_on=$(primary_mapq "$TMPD/alt-on.sam") +mq_off=$(primary_mapq "$TMPD/alt-off.sam") +[ "${mq_on:-}" = "60" ] \ + || fail "ALT on: primary MAPQ is '${mq_on:-}', expected 60 (is the .alt being loaded at all?)" +ok "ALT on: primary MAPQ 60 (ALT twin recognised, MAPQ preserved)" +[ "${mq_off:-}" = "0" ] \ + || fail "--no-alt: primary MAPQ is '${mq_off:-}', expected 0 (ALT-awareness was not suppressed)" +ok "--no-alt: primary MAPQ 0 (ALT twin looks like a repeat, as it should)" + +cmp -s "$TMPD/alt-on.body" "$TMPD/alt-off.body" \ + && fail "--no-alt produced byte-identical output to ALT-on; the flag is a no-op" +ok "--no-alt changes the alignment records" + +# ============================================================ +# 2. Equivalence: --no-alt == no .alt file on disk +# ============================================================ +echo "== equivalence with a deleted .alt ==" +mv "$TMPD/ref.fa.alt" "$TMPD/hidden.alt" +run alt-absent +mv "$TMPD/hidden.alt" "$TMPD/ref.fa.alt" + +cmp -s "$TMPD/alt-off.body" "$TMPD/alt-absent.body" \ + || fail "--no-alt differs from running against an index with no .alt beside it" +ok "--no-alt is byte-identical to having no .alt at all" + +# ============================================================ +# 3. Precedence: --no-alt beats an explicit --alt, in either order +# ============================================================ +echo "== precedence over --alt FILE ==" +cp "$TMPD/ref.fa.alt" "$TMPD/elsewhere.alt" + +run alt-explicit --alt "$TMPD/elsewhere.alt" +cmp -s "$TMPD/alt-explicit.body" "$TMPD/alt-on.body" \ + || fail "--alt FILE with the same content differs from the auto-detected .alt" +ok "--alt FILE reproduces the auto-detected result (control)" + +run alt-then-no --alt "$TMPD/elsewhere.alt" --no-alt +run no-then-alt --no-alt --alt "$TMPD/elsewhere.alt" +cmp -s "$TMPD/alt-then-no.body" "$TMPD/alt-off.body" \ + || fail "'--alt FILE --no-alt' did not suppress ALT-awareness" +ok "'--alt FILE --no-alt' suppresses ALT-awareness" +cmp -s "$TMPD/no-then-alt.body" "$TMPD/alt-off.body" \ + || fail "'--no-alt --alt FILE' did not suppress ALT-awareness (order-dependent!)" +ok "'--no-alt --alt FILE' suppresses ALT-awareness (order-independent)" + +# ============================================================ +# 4. mmap parity, ALT on and ALT off +# ============================================================ +echo "== --mmap parity ==" +run mmap-on --mmap +run mmap-off --mmap --no-alt + +mq_mmap=$(primary_mapq "$TMPD/mmap-on.sam") +[ "${mq_mmap:-}" = "60" ] \ + || fail "--mmap with an adjacent .alt: primary MAPQ is '${mq_mmap:-}', expected 60 -- the mmap loader is not seeing the .alt" +cmp -s "$TMPD/mmap-on.body" "$TMPD/alt-on.body" \ + || fail "--mmap changed the alignment records with ALT on (loader-dependent .alt resolution)" +ok "--mmap matches the normal loader with ALT on" + +cmp -s "$TMPD/mmap-off.body" "$TMPD/alt-off.body" \ + || fail "--mmap changed the alignment records under --no-alt" +ok "--mmap matches the normal loader under --no-alt" + +# ============================================================ +# 5. `mem` resolves .alt the same way `map` does +# ============================================================ +echo "== bwa-compat \`mem\` parity ==" +"$MINIBWA" mem "$TMPD/ref.fa" "$TMPD/r1.fq" 2>/dev/null > "$TMPD/mem-on.sam" \ + || fail "minibwa mem failed on the ALT fixture" +[ -s "$TMPD/mem-on.sam" ] || fail "minibwa mem produced no output" + +mq_mem=$(primary_mapq "$TMPD/mem-on.sam") +[ "${mq_mem:-}" = "$mq_on" ] \ + || fail "mem: primary MAPQ is '${mq_mem:-}', but map gives '$mq_on' -- mem is not resolving .alt" +ok "mem: primary MAPQ $mq_mem matches map (both resolve the adjacent .alt)" + +# And with the .alt gone, mem must degrade exactly as map does -- otherwise the +# check above could pass for a reason unrelated to .alt resolution. +mv "$TMPD/ref.fa.alt" "$TMPD/hidden.alt" +"$MINIBWA" mem "$TMPD/ref.fa" "$TMPD/r1.fq" 2>/dev/null > "$TMPD/mem-absent.sam" \ + || fail "minibwa mem failed with no .alt present" +mv "$TMPD/hidden.alt" "$TMPD/ref.fa.alt" +mq_mem_absent=$(primary_mapq "$TMPD/mem-absent.sam") +[ "${mq_mem_absent:-}" = "$mq_off" ] \ + || fail "mem without .alt: primary MAPQ is '${mq_mem_absent:-}', expected '$mq_off'" +ok "mem without .alt: primary MAPQ $mq_mem_absent matches map --no-alt" + +# ============================================================ +# 6. An unreadable --alt FILE is an error, not a silent fallback +# ============================================================ +echo "== unreadable --alt FILE ==" +# Exit status is deliberately not asserted: this branch is based before upstream +# r422 (#67), where main() discarded every subcommand's return value. Assert on +# the diagnostic and on the absence of output instead, both of which are stable. +"$MINIBWA" map --outn=5 --alt "$TMPD/does-not-exist.alt" \ + "$TMPD/ref.fa" "$TMPD/r1.fq" > "$TMPD/bad-alt.sam" 2> "$TMPD/bad-alt.err" || true + +grep -q 'failed to load the ALT file' "$TMPD/bad-alt.err" \ + || fail "--alt with an unreadable path printed no diagnostic (stderr: $(cat "$TMPD/bad-alt.err"))" +ok "--alt with an unreadable path reports the failure" + +[ ! -s "$TMPD/bad-alt.sam" ] \ + || fail "--alt with an unreadable path still emitted alignments; it silently fell back to stock behaviour" +ok "--alt with an unreadable path emits no alignments" + +echo "[test-noalt] PASS" +exit 0 diff --git a/test/altlg/test-output.sh b/test/altlg/test-output.sh new file mode 100755 index 0000000..889353d --- /dev/null +++ b/test/altlg/test-output.sh @@ -0,0 +1,200 @@ +#!/bin/sh +# Integration test for Task 5: --alt-records flag and --alt FILE CLI. +# +# Verifies three things: +# 1. WITHOUT --alt-records, no ALT-contig record appears in SAM output +# (default out_n=0 drops secondaries/supplements on ALT contigs). +# 2. WITH --alt-records, the ALT-contig record IS emitted with FULL-LENGTH SEQ +# (length == read length, not '*'). +# 3. --alt FILE loads a .alt from a non-adjacent path and enables ALT mapping. +# 4. chrM baseline (no .alt): output is byte-identical with/without --alt-records. +# +# Usage: test/altlg/test-output.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" + +TMPD=$(mktemp -d /tmp/altlg-output.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# ============================================================ +# Build a minimal fixture: +# chrP (300bp primary) + chrP_altA (150bp ALT contig) +# .alt maps chrP_altA -> chrP at POS 101 (1-based), 150M +# read = chrP_altA sequence (unique; maps cleanly to the ALT contig) +# ============================================================ +echo "[test-output] building fixture ..." + +python3 -c " +import random, sys +random.seed(42) +def seq(n): return ''.join(random.choice('ACGT') for _ in range(n)) + +pad1 = seq(100) +core = seq(150) # shared core: appears in chrP at offset 100, and IS chrP_altA +pad2 = seq(50) + +chrp = pad1 + core + pad2 # 300bp primary +alt = core # 150bp ALT contig == core + +print(f'>chrP\n{chrp}') +print(f'>chrP_altA\n{alt}') +" > "$TMPD/ref.fa" + +# .alt: chrP_altA (0-based [0,150)) -> chrP POS 101 (1-based), 150M +printf 'chrP_altA\t0\tchrP\t101\t60\t150M\t*\t0\t0\t*\t*\n' > "$TMPD/ref.fa.alt" + +# Read: the ALT contig sequence +python3 -c " +import random +random.seed(42) +def seq(n): return ''.join(random.choice('ACGT') for _ in range(n)) +seq(100) # consume pad1 seed state +core = seq(150) +qual = 'I' * 150 +print(f'@r-alt') +print(core) +print('+') +print(qual) +" > "$TMPD/reads.fq" + +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +READ_LEN=$(python3 -c " +with open('$TMPD/reads.fq') as f: + next(f); seq=next(f).strip(); print(len(seq)) +") +echo "[test-output] read length = $READ_LEN; ALT contig = chrP_altA; lifted to chrP POS 101" + +# ============================================================ +# 1. WITHOUT --alt-records: no chrP_altA record in SAM +# ============================================================ +echo "== WITHOUT --alt-records ==" +"$MINIBWA" map "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/no-flag.sam" +alt_lines=$(mawk '$1!~/^@/ && $3=="chrP_altA"' "$TMPD/no-flag.sam" | wc -l | tr -d ' ') +[ "$alt_lines" -eq 0 ] \ + || fail "without --alt-records: $alt_lines chrP_altA record(s) emitted (expected 0)" +ok "without --alt-records: no chrP_altA record (ALT hits suppressed by default)" + +# ============================================================ +# 2. WITH --alt-records: chrP_altA IS emitted with full-length SEQ +# ============================================================ +echo "== WITH --alt-records ==" +"$MINIBWA" map --alt-records "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/with-flag.sam" + +# Check the ALT record exists +alt_lines=$(mawk '$1!~/^@/ && $3=="chrP_altA"' "$TMPD/with-flag.sam" | wc -l | tr -d ' ') +[ "$alt_lines" -gt 0 ] \ + || fail "with --alt-records: no chrP_altA record emitted" +ok "with --alt-records: chrP_altA record is present ($alt_lines line(s))" + +# Check the SEQ field is full-length (not '*') +alt_seq=$(mawk '$1!~/^@/ && $3=="chrP_altA"{print $10; exit}' "$TMPD/with-flag.sam") +[ "$alt_seq" != "*" ] \ + || fail "with --alt-records: SEQ field for chrP_altA is '*' (expected full SEQ)" +alt_seq_len=${#alt_seq} +[ "$alt_seq_len" -eq "$READ_LEN" ] \ + || fail "with --alt-records: SEQ length $alt_seq_len != read length $READ_LEN" +ok "with --alt-records: SEQ is full-length ($alt_seq_len == $READ_LEN)" + +# Report the FLAG bit for Task 6 reference +alt_flag=$(mawk '$1!~/^@/ && $3=="chrP_altA"{print $2; exit}' "$TMPD/with-flag.sam") +echo " (info) ALT record FLAG = $alt_flag (0x$(printf '%x' $alt_flag)) -- noted for Task 6" + +# ============================================================ +# 3. --alt FILE: load .alt from a non-adjacent path +# ============================================================ +echo "== --alt FILE from a non-adjacent path ==" +mkdir -p "$TMPD/altdir" +cp "$TMPD/ref.fa.alt" "$TMPD/altdir/custom.alt" + +# Build index without adjacent .alt, then specify it via --alt +python3 -c " +import random +random.seed(99) +def seq(n): return ''.join(random.choice('ACGT') for _ in range(n)) +pad = seq(50); core = seq(100); pad2 = seq(50) +chrp = pad + core + pad2 +alt = core +print(f'>chrQ\n{chrp}') +print(f'>chrQ_altX\n{alt}') +" > "$TMPD/ref2.fa" +printf 'chrQ_altX\t0\tchrQ\t51\t60\t100M\t*\t0\t0\t*\t*\n' > "$TMPD/altdir/ref2.alt" +python3 -c " +import random +random.seed(99) +def seq(n): return ''.join(random.choice('ACGT') for _ in range(n)) +seq(50); core = seq(100); qual = 'I'*100 +print('@r-alt2'); print(core); print('+'); print(qual) +" > "$TMPD/reads2.fq" +"$MINIBWA" index "$TMPD/ref2.fa" 2>/dev/null + +# With --alt pointing to the non-adjacent file, ALT record should appear +"$MINIBWA" map --alt-records --alt "$TMPD/altdir/ref2.alt" \ + "$TMPD/ref2.fa" "$TMPD/reads2.fq" 2>/dev/null > "$TMPD/alt-file.sam" +alt2_lines=$(mawk '$1!~/^@/ && $3=="chrQ_altX"' "$TMPD/alt-file.sam" | wc -l | tr -d ' ') +[ "$alt2_lines" -gt 0 ] \ + || fail "--alt FILE: no chrQ_altX record (--alt did not load the non-adjacent .alt)" +ok "--alt FILE: chrQ_altX ALT record present ($alt2_lines line(s)) from non-adjacent .alt" + +# WITHOUT --alt FILE and no adjacent .alt: chrQ_altX is not is_alt, so it +# appears as a plain primary (FLAG has no 0x100/0x800 bits set), not an ALT hit. +# With --alt FILE: chrQ_altX IS is_alt -> FLAG has 0x100 or 0x800. +"$MINIBWA" map --alt-records "$TMPD/ref2.fa" "$TMPD/reads2.fq" 2>/dev/null > "$TMPD/no-alt-file.sam" +alt2_lines_without=$(mawk '$1!~/^@/ && $3=="chrQ_altX"' "$TMPD/no-alt-file.sam" | wc -l | tr -d ' ') +[ "$alt2_lines_without" -gt 0 ] \ + || fail "--alt FILE sanity: without --alt, chrQ_altX record missing; cannot compare FLAG classification" +flag_without=$(mawk '$1!~/^@/ && $3=="chrQ_altX"{print $2; exit}' "$TMPD/no-alt-file.sam") +flag_with=$(mawk '$1!~/^@/ && $3=="chrQ_altX"{print $2; exit}' "$TMPD/alt-file.sam") +# Without --alt: chrQ_altX is primary (not secondary/supplementary) +[ $(( ${flag_without:-0} & 0x900 )) -eq 0 ] \ + || fail "--alt FILE sanity: without --alt, chrQ_altX FLAG=$flag_without has 0x100/0x800 set (unexpected)" +ok "--alt FILE sanity: without --alt, chrQ_altX FLAG=$flag_without (primary, not ALT-classified)" +# With --alt: chrQ_altX is secondary or supplementary (0x100 or 0x800) +[ $(( ${flag_with:-0} & 0x900 )) -ne 0 ] \ + || fail "--alt FILE: with --alt, chrQ_altX FLAG=$flag_with lacks 0x100/0x800 (not treated as ALT)" +ok "--alt FILE: with --alt, chrQ_altX FLAG=$flag_with (secondary/supplementary = ALT-classified)" + +# ============================================================ +# 4. chrM baseline: byte-identical with/without --alt-records (no .alt => inert) +# ============================================================ +echo "== chrM baseline (no .alt; --alt-records inert) ==" +CHRM_FA="$MDIR/test/chrM-human.fa.gz" +CHRM_R1="$MDIR/test/chrM-read_1.fa.gz" +CHRM_R2="$MDIR/test/chrM-read_2.fa.gz" +# Filter helper: strip header lines for body-only comparison + +if [ -f "$CHRM_FA" ] && [ -f "$CHRM_R1" ]; then + # Copy and index inside TMPD rather than beside the shared fixture (same reason as + # lib-baseline.sh): keeps the check hermetic, avoids racing another script indexing + # the same chrM, and leaves no untracked .mbw/.l2b in test/. + cp "$CHRM_FA" "$TMPD/chrM-ref.fa.gz"; CHRM_FA="$TMPD/chrM-ref.fa.gz" + "$MINIBWA" index "$CHRM_FA" 2>/dev/null + "$MINIBWA" map --outn=5 "$CHRM_FA" "$CHRM_R1" 2>/dev/null > "$TMPD/chrM-se-no-flag.sam" + "$MINIBWA" map --outn=5 --alt-records "$CHRM_FA" "$CHRM_R1" 2>/dev/null > "$TMPD/chrM-se-with-flag.sam" + [ -s "$TMPD/chrM-se-no-flag.sam" ] || fail "chrM SE baseline: empty output" + sam_body "$TMPD/chrM-se-no-flag.sam" > "$TMPD/chrM-se-no-flag.body" + sam_body "$TMPD/chrM-se-with-flag.sam" > "$TMPD/chrM-se-with-flag.body" + cmp -s "$TMPD/chrM-se-no-flag.body" "$TMPD/chrM-se-with-flag.body" \ + || fail "chrM SE baseline: --alt-records changed alignment records without .alt" + ok "chrM SE baseline: alignment records byte-identical with/without --alt-records (no .alt => inert)" + if [ -f "$CHRM_R2" ]; then + "$MINIBWA" map --outn=5 "$CHRM_FA" "$CHRM_R1" "$CHRM_R2" 2>/dev/null > "$TMPD/chrM-pe-no-flag.sam" + "$MINIBWA" map --outn=5 --alt-records "$CHRM_FA" "$CHRM_R1" "$CHRM_R2" 2>/dev/null > "$TMPD/chrM-pe-with-flag.sam" + sam_body "$TMPD/chrM-pe-no-flag.sam" > "$TMPD/chrM-pe-no-flag.body" + sam_body "$TMPD/chrM-pe-with-flag.sam" > "$TMPD/chrM-pe-with-flag.body" + cmp -s "$TMPD/chrM-pe-no-flag.body" "$TMPD/chrM-pe-with-flag.body" \ + || fail "chrM PE baseline: --alt-records changed alignment records without .alt" + ok "chrM PE baseline: alignment records byte-identical with/without --alt-records (no .alt => inert)" + else + echo " skip: chrM R2 not found ($CHRM_R2)" + fi +else + echo " skip: chrM baseline files not found ($CHRM_FA)" +fi + +echo "[test-output] PASS" +exit 0 diff --git a/test/altlg/test-pe.sh b/test/altlg/test-pe.sh new file mode 100755 index 0000000..98cff32 --- /dev/null +++ b/test/altlg/test-pe.sh @@ -0,0 +1,154 @@ +#!/bin/sh +# Integration test for Task 8: PE liftover-group reconciliation integration. +# +# Runs `minibwa mem` in PAIRED-END mode on synthetic fixtures and asserts on the +# SAM flags / MAPQ that the three PE hooks produce. WGS is always PE, so this is +# what makes the ALT liftover-group feature actually work on the real target. +# +# Cases: +# (HAPPY) Hook B: a pair whose BOTH mates lie inside a primary region with an +# identical ALT twin contig. On primary the pair is best; on the ALT twin an +# equally-good second PAIR forms (pairs form only within one contig). Without +# Hook B that twin pair depresses paux.sub_sc => mapq_pe ~= 0 and the ALT twin +# can even win sam_pri. With Hook B (exclude non-rep group members from pair +# enumeration) the twin pair never forms: chrP is the proper-pair primary for +# BOTH mates with MAPQ>0, the chrP_altF twin records are secondary (0x100). +# [RED proven: pre-Hook-B mapq_pe collapses and chrP_altF wins the pair.] +# (PARALOG) Hook C: a pair where one mate is an ambiguous paralog (COPY1/COPY2 on +# chrP whose ALT twins lift just outside MB_LIFT_TOL). The chimeric demotion +# loop (pe.c:524-535) would, by query overlap alone, fold the COPY2 hit under +# the chosen COPY1 pair-hit -- merging two DISTINCT lifted groups and undoing +# the Task-4 paralog guard. Hook C skips the demotion across distinct lifted +# groups, so COPY2 stays its own group representative: it appears as a +# supplementary (0x800) record, NOT a demoted secondary (0x100). +# [RED proven: pre-Hook-C COPY2 is 0x100 (merged).] +# (RESCUE) is_alt stamp + Hook A: one mate is peppered so it cannot seed and is +# recovered only by mate rescue, landing inside an ALT-twinned window. The +# rescued ALT hit must carry is_alt (mb_matesw_align memset clears it) so the +# reconciliation groups it. Asserts: the unseedable mate is rescue-only +# (unmapped single-end), the rescued chrP_altG hit is grouped (secondary, +# 0x100) rather than a second proper-pair primary, chrP is the proper-pair +# primary with MAPQ>0. +# (BASELINE) chrM PE with NO .alt: every hook is gated (mb_any_alt false / no ALT +# hits), so output is byte-identical across runs. +# +# Usage: test/altlg/test-pe.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" + +# Shared no-.alt baseline check (see lib-baseline.sh). +. "$(dirname "$0")/lib-baseline.sh" +MK_HAPPY="$MDIR/test/altlg/mkfixture-pe-happy.sh" +MK_PARA="$MDIR/test/altlg/mkfixture-pe-paralog.sh" +MK_RESC="$MDIR/test/altlg/mkfixture-pe-rescue.sh" + +TMPD=$(mktemp -d /tmp/altlg-pe.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# has_bit -> "1" if (flag & bit) else "0" (single power-of-two bit). +# flag_of -> FLAG of the matching record (pos optional). +# proper-pair primary FLAG for a given mate-bit (0x40 first / 0x80 last). +prim_flag_mate() { mawk -v q="$2" -v mb="$3" '$1==q { f=int($2); + if (int(f/256)%2==0 && int(f/2048)%2==0 && int(f/mb)%2==1) { print $2; exit } }' "$1"; } +prim_mapq_mate() { mawk -v q="$2" -v mb="$3" '$1==q { f=int($2); + if (int(f/256)%2==0 && int(f/2048)%2==0 && int(f/mb)%2==1) { print $5; exit } }' "$1"; } + +run_pe() { # + "$MINIBWA" index "$1/ref.fa" 2>/dev/null + "$MINIBWA" map --outn=50 "$1/ref.fa" "$1/reads_1.fq" "$1/reads_2.fq" 2>/dev/null \ + | mawk '$1 !~ /^@/' > "$2" + [ -s "$2" ] || fail "mapping produced no alignments ($1)" +} + +# ========================================================================= +echo "[test-pe] (HAPPY / Hook B) building fixture ..." +HD="$TMPD/happy"; /bin/sh "$MK_HAPPY" "$HD" 2>/dev/null +run_pe "$HD" "$TMPD/happy.sam" +echo "----- happy r-twin SAM -----" +mawk '$1=="r-twin"{printf "%s flag=%d %s pos=%s mapq=%s\n",$1,$2,$3,$4,$5}' "$TMPD/happy.sam" +echo "----------------------------" + +# chrP must be the proper-pair primary for BOTH mates with MAPQ>0. +for mb in 64 128; do # 0x40 first mate, 0x80 last mate + pf=$(prim_flag_mate "$TMPD/happy.sam" r-twin "$mb") + pm=$(prim_mapq_mate "$TMPD/happy.sam" r-twin "$mb") + [ -n "$pf" ] || fail "(HAPPY) no proper-pair primary for mate bit $mb" + # primary must be on chrP (not the ALT twin) and a proper pair (0x2) + rn=$(mawk -v f="$pf" '$1=="r-twin" && $2==f{print $3; exit}' "$TMPD/happy.sam") + [ "$rn" = "chrP" ] || fail "(HAPPY) mate $mb primary on $rn, expected chrP (ALT twin won the pair)" + [ "$(has_bit "$pf" 2)" = "1" ] || fail "(HAPPY) mate $mb primary not a proper pair (flag=$pf)" + [ "$pm" -gt 0 ] || fail "(HAPPY) mate $mb MAPQ=$pm; twin depressed mapq_pe (expected >0)" + ok "(HAPPY) mate $mb: chrP proper-pair primary, MAPQ=$pm > 0" +done +# every chrP_altF twin record must be secondary (0x100). Run the assertion in a +# single mawk that exits nonzero on the first non-secondary record; a `while read` +# pipeline would `exit 1` only from its subshell under /bin/sh, falsely PASSing. +if ! mawk '$1=="r-twin" && $3=="chrP_altF" && int($2/256)%2==0 { bad=1; exit } END{ exit (bad?1:0) }' "$TMPD/happy.sam"; then + fail "(HAPPY) found chrP_altF record that is not secondary (0x100)" +fi +ok "(HAPPY) all chrP_altF twin records are secondary (0x100)" + +# ========================================================================= +echo "[test-pe] (PARALOG / Hook C) building fixture ..." +PD="$TMPD/para"; /bin/sh "$MK_PARA" "$PD" 2>/dev/null +run_pe "$PD" "$TMPD/para.sam" +echo "----- paralog r-para SAM -----" +mawk '$1=="r-para"{printf "%s flag=%d %s pos=%s mapq=%s\n",$1,$2,$3,$4,$5}' "$TMPD/para.sam" +echo "------------------------------" +# COPY1 is the chosen pair-hit (first mate, the unique R2 anchors it near COPY1). +# COPY2 lives at chrP pos 2371 (0-based 2370). Hook C must keep it a DISTINCT +# lifted group: it must NOT be demoted to 0x100 secondary (which is what the +# cross-group merge would do). A separate group rep that is not sam_pri shows as +# 0x800 supplementary. +c2_flag=$(flag_of "$TMPD/para.sam" r-para chrP 2371) +[ -n "$c2_flag" ] || fail "(PARALOG) no chrP record at pos 2371 (COPY2)" +c2_sec=$(has_bit "$c2_flag" 256) +[ "$c2_sec" = "0" ] || fail "(PARALOG) COPY2 (pos 2371) is 0x100 secondary (flag=$c2_flag): merged into COPY1's group (Hook C failed)" +# Pin the CURRENT state: a distinct group rep that is not sam_pri shows as 0x800 +# supplementary. NOTE: whether distinct paralog loci should be 0x800 vs 0x100 +# (per bwa-postalt) is a DEFERRED open question for Task 6; this assertion locks +# in today's behavior so any future change here is caught and reviewed. +[ "$(has_bit "$c2_flag" 2048)" = "1" ] || fail "(PARALOG) COPY2 expected supplementary (0x800), got flag=$c2_flag" +ok "(PARALOG) COPY2 (pos 2371, flag=$c2_flag) NOT demoted to secondary: distinct lifted group preserved (0x800 supplementary)" + +# ========================================================================= +echo "[test-pe] (RESCUE / is_alt stamp + Hook A) building fixture ..." +RD="$TMPD/resc"; /bin/sh "$MK_RESC" "$RD" 2>/dev/null +"$MINIBWA" index "$RD/ref.fa" 2>/dev/null +# (1) the peppered mate is rescue-only: unmapped when mapped single-end. +mawk '/^@r-rescue/{p=4} p>0{print;p--}' "$RD/reads_2.fq" > "$RD/r2.fq" +r2se=$("$MINIBWA" map --outn=50 "$RD/ref.fa" "$RD/r2.fq" 2>/dev/null | mawk '$1=="r-rescue/2"{print $2; exit}') +[ -n "$r2se" ] || fail "(RESCUE) no single-end record for R2" +[ "$(has_bit "$r2se" 4)" = "1" ] || fail "(RESCUE) R2 mapped single-end (flag=$r2se); fixture must be rescue-only" +ok "(RESCUE) R2 is rescue-only (unmapped single-end, flag=$r2se)" +# (2) full PE run. +run_pe "$RD" "$TMPD/resc.sam" +echo "----- rescue r-rescue SAM -----" +mawk '$1=="r-rescue"{printf "%s flag=%d %s pos=%s mapq=%s\n",$1,$2,$3,$4,$5}' "$TMPD/resc.sam" +echo "-------------------------------" +# chrP must be the proper-pair primary for the rescued (last) mate with MAPQ>0. +rp_flag=$(prim_flag_mate "$TMPD/resc.sam" r-rescue 128) +rp_mapq=$(prim_mapq_mate "$TMPD/resc.sam" r-rescue 128) +[ -n "$rp_flag" ] || fail "(RESCUE) no proper-pair primary for the rescued mate" +rp_rn=$(mawk -v f="$rp_flag" '$1=="r-rescue" && $2==f{print $3; exit}' "$TMPD/resc.sam") +[ "$rp_rn" = "chrP" ] || fail "(RESCUE) rescued-mate primary on $rp_rn, expected chrP" +[ "$rp_mapq" -gt 0 ] || fail "(RESCUE) rescued-mate MAPQ=$rp_mapq; expected >0" +ok "(RESCUE) rescued mate: chrP proper-pair primary, MAPQ=$rp_mapq > 0" +# the rescued ALT hit must be GROUPED (secondary, 0x100) -- not a 2nd proper-pair +# primary on the ALT contig. +rg_flag=$(mawk '$1=="r-rescue" && $3=="chrP_altG" && int($2/128)%2==1{print $2; exit}' "$TMPD/resc.sam") +[ -n "$rg_flag" ] || fail "(RESCUE) rescued ALT hit chrP_altG absent for the rescued mate" +[ "$(has_bit "$rg_flag" 256)" = "1" ] || fail "(RESCUE) rescued chrP_altG flag=$rg_flag not secondary; ALT mate not grouped" +[ "$(has_bit "$rg_flag" 2)" = "0" ] || fail "(RESCUE) rescued chrP_altG flag=$rg_flag is a proper pair; ALT mate competed" +ok "(RESCUE) rescued chrP_altG hit is grouped (secondary 0x100, not a competing pair)" + +# ========================================================================= +echo "[test-pe] (BASELINE) chrM PE, no .alt: byte-identical to stock ..." +chrm_baseline "(BASELINE) chrM PE" pe --outn=5 + +echo "[test-pe] PASS" +exit 0 diff --git a/test/altlg/test-projrev.sh b/test/altlg/test-projrev.sh new file mode 100755 index 0000000..539a2d5 --- /dev/null +++ b/test/altlg/test-projrev.sh @@ -0,0 +1,164 @@ +#!/bin/sh +# Integration test: REVERSE-strand ALT-seed -> primary anchor projection, +# observable end-to-end via MAPQ in the SAM output (no probe required). +# +# WHAT IS TESTED +# -------------- +# Proves that mb_anchor_project_alt's reverse-block code path (blk_rev ^ alt_rev, +# Fix 1 + Fix 2 in seed.c) recovers a primary alignment that is SAM-observable +# as MAPQ > 0 on the primary assembly, not just through the MB_PROJ_TRACE +# diagnostic probe used by test-segdup-rev.sh. +# +# MECHANISM (see mkfixture-projrev.sh for the full derivation) +# ------------------------------------------------------------ +# The read's 150bp SMEM has SA=2: the ALT contig (forward) and the primary LIFT +# locus (reverse complement). -c 1 (max_occ=1) subsamples this SA=2 interval +# to 1 hit; the SA-array lexicographic ordering for the chosen sequences places +# the ALT-contig hit first, so the strided sampler takes only that hit. +# +# RED (MB_NO_ALT_PROJECT=1, projection disabled): +# Only the ALT-contig anchor exists. The read aligns to chrP_altS (SAM +# primary, FLAG=0, MAPQ=60). No chrP record at LIFT_POS1 is emitted. +# +# GREEN (projection enabled, default): +# The ALT anchor is lifted through the RC .alt block: +# blk_rev=1, alt_rev=0 -> folded_rev = blk_rev ^ alt_rev = 1 (reverse). +# A native reverse-strand primary anchor is injected at LIFT_POS1. DP at +# LIFT: read maps perfectly (150M, NM=0, score=300) because the primary +# stores RC(read) at LIFT_POS1. Reconciliation groups {chrP_altS, chrP LIFT}; +# scores tie at 300; non-ALT preference promotes chrP -> chrP is SAM primary +# (FLAG=16, MAPQ=60). chrP_altS becomes secondary (FLAG=256, MAPQ=0). +# +# Assertions: +# RED: chrP POS=LIFT_POS1 absent (primary seed subsampled, no projection). +# GREEN: chrP POS=LIFT_POS1 present with MAPQ > 0 (reverse projection works). +# GREEN: chrP POS=LIFT_POS1 is the SAM primary record (FLAG & 0x900 == 0). +# Also: chrM byte-identical baseline (no .alt => projection is a strict no-op). +# +# Usage: test/altlg/test-projrev.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" +MK="$MDIR/test/altlg/mkfixture-projrev.sh" + +# max_occ=1: subsamples the 150bp SMEM (SA=2) to 1 hit; the SA-array ordering +# for the chosen sequence seeds deterministically picks the ALT-contig hit, +# leaving the primary LIFT locus unreachable without projection. +C=1 +N=20 + +TMPD=$(mktemp -d /tmp/altlg-projrev.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# chrP POS lookup helpers (same style as test-segdup.sh). +mapq_at_pos() { mawk -v p="$2" '$1!~/^@/ && $3=="chrP" && $4==p{print $5; exit}' "$1"; } +flag_at_pos() { mawk -v p="$2" '$1!~/^@/ && $3=="chrP" && $4==p{print $2; exit}' "$1"; } + +echo "[test-projrev] building reverse-projection fixture (N=$N copies) ..." +/bin/sh "$MK" "$TMPD" "$N" 2>/dev/null +. "$TMPD/meta.txt" # sets LIFT_POS1 +[ -n "${LIFT_POS1:-}" ] || fail "fixture did not export LIFT_POS1" +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +echo "[test-projrev] LIFT locus is chrP POS=$LIFT_POS1; max_occ (-c) = $C" +echo "[test-projrev] 150bp SMEM SA=2 (ALT-contig + LIFT-reverse); -c 1 subsamples to ALT hit first" + +# ========================================================================= +# --- RED: projection disabled --- +echo "== RED: MB_NO_ALT_PROJECT=1 (no projection) ==" +"$MINIBWA" map --dbg-no-alt-proj -c "$C" --outn=999 "$TMPD/ref.fa" "$TMPD/reads.fq" \ + 2>/dev/null > "$TMPD/red.sam" + +red_hit=$(has_pos "$TMPD/red.sam" "$LIFT_POS1") +[ "$red_hit" = "0" ] || fail "RED: chrP record exists at LIFT POS=$LIFT_POS1 without projection; the primary LIFT locus is reachable by direct seeds (SA ordering or max_occ changed - check sequence seeds and -c value)" +ok "RED: no chrP record at LIFT POS=$LIFT_POS1 (primary seed subsampled out, ALT hit sampled instead)" + +# Sanity: the ALT contig IS present (the ALT seed was taken by the sampler). +alt_present=$(mawk '$1!~/^@/ && $3=="chrP_altS"{print "1"; exit}' "$TMPD/red.sam") +[ "${alt_present:-0}" = "1" ] || fail "RED: chrP_altS alignment missing (ALT seed not taken by sampler)" +ok "RED: chrP_altS alignment present (ALT seed was the sampled hit, as expected)" + +# ========================================================================= +# --- GREEN: projection enabled (default) --- +echo "== GREEN: projection enabled (default) ==" +"$MINIBWA" map -c "$C" --outn=999 "$TMPD/ref.fa" "$TMPD/reads.fq" \ + 2>/dev/null > "$TMPD/green.sam" + +green_hit=$(has_pos "$TMPD/green.sam" "$LIFT_POS1") +[ "$green_hit" = "1" ] || fail "GREEN: projection did not produce a chrP record at LIFT POS=$LIFT_POS1" +ok "GREEN: chrP record present at LIFT POS=$LIFT_POS1 (reverse ALT seed projected to primary)" + +# MAPQ > 0: the projected reverse alignment is the SAM primary and has a +# confident score because the primary stores RC(read) at LIFT_POS1 (perfect +# 150M match; score=300, same as the ALT alignment -> non-ALT preference +# promotes chrP to group representative). +green_mapq=$(mapq_at_pos "$TMPD/green.sam" "$LIFT_POS1") +[ -n "$green_mapq" ] || fail "GREEN: could not read MAPQ from chrP POS=$LIFT_POS1 record" +[ "$green_mapq" -gt 0 ] || fail "GREEN: chrP POS=$LIFT_POS1 has MAPQ=$green_mapq (not > 0); ALT contig may have won group rep (check scores)" +ok "GREEN: chrP POS=$LIFT_POS1 has MAPQ=$green_mapq > 0 (confident primary alignment via reverse projection)" + +# The chrP LIFT record must be the SAM PRIMARY (FLAG & 0x900 == 0), not a +# secondary. non-ALT preference makes it the group representative. +green_flag=$(flag_at_pos "$TMPD/green.sam" "$LIFT_POS1") +primary_mask=$(( green_flag & 0x900 )) +[ "$primary_mask" -eq 0 ] || fail "GREEN: chrP POS=$LIFT_POS1 has FLAG=$green_flag (0x$(printf '%x' $green_flag)); expected SAM primary (FLAG & 0x900 == 0)" +ok "GREEN: chrP POS=$LIFT_POS1 is the SAM primary record (FLAG=$green_flag, FLAG & 0x900 = 0)" + +# The reverse strand bit must be set (FLAG & 0x10): the projected anchor is +# reverse (folded_rev = blk_rev ^ alt_rev = 1 ^ 0 = 1). +rev_bit=$(( green_flag & 0x10 )) +[ "$rev_bit" -ne 0 ] || fail "GREEN: chrP POS=$LIFT_POS1 is NOT on the reverse strand (FLAG=$green_flag); expected FLAG&0x10 set (folded_rev=1)" +ok "GREEN: chrP POS=$LIFT_POS1 is on the reverse strand (FLAG&0x10 set, folded_rev=blk_rev^alt_rev=1)" + +# The ALT twin chrP_altS must be present as a secondary (subordinate to chrP). +alt_in_green=$(mawk '$1!~/^@/ && $3=="chrP_altS"{print "1"; exit}' "$TMPD/green.sam") +[ "${alt_in_green:-0}" = "1" ] || fail "GREEN: chrP_altS secondary missing (ALT seed should still produce an ALT alignment)" +ok "GREEN: chrP_altS present as secondary subordinate in the liftover group" + +# Determinism: chrP POS=LIFT_POS1 MAPQ is stable across runs. +"$MINIBWA" map -c "$C" --outn=999 "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/green2.sam" +green2_mapq=$(mapq_at_pos "$TMPD/green2.sam" "$LIFT_POS1") +[ "$green2_mapq" = "$green_mapq" ] \ + || fail "GREEN: chrP POS=$LIFT_POS1 MAPQ not deterministic (run1=$green_mapq run2=$green2_mapq)" +ok "GREEN: MAPQ=$green_mapq at LIFT POS=$LIFT_POS1 deterministic across runs" + +# ========================================================================= +# --- baseline: chrM (no .alt) -- projection is a strict no-op; SE + PE --- +echo "[test-projrev] chrM baseline (no .alt; projection inert) ..." +CHRM_FA="$MDIR/test/chrM-human.fa.gz" +CHRM_R1="$MDIR/test/chrM-read_1.fa.gz" +CHRM_R2="$MDIR/test/chrM-read_2.fa.gz" +if [ -f "$CHRM_FA" ] && [ -f "$CHRM_R1" ]; then + # Copy and index inside TMPD rather than beside the shared fixture (same reason as + # lib-baseline.sh): keeps the check hermetic, avoids racing another script indexing + # the same chrM, and leaves no untracked .mbw/.l2b in test/. + cp "$CHRM_FA" "$TMPD/chrM-ref.fa.gz"; CHRM_FA="$TMPD/chrM-ref.fa.gz" + "$MINIBWA" index "$CHRM_FA" 2>/dev/null + "$MINIBWA" map --outn=5 "$CHRM_FA" "$CHRM_R1" 2>/dev/null > "$TMPD/chrM-se-on.sam" + "$MINIBWA" map --dbg-no-alt-proj --outn=5 "$CHRM_FA" "$CHRM_R1" 2>/dev/null > "$TMPD/chrM-se-off.sam" + [ -s "$TMPD/chrM-se-on.sam" ] || fail "chrM SE baseline: empty output" + grep -v '^@' "$TMPD/chrM-se-on.sam" > "$TMPD/chrM-se-on.body" || true + grep -v '^@' "$TMPD/chrM-se-off.sam" > "$TMPD/chrM-se-off.body" || true + cmp -s "$TMPD/chrM-se-on.body" "$TMPD/chrM-se-off.body" \ + || fail "chrM SE baseline: projection changed alignments without .alt" + ok "chrM SE baseline: identical alignments with/without projection (no .alt => inert)" + if [ -f "$CHRM_R2" ]; then + "$MINIBWA" map --outn=5 "$CHRM_FA" "$CHRM_R1" "$CHRM_R2" 2>/dev/null > "$TMPD/chrM-pe-on.sam" + "$MINIBWA" map --dbg-no-alt-proj --outn=5 "$CHRM_FA" "$CHRM_R1" "$CHRM_R2" 2>/dev/null > "$TMPD/chrM-pe-off.sam" + grep -v '^@' "$TMPD/chrM-pe-on.sam" > "$TMPD/chrM-pe-on.body" || true + grep -v '^@' "$TMPD/chrM-pe-off.sam" > "$TMPD/chrM-pe-off.body" || true + cmp -s "$TMPD/chrM-pe-on.body" "$TMPD/chrM-pe-off.body" \ + || fail "chrM PE baseline: projection changed alignments without .alt" + ok "chrM PE baseline: identical alignments with/without projection (no .alt => inert)" + else + echo " skip: chrM R2 not found ($CHRM_R2)" + fi +else + echo " skip: chrM baseline files not found ($CHRM_FA)" +fi + +echo "[test-projrev] PASS" +exit 0 diff --git a/test/altlg/test-reconcile.sh b/test/altlg/test-reconcile.sh new file mode 100755 index 0000000..8479757 --- /dev/null +++ b/test/altlg/test-reconcile.sh @@ -0,0 +1,236 @@ +#!/bin/sh +# Integration test for Task 4: mb_reconcile_alt (the reconciliation pass). +# +# Runs `minibwa mem` end-to-end on three synthetic fixtures and asserts on the +# SAM flags / MAPQ that the post-extension liftover-group reconciliation pass +# produces. Reconciliation groups surviving hits by lifted placement (same +# pri_tid / rev, |Δlifted_st| <= MB_LIFT_TOL), picks the best member per group, +# and recomputes the group-scoped suboptimal fields so MAPQ reflects the +# second-best GROUP — not an ALT twin of the same locus. +# +# Cases: +# (a) clean shared-flank read: chrP primary + identical chrP_altC twin. Before +# reconciliation the twin is a co-scoring competitor => MAPQ 0. After, +# they group => chrP is sam_pri (FLAG&0x100==0) with MAPQ>0; the twin is +# flagged secondary (0x100) or dropped. +# (b) PARALOG GUARD (boundary stress): two near-identical primary loci whose +# ALT twins lift to lifted_st 20bp apart (== 2*MB_LIFT_TOL). The groups +# must NOT merge => read stays a low-MAPQ multi-mapper (MAPQ 0). Proven by +# the placement dump showing altA/altB lst differ by > MB_LIFT_TOL. +# (c) homologous-divergent twin (lower score): best member (chrP) is the +# primary with MAPQ>0; the lower-scoring chrP_altD twin is in the SAME +# group, contributes nothing, and does NOT zero the MAPQ. +# (d) CHIMERA (locks the query-overlap requirement of the merge predicate): a +# read with two DISJOINT query segments whose hits lift to the SAME primary +# lifted_st (same pri_tid/strand, |Δlst| <= MB_LIFT_TOL) must NOT be merged +# — they are chimeric segments, not the same alignment on primary vs ALT. +# The ALT-contig hit for the second segment stays an INDEPENDENT group +# representative (parent == id), preserving the genuine competitor so MAPQ +# is not inflated. Before the fix the merge ignored query overlap and the +# segment-2 hit was demoted to a same-group subordinate (parent != id). +# Asserted via the ex-group-check probe (parent/subsc per hit). +# Baseline: chrM (no .alt) — mb_any_alt gate => pass never runs => byte-identical. +# +# Usage: test/altlg/test-reconcile.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" + +# Shared no-.alt baseline check (see lib-baseline.sh). +. "$(dirname "$0")/lib-baseline.sh" +EX_PLACE="$MDIR/api-test/ex-place-check" +EX_GROUP="$MDIR/api-test/ex-group-check" +MK_REC="$MDIR/test/altlg/mkfixture-reconcile.sh" +MK_PAR="$MDIR/test/altlg/mkfixture-paralog.sh" +MK_CHIM="$MDIR/test/altlg/mkfixture-chimera.sh" + +TMPD=$(mktemp -d /tmp/altlg-reconcile.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# flag_of -> FLAG of the first matching record +# has_bit -> "1" if (flag & bit) else "0". must be a single +# power-of-two (FLAG bits via integer arithmetic; mawk lacks a portable &). +# primary_flag -> FLAG of the record without 0x100 (secondary) +# and without 0x800 (supplementary) — i.e. the SAM primary line. + +# ========================================================================= +echo "[test-reconcile] building reconcile fixture ..." +/bin/sh "$MK_REC" "$TMPD" 2>/dev/null +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +echo "[test-reconcile] mapping reconcile reads (--outn=50) ..." +"$MINIBWA" map --outn=50 "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null \ + > "$TMPD/rec.full.sam" +mawk '$1 !~ /^@/' "$TMPD/rec.full.sam" > "$TMPD/rec.sam" +[ -s "$TMPD/rec.sam" ] || fail "reconcile mapping: no alignments emitted" +echo "----- reconcile SAM -----" +mawk '{printf "%s flag=%d %s pos=%s mapq=%s\n",$1,$2,$3,$4,$5}' "$TMPD/rec.sam" +echo "-------------------------" + +# --- case (a): clean shared-flank read --- +echo "== case (a): clean twin groups; chrP primary, MAPQ>0 ==" +ca_flag=$(flag_of "$TMPD/rec.sam" r-clean chrP) +ca_mapq=$(mapq_of "$TMPD/rec.sam" r-clean chrP) +[ -n "$ca_flag" ] || fail "(a) no chrP record for r-clean" +# chrP must be primary: 0x100 (256) secondary bit NOT set +sec=$(has_bit "$ca_flag" 256) +[ "$sec" = "0" ] || fail "(a) chrP r-clean is secondary (flag=$ca_flag); expected primary" +ok "(a) chrP r-clean is primary (flag=$ca_flag, 0x100 clear)" +[ "$ca_mapq" -gt 0 ] || fail "(a) chrP r-clean MAPQ=$ca_mapq; expected >0 after reconciliation" +ok "(a) chrP r-clean MAPQ=$ca_mapq > 0" +# the identical twin, if present, must be secondary (0x100 set) +ca_alt_flag=$(flag_of "$TMPD/rec.sam" r-clean chrP_altC) +if [ -n "$ca_alt_flag" ]; then + altsec=$(has_bit "$ca_alt_flag" 256) + [ "$altsec" = "1" ] || fail "(a) chrP_altC twin present but not secondary (flag=$ca_alt_flag)" + ok "(a) chrP_altC twin is secondary (flag=$ca_alt_flag, 0x100 set)" +else + ok "(a) chrP_altC twin absent (collapsed into primary group)" +fi + +# --- case (c): homologous-divergent twin --- +echo "== case (c): divergent twin does not zero the MAPQ ==" +# as_of -> integer of the AS:i: tag of the first match +cc_flag=$(flag_of "$TMPD/rec.sam" r-diverge chrP) +cc_mapq=$(mapq_of "$TMPD/rec.sam" r-diverge chrP) +[ -n "$cc_flag" ] || fail "(c) no chrP record for r-diverge" +sec=$(has_bit "$cc_flag" 256) +[ "$sec" = "0" ] || fail "(c) chrP r-diverge is secondary (flag=$cc_flag); expected primary (best member)" +ok "(c) chrP r-diverge is primary (best member; flag=$cc_flag)" +[ "$cc_mapq" -gt 0 ] || fail "(c) chrP r-diverge MAPQ=$cc_mapq; divergent twin should NOT zero it" +ok "(c) chrP r-diverge MAPQ=$cc_mapq > 0 (twin in same group, contributes nothing)" +# the lower-scoring divergent twin MUST survive to reconciliation, be flagged +# secondary, and score strictly lower than the primary (proving it is a real +# same-group member that contributes nothing rather than having been pruned). +ccx_flag=$(flag_of "$TMPD/rec.sam" r-diverge chrP_altD) +[ -n "$ccx_flag" ] || fail "(c) divergent twin chrP_altD absent; expected to survive as a secondary" +ccx_sec=$(has_bit "$ccx_flag" 256) +[ "$ccx_sec" = "1" ] || fail "(c) chrP_altD not secondary (flag=$ccx_flag)" +ok "(c) divergent twin chrP_altD present and secondary (flag=$ccx_flag)" +cc_as=$(as_of "$TMPD/rec.sam" r-diverge chrP) +ccx_as=$(as_of "$TMPD/rec.sam" r-diverge chrP_altD) +[ -n "$cc_as" ] || fail "(c) missing AS for chrP" +[ -n "$ccx_as" ] || fail "(c) missing AS for chrP_altD" +[ "$ccx_as" -lt "$cc_as" ] || fail "(c) divergent twin AS=$ccx_as not < primary AS=$cc_as" +ok "(c) divergent twin scores lower (AS $ccx_as < $cc_as) yet did not zero MAPQ" + +# ========================================================================= +# --- case (b): paralog guard (boundary stress) --- +echo "[test-reconcile] building paralog fixture ..." +PARD="$TMPD/para"; mkdir -p "$PARD" +/bin/sh "$MK_PAR" "$PARD" 2>/dev/null +"$MINIBWA" index "$PARD/ref.fa" 2>/dev/null + +echo "[test-reconcile] paralog placement dump ..." +"$EX_PLACE" "$PARD/ref.fa" "$PARD/reads.fq" all 2>/dev/null > "$PARD/place.txt" +cat "$PARD/place.txt" +# Prove the boundary: altA and altB lift to lifted_st > MB_LIFT_TOL apart. +lstA=$(mawk '$1=="r-para" && $2=="chrP_altA"{for(i=1;i<=NF;i++){n=index($i,"lst=");if(n==1){print substr($i,5);exit}}}' "$PARD/place.txt") +lstB=$(mawk '$1=="r-para" && $2=="chrP_altB"{for(i=1;i<=NF;i++){n=index($i,"lst=");if(n==1){print substr($i,5);exit}}}' "$PARD/place.txt") +[ -n "$lstA" ] && [ -n "$lstB" ] || fail "(b) could not read altA/altB lifted_st from placement dump" +dlt=$(( lstB - lstA )); [ "$dlt" -lt 0 ] && dlt=$(( -dlt )) +[ "$dlt" -gt 10 ] || fail "(b) altA lst=$lstA altB lst=$lstB only $dlt apart; fixture not stressing MB_LIFT_TOL boundary" +ok "(b) altA lst=$lstA altB lst=$lstB are $dlt bp apart (> MB_LIFT_TOL=10): distinct groups" + +echo "[test-reconcile] mapping paralog read (--outn=50) ..." +"$MINIBWA" map --outn=50 "$PARD/ref.fa" "$PARD/reads.fq" 2>/dev/null \ + > "$PARD/par.full.sam" +mawk '$1 !~ /^@/' "$PARD/par.full.sam" > "$PARD/par.sam" +[ -s "$PARD/par.sam" ] || fail "paralog mapping: no alignments emitted" +echo "----- paralog SAM -----" +mawk '{printf "%s flag=%d %s pos=%s mapq=%s\n",$1,$2,$3,$4,$5}' "$PARD/par.sam" +echo "-----------------------" +# The read maps to multiple distinct primary loci (COPY1, COPY2) plus altB's own +# group => genuine multi-mapper => MAPQ must stay 0 (groups not collapsed). +pb_mapq=$(primary_mapq "$PARD/par.sam" r-para) +[ -n "$pb_mapq" ] || fail "(b) no primary record for r-para" +[ "$pb_mapq" = "0" ] || fail "(b) r-para primary MAPQ=$pb_mapq; expected 0 (paralogs must not merge)" +ok "(b) r-para primary MAPQ=$pb_mapq (low-MAPQ multi-mapper; groups not merged)" + +# The MAPQ==0 check above is necessary but NOT sufficient: COPY1 and COPY2 are two +# distinct non-ALT primary loci, so they force MAPQ 0 regardless of what +# reconciliation does with altB — a wrongful merge of altB into COPY1's group +# would still leave two co-scoring groups and MAPQ 0, so it passes either way. +# Probe the grouping DIRECTLY: altB is 2*tol from COPY1 and MUST remain its own +# group representative (parent == id); a wrongful cross-tolerance merge would set +# its parent to COPY1's id. This is the assertion that actually exercises the +# paralog guard under test. +[ -x "$EX_GROUP" ] || fail "(b) ex-group-check probe not built ($EX_GROUP); run 'make -C api-test'" +echo "[test-reconcile] paralog grouping dump ..." +"$EX_GROUP" "$PARD/ref.fa" "$PARD/reads.fq" 2>/dev/null > "$PARD/grp.txt" +[ -s "$PARD/grp.txt" ] || fail "(b) ex-group-check emitted no grouping rows" +cat "$PARD/grp.txt" +grp_field() { # -> value of key= on the first row for that ctg + mawk -v c="$2" -v k="$3" ' + { ctg=""; want=""; for(i=1;i<=NF;i++){ n=index($i,"="); + if(n>0){ key=substr($i,1,n-1); val=substr($i,n+1); + if(key=="ctg") ctg=val; if(key==k) want=val; } } + if(ctg==c){ print want; exit } }' "$1" +} +b_id=$(grp_field "$PARD/grp.txt" chrP_altB id) +b_parent=$(grp_field "$PARD/grp.txt" chrP_altB parent) +[ -n "$b_id" ] && [ -n "$b_parent" ] || fail "(b) chrP_altB not present in grouping dump" +[ "$b_parent" = "$b_id" ] || fail "(b) chrP_altB merged into another group (parent=$b_parent != id=$b_id); paralog twin 2*tol away must NOT merge" +ok "(b) chrP_altB is its own group rep (parent=$b_parent == id=$b_id): paralog twin not merged across tolerance" + +# ========================================================================= +# --- case (d): chimera — disjoint query spans must NOT merge --- +# This case LOCKS the query-overlap requirement of mb_reconcile_alt's merge +# predicate. The fixture builds a read with two disjoint query segments whose +# hits lift to the SAME primary lifted_st: SEG1 -> chrP, SEG2 -> chrP_altE which +# is (deliberately, via .alt) aligned onto SEG1's primary window. They are +# co-located (same pri_tid/strand, |Δlst| <= MB_LIFT_TOL) but query-DISJOINT. +# +# Probed at the grouping level (SAM flags are insensitive here — chrP_altE is +# 0x100 secondary regardless, via mb_set_sam_pri's own overlap logic): +# BEFORE the fix: the merge ignored query overlap -> chrP_altE demoted to a +# same-group SUBORDINATE (parent == the SEG1 hit's id, != its +# own id), competitor suppressed (subsc == 0), MAPQ inflated. +# AFTER the fix: chrP_altE stays its own group REPRESENTATIVE (parent == id), +# its competing score is preserved (subsc > 0). +echo "[test-reconcile] building chimera fixture ..." +CHIM="$TMPD/chim"; mkdir -p "$CHIM" +/bin/sh "$MK_CHIM" "$CHIM" 2>/dev/null +"$MINIBWA" index "$CHIM/ref.fa" 2>/dev/null + +[ -x "$EX_GROUP" ] || fail "(d) ex-group-check probe not built ($EX_GROUP); run 'make -C api-test'" +echo "[test-reconcile] chimera grouping dump ..." +"$EX_GROUP" "$CHIM/ref.fa" "$CHIM/reads.fq" 2>/dev/null > "$CHIM/grp.txt" +[ -s "$CHIM/grp.txt" ] || fail "(d) ex-group-check emitted no grouping rows" +cat "$CHIM/grp.txt" + +# field_of -> value of key= on the first chrP_altE row, +# stripping the "key=" prefix. is_alt=1 disambiguates the ALT-contig hit. +field_of() { + mawk -v c="$2" -v k="$3" ' + { ctg=""; want=""; for(i=1;i<=NF;i++){ + n=index($i,"="); + if(n>0){ key=substr($i,1,n-1); val=substr($i,n+1); + if(key=="ctg") ctg=val; + if(key==k) want=val; } } + if(ctg==c){ print want; exit } }' "$1" +} +d_id=$(field_of "$CHIM/grp.txt" chrP_altE id) +d_parent=$(field_of "$CHIM/grp.txt" chrP_altE parent) +d_subsc=$(field_of "$CHIM/grp.txt" chrP_altE subsc) +[ -n "$d_id" ] || fail "(d) no chrP_altE hit in grouping dump" +[ -n "$d_parent" ] || fail "(d) chrP_altE missing parent field" +[ -n "$d_subsc" ] || fail "(d) chrP_altE missing subsc field" +# Core assertion: the disjoint-span ALT hit is its OWN representative, not a +# subordinate folded into the SEG1 group. parent == id <=> not merged. +[ "$d_parent" = "$d_id" ] || fail "(d) chrP_altE merged into another group (parent=$d_parent != id=$d_id); query-disjoint chimeric segments must NOT merge" +ok "(d) chrP_altE is its own group rep (parent=$d_parent == id=$d_id): disjoint spans not merged" +# And the genuine competitor's score is preserved (not zeroed by a wrongful merge). +[ "$d_subsc" -gt 0 ] || fail "(d) chrP_altE subsc=$d_subsc; competitor score should be preserved (>0) when kept separate" +ok "(d) chrP_altE competitor score preserved (subsc=$d_subsc > 0): MAPQ not inflated" + +# ========================================================================= +# --- baseline: chrM (no .alt) — mb_any_alt gate => byte-identical --- +echo "[test-reconcile] chrM baseline (no .alt): byte-identical to stock ..." +chrm_baseline "(BASELINE) chrM" se --outn=5 + +echo "[test-reconcile] PASS" +exit 0 diff --git a/test/altlg/test-revmulti.sh b/test/altlg/test-revmulti.sh new file mode 100644 index 0000000..00967b7 --- /dev/null +++ b/test/altlg/test-revmulti.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Regression test for reverse-multi-block and multi-record/hard-clip .alt lifting. +# Drives ex-lift-check in check mode: each " " triple asserts +# l2b_lift(ctg, pos) maps to primary , and that the contig's lift[] is +# sorted by alt_st (the invariant l2b_lift's binary search relies on). +# +# Expected primary coordinates (0-based), derived by hand from the .alt CIGARs: +# chrPrev_alt (reverse, 75M2000D75M @ chrP POS 201 -> 0-based 200): +# forward-ALT base 0 = RC base 149 -> primary 2349 +# forward-ALT base 74 = RC base 75 -> primary 2275 +# forward-ALT base 75 = RC base 74 -> primary 274 +# forward-ALT base 149 = RC base 0 -> primary 200 +# chrPhc_alt (50M @1001 + 50H50M @2001, both forward; POS 0-based 1000/2000): +# base 0 -> 1000 ; base 49 -> 1049 ; base 50 -> 2000 ; base 99 -> 2049 +# Usage: test/altlg/test-revmulti.sh [] +set -eu + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MKFIXTURE="$MDIR/test/altlg/mkfixture-revmulti.sh" +MINIBWA="$MDIR/minibwa" +EX_LIFT="$MDIR/api-test/ex-lift-check" + +TMPD=$(mktemp -d /tmp/altlg-test.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + +echo "[test-revmulti] building fixture in $TMPD ..." +/bin/sh "$MKFIXTURE" "$TMPD" + +echo "[test-revmulti] indexing with minibwa ..." +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +echo "[test-revmulti] running ex-lift-check (check mode) ..." +# ex-lift-check exits non-zero on any MISMATCH or unsorted lift[]. +OUT=$("$EX_LIFT" "$TMPD/ref.fa" \ + chrPrev_alt 0 2349 \ + chrPrev_alt 74 2275 \ + chrPrev_alt 75 274 \ + chrPrev_alt 149 200 \ + chrPhc_alt 0 1000 \ + chrPhc_alt 49 1049 \ + chrPhc_alt 50 2000 \ + chrPhc_alt 99 2049) +echo "$OUT" | sed 's/^/ /' + +if echo "$OUT" | grep -q MISMATCH; then + echo "FAIL: at least one lift MISMATCH (see above)" + exit 1 +fi +if echo "$OUT" | grep -q 'sorted=0'; then + echo "FAIL: a contig's lift[] is not sorted by alt_st" + exit 1 +fi + +echo "[test-revmulti] PASS: reverse-multi-block and hard-clip multi-record .alt lift correctly" +exit 0 diff --git a/test/altlg/test-segdup-rev.sh b/test/altlg/test-segdup-rev.sh new file mode 100755 index 0000000..4866ac5 --- /dev/null +++ b/test/altlg/test-segdup-rev.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# Regression test for Task 7 fix round 1, Fix 1 + Fix 2: REVERSE-strand ALT-seed +# -> primary projection. +# +# The original forward-only segdup fixture never exercised a reverse .alt block, +# so two reverse coordinate bugs slipped through: +# Fix 1: the ALT contig-local forward span recovery was forward-only. +# Fix 2: the projected primary tpos was re-folded for the reverse strand, but +# process_batch keeps the position component in the FORWARD contig frame +# for both strands -> the lifted POS was the mirror image (wrong end of +# the contig) instead of the true locus. +# +# This test uses a REVERSE .alt block (FLAG 0x10) and asserts, via the +# MB_PROJ_TRACE probe seam, that the surviving full-read ALT seed projects to the +# CORRECT primary coordinate on the REVERSE strand: chrP POS=LIFT_POS1, strand -. +# +# Why a probe instead of a pure SAM assertion: under a reverse (RC) alignment the +# shared CORE collapses to one of many identical-scoring paralog loci, so the +# projected reverse anchor is masked at SAM level by paralog selection (the same +# happens with projection OFF -- the LIFT locus simply isn't one of the few +# secondaries minibwa surfaces). The probe asserts the load-bearing coordinate +# the production projection actually computes, which is exactly what Fix 1+2 set. +# RED (projection off) emits NO probe line; GREEN (default) emits the correct one. +# +# Usage: test/altlg/test-segdup-rev.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" +MK="$MDIR/test/altlg/mkfixture-segdup-rev.sh" + +C=4 +N=20 + +TMPD=$(mktemp -d /tmp/altlg-segdup-rev.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# proj_line -> the MB_PROJ POS for chrP on that strand, +# for the FULL-read seed (len=150); empty if none. +proj_pos() { mawk -v s="$2" '$1=="MB_PROJ" && $2=="chrP" && $4==s && $5=="len=150"{print $3}' "$1"; } + +echo "[test-segdup-rev] building reverse segdup fixture (N=$N) ..." +/bin/sh "$MK" "$TMPD" "$N" 2>/dev/null +. "$TMPD/meta.txt" +[ -n "${LIFT_POS1:-}" ] || fail "fixture did not export LIFT_POS1" +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null +echo "[test-segdup-rev] expected projected locus: chrP POS=$LIFT_POS1 strand=- (full-read seed)" + +# --- RED: projection disabled -> no projected anchor at all --- +echo "== RED: MB_NO_ALT_PROJECT=1 (no projection) ==" +"$MINIBWA" map --dbg-no-alt-proj --dbg-alt-proj -c "$C" --outn=999 \ + "$TMPD/ref.fa" "$TMPD/reads.fq" 2>"$TMPD/red.err" >/dev/null +red_pos=$(proj_pos "$TMPD/red.err" -) +[ -z "$red_pos" ] || fail "RED: projection emitted a trace with seam off (got POS=$red_pos)" +ok "RED: no projected anchor with projection disabled" + +# --- GREEN: projection enabled (default) --- +echo "== GREEN: projection enabled (default) ==" +"$MINIBWA" map --dbg-alt-proj -c "$C" --outn=999 \ + "$TMPD/ref.fa" "$TMPD/reads.fq" 2>"$TMPD/green.err" >/dev/null +echo " MB_PROJ traces (GREEN):" +grep '^MB_PROJ' "$TMPD/green.err" | sed 's/^/ /' || true +green_pos=$(proj_pos "$TMPD/green.err" -) +[ -n "$green_pos" ] || fail "GREEN: no reverse-strand projected anchor for the full-read seed" +echo " expected POS=$LIFT_POS1 strand=-, actual POS=$green_pos strand=-" +[ "$green_pos" = "$LIFT_POS1" ] \ + || fail "GREEN: reverse projection landed at POS=$green_pos, expected $LIFT_POS1 (Fix 1/2 coordinate bug)" +ok "GREEN: reverse ALT seed projected to the CORRECT primary locus (POS=$LIFT_POS1, strand -)" + +# Determinism. +"$MINIBWA" map --dbg-alt-proj -c "$C" --outn=999 \ + "$TMPD/ref.fa" "$TMPD/reads.fq" 2>"$TMPD/green2.err" >/dev/null +[ "$(proj_pos "$TMPD/green2.err" -)" = "$green_pos" ] \ + || fail "GREEN: reverse projected POS not deterministic across runs" +ok "GREEN: reverse projected POS deterministic across runs" + +echo "[test-segdup-rev] PASS" +exit 0 diff --git a/test/altlg/test-segdup.sh b/test/altlg/test-segdup.sh new file mode 100755 index 0000000..9b68e33 --- /dev/null +++ b/test/altlg/test-segdup.sh @@ -0,0 +1,111 @@ +#!/bin/sh +# Integration test for Task 7: ALT-seed -> primary anchor projection. +# +# Proves the segdup-recovery mechanism end-to-end on a single binary by toggling +# the projection with the MB_NO_ALT_PROJECT test seam: +# +# RED (MB_NO_ALT_PROJECT=1): a segduplicated primary locus whose CORE seed is +# subsampled out by `-c` produces NO primary candidate -> there is NO chrP +# record at the ALT-lifted locus (POS 201). +# GREEN (default): the surviving, unique ALT seed lifts to that locus and +# injects a primary anchor -> a chrP record at POS 201 appears. +# +# Also asserts the chrM baseline is byte-identical with/without the seam (no .alt +# => no ALT anchors => projection is a strict no-op), for SE and PE. +# +# Usage: test/altlg/test-segdup.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" +MK_SEG="$MDIR/test/altlg/mkfixture-segdup.sh" + +C=4 # max_occ: < N+1 so the CORE seed interval is subsampled +N=20 # primary copies of CORE + +TMPD=$(mktemp -d /tmp/altlg-segdup.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# chrP_pos_list -> sorted space-separated list of chrP POS values +chrP_pos_list() { mawk '$1!~/^@/ && $3=="chrP"{print $4}' "$1" | sort -n | tr '\n' ' '; } +# has_pos -> "1" if any chrP record has that POS, else "0" + +echo "[test-segdup] building segdup fixture (N=$N copies) ..." +/bin/sh "$MK_SEG" "$TMPD" "$N" 2>/dev/null +. "$TMPD/meta.txt" # sets LIFT_POS1 +[ -n "${LIFT_POS1:-}" ] || fail "fixture did not export LIFT_POS1" +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +echo "[test-segdup] LIFT locus is chrP POS=$LIFT_POS1; max_occ (-c) = $C" + +# --- RED: projection disabled --- +echo "== RED: MB_NO_ALT_PROJECT=1 (no projection) ==" +"$MINIBWA" map --dbg-no-alt-proj -c "$C" --outn=999 "$TMPD/ref.fa" "$TMPD/reads.fq" \ + 2>/dev/null > "$TMPD/red.sam" +mawk '$1 !~ /^@/' "$TMPD/red.sam" > "$TMPD/red.body.sam" +[ -s "$TMPD/red.body.sam" ] || fail "RED: no alignments emitted" +echo " chrP POS (RED): $(chrP_pos_list "$TMPD/red.sam")" +red_hit=$(has_pos "$TMPD/red.sam" "$LIFT_POS1") +[ "$red_hit" = "0" ] || fail "RED: a chrP record exists at LIFT POS=$LIFT_POS1 without projection; fixture is not forcing the segdup-subsample drop (try a smaller -c or larger N)" +ok "RED: no chrP candidate at LIFT POS=$LIFT_POS1 (segdup seed subsampled out)" + +# --- GREEN: projection enabled (default) --- +echo "== GREEN: projection enabled (default) ==" +"$MINIBWA" map -c "$C" --outn=999 "$TMPD/ref.fa" "$TMPD/reads.fq" \ + 2>/dev/null > "$TMPD/green.sam" +echo " chrP POS (GREEN): $(chrP_pos_list "$TMPD/green.sam")" +green_hit=$(has_pos "$TMPD/green.sam" "$LIFT_POS1") +[ "$green_hit" = "1" ] || fail "GREEN: projection did not produce a chrP candidate at LIFT POS=$LIFT_POS1" +ok "GREEN: chrP candidate recovered at LIFT POS=$LIFT_POS1 (ALT seed projected to primary)" + +# The ALT twin itself must still be present (sanity: the ALT seed survived). +alt_present=$(mawk '$1!~/^@/ && $3=="chrP_altS"{print "1"; exit}' "$TMPD/green.sam") +[ "${alt_present:-0}" = "1" ] || fail "GREEN: chrP_altS ALT hit missing (ALT seed did not survive)" +ok "GREEN: chrP_altS ALT hit present (the projected anchor's source seed)" + +# Determinism: GREEN chrP POS list is stable across runs. +"$MINIBWA" map -c "$C" --outn=999 "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/green2.sam" +[ "$(chrP_pos_list "$TMPD/green.sam")" = "$(chrP_pos_list "$TMPD/green2.sam")" ] \ + || fail "GREEN: chrP POS list not deterministic across runs" +ok "GREEN: chrP POS list deterministic across runs" + +# ========================================================================= +# --- baseline: chrM (no .alt) -- projection is a strict no-op; SE + PE --- +echo "[test-segdup] chrM baseline (no .alt; projection inert) ..." +CHRM_FA="$MDIR/test/chrM-human.fa.gz" +CHRM_R1="$MDIR/test/chrM-read_1.fa.gz" +CHRM_R2="$MDIR/test/chrM-read_2.fa.gz" +if [ -f "$CHRM_FA" ] && [ -f "$CHRM_R1" ]; then + # Copy and index inside TMPD rather than beside the shared fixture (same reason as + # lib-baseline.sh): keeps the check hermetic, avoids racing another script indexing + # the same chrM, and leaves no untracked .mbw/.l2b in test/. + cp "$CHRM_FA" "$TMPD/chrM-ref.fa.gz"; CHRM_FA="$TMPD/chrM-ref.fa.gz" + "$MINIBWA" index "$CHRM_FA" 2>/dev/null + # SE: default vs seam-off must be byte-identical (no ALT anchors either way). + "$MINIBWA" map --outn=5 "$CHRM_FA" "$CHRM_R1" 2>/dev/null > "$TMPD/chrM-se-on.sam" + "$MINIBWA" map --dbg-no-alt-proj --outn=5 "$CHRM_FA" "$CHRM_R1" 2>/dev/null > "$TMPD/chrM-se-off.sam" + [ -s "$TMPD/chrM-se-on.sam" ] || fail "chrM SE baseline: empty output" + grep -v '^@' "$TMPD/chrM-se-on.sam" > "$TMPD/chrM-se-on.body" || true + grep -v '^@' "$TMPD/chrM-se-off.sam" > "$TMPD/chrM-se-off.body" || true + cmp -s "$TMPD/chrM-se-on.body" "$TMPD/chrM-se-off.body" \ + || fail "chrM SE baseline: projection changed alignments without .alt" + ok "chrM SE baseline: identical alignments with/without projection (no .alt => inert)" + if [ -f "$CHRM_R2" ]; then + "$MINIBWA" map --outn=5 "$CHRM_FA" "$CHRM_R1" "$CHRM_R2" 2>/dev/null > "$TMPD/chrM-pe-on.sam" + "$MINIBWA" map --dbg-no-alt-proj --outn=5 "$CHRM_FA" "$CHRM_R1" "$CHRM_R2" 2>/dev/null > "$TMPD/chrM-pe-off.sam" + grep -v '^@' "$TMPD/chrM-pe-on.sam" > "$TMPD/chrM-pe-on.body" || true + grep -v '^@' "$TMPD/chrM-pe-off.sam" > "$TMPD/chrM-pe-off.body" || true + cmp -s "$TMPD/chrM-pe-on.body" "$TMPD/chrM-pe-off.body" \ + || fail "chrM PE baseline: projection changed alignments without .alt" + ok "chrM PE baseline: identical alignments with/without projection (no .alt => inert)" + else + echo " skip: chrM R2 not found ($CHRM_R2)" + fi +else + echo " skip: chrM baseline files not found ($CHRM_FA)" +fi + +echo "[test-segdup] PASS" +exit 0 diff --git a/test/altlg/test-survive.sh b/test/altlg/test-survive.sh new file mode 100755 index 0000000..a0c8746 --- /dev/null +++ b/test/altlg/test-survive.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# Integration test for Task 3 (survival guard in mb_select_sub). +# +# Fixture: chrP (400bp unique sequence) + chrP_alt (ALT = chrP with 1bp SNP at +# 1-based position 126, putting the SNP at read position 55 of the 100bp read +# chrP[70,170)). +# +# With -p 0.9 (pri_ratio=0.9) the chrP_alt twin chains LOWER (sub-SMEM score +# ~55 vs primary ~100) and fails both the score-ratio and min-diff checks in +# mb_select_sub (pre-DP at :617), so it is DROPPED without the guard. +# With the survival guard the twin is force-kept because its lifted placement +# co-locates with the kept chrP primary (same pri_tid=chrP, same rev, |Δlst|=0). +# +# GREEN assertion (guard present): chrP_alt IS in SAM output. +# RED assertion (guard ablated): with --dbg-no-alt-survive the guard is skipped +# and chrP_alt must be ABSENT. Asserting both states proves the fixture actually +# exercises the guard's drop path (not that chrP_alt survives for some unrelated +# reason), so the test cannot silently stop stressing the guard. +# +# Baseline (chrM, no .alt): guard never fires; output byte-identical without +# guard because l2b has no ALT contigs (guard gated on l2b ALT presence). +# +# Usage: test/altlg/test-survive.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MKFIXTURE="$MDIR/test/altlg/mkfixture-survive.sh" +MINIBWA="$MDIR/minibwa" + +# Shared no-.alt baseline check (see lib-baseline.sh). +. "$(dirname "$0")/lib-baseline.sh" + +TMPD=$(mktemp -d /tmp/altlg-survive.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# --------------------------------------------------------------------------- +# Part 1: survival fixture (unique chrP + chrP_alt with 1bp SNP at read pos 55) +# --------------------------------------------------------------------------- +echo "[test-survive] building fixture ..." +/bin/sh "$MKFIXTURE" "$TMPD" 2>/dev/null + +echo "[test-survive] indexing ..." +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null + +# Map with -p 0.9 (high pri_ratio to drop the ~55bp chain) --outn=50 +echo "[test-survive] mapping with -p 0.9 --outn=50 ..." +"$MINIBWA" map -p 0.9 --outn=50 "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null \ + > "$TMPD/out.sam" + +echo "----- SAM output (-p 0.9 --outn=50) -----" +grep -v "^@" "$TMPD/out.sam" +echo "----- end -----" + +# Primary alignment on chrP must always be present +pri_present=$(mawk '$1=="r-survive" && $3=="chrP" && ($2+0 == 0) {found=1} END{print found+0}' \ + "$TMPD/out.sam") +if [ "$pri_present" != "1" ]; then + fail "primary chrP alignment should always be present (got: $pri_present)" +fi +ok "primary chrP alignment present (RNAME=chrP, FLAG=0)" + +# chrP_alt alignment must be PRESENT: the guard force-kept it. +# (Without the guard, this fails: RED state.) +alt_present=$(mawk '$1=="r-survive" && $3=="chrP_alt" {found=1} END{print found+0}' \ + "$TMPD/out.sam") +if [ "$alt_present" != "1" ]; then + fail "chrP_alt alignment should survive (guard keeps it); got: absent [RED: guard not working]" +fi +ok "chrP_alt alignment survived through guard (GREEN)" + +# Also verify the lifted placement of chrP_alt co-locates with the primary: +# both map to chrP at approximately the same position. +pri_pos=$(mawk '$1=="r-survive" && $3=="chrP" && ($2+0==0) {print $4}' "$TMPD/out.sam") +alt_pos=$(mawk '$1=="r-survive" && $3=="chrP_alt" {print $4}' "$TMPD/out.sam" | head -1) +if [ -n "$pri_pos" ] && [ -n "$alt_pos" ]; then + d=$(( alt_pos - pri_pos )) + [ "$d" -lt 0 ] && d=$(( -d )) + if [ "$d" -gt 10 ]; then + fail "chrP_alt POS=$alt_pos not co-located with chrP POS=$pri_pos (|diff|=$d > 10)" + fi + ok "chrP_alt POS=$alt_pos co-locates with chrP POS=$pri_pos (|diff|=$d <= 10)" +fi + +# --- RED state: ablate ONLY the survival guard and confirm chrP_alt is dropped. +# This proves the fixture geometry genuinely drives the twin below the +# score-ratio/min-diff thresholds, so the GREEN assertion above is meaningful. --- +echo "[test-survive] mapping with --dbg-no-alt-survive (guard ablated) ..." +"$MINIBWA" map -p 0.9 --outn=50 --dbg-no-alt-survive "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null \ + > "$TMPD/out.red.sam" +echo "----- SAM output (guard ablated) -----" +grep -v "^@" "$TMPD/out.red.sam" +echo "----- end -----" +red_pri=$(mawk '$1=="r-survive" && $3=="chrP" && ($2+0==0) {found=1} END{print found+0}' "$TMPD/out.red.sam") +[ "$red_pri" = "1" ] || fail "(RED) primary chrP alignment should still be present with the guard ablated" +red_alt=$(mawk '$1=="r-survive" && $3=="chrP_alt" {found=1} END{print found+0}' "$TMPD/out.red.sam") +if [ "$red_alt" != "0" ]; then + fail "(RED) chrP_alt should be DROPPED with the guard ablated, but it survived; the fixture is not exercising the guard's drop path" +fi +ok "chrP_alt dropped when guard ablated (RED): fixture provably stresses the survival guard" + +# --------------------------------------------------------------------------- +# Part 2: chrM baseline — guard never fires (no .alt -> l2b has no ALT ctgs) +# --------------------------------------------------------------------------- +echo "[test-survive] chrM baseline (no .alt): byte-identical to stock ..." +chrm_baseline "(BASELINE) chrM" se --outn=5 + +echo "[test-survive] PASS" +exit 0 diff --git a/test/altlg/test-svbreak.sh b/test/altlg/test-svbreak.sh new file mode 100755 index 0000000..c9c2b54 --- /dev/null +++ b/test/altlg/test-svbreak.sh @@ -0,0 +1,144 @@ +#!/bin/sh +# Integration test: SV-breakpoint-aware ALT grouping (multi-interval mb_place_t). +# +# THE BUG (verified on real WGS): an ALT contig whose .alt CIGAR carries an +# SV-scale indel describes two M blocks lifting to primary positions thousands of +# bp apart. A read whose ALT footprint spans the breakpoint collapsed (under the +# old single-interval placement) to a lifted_st thousands of bp from the read's +# PRIMARY twin -> the ALT hit failed to co-locate with the twin -> they competed +# -> a FALSE MAPQ 0 (bwa-postalt keeps these confident). +# +# THE FIX (Option 1, multi-interval): mb_hit_place records one sub-placement per +# overlapping .alt lift block; two hits co-locate iff ANY sub-interval pair shares +# (pri_tid, rev, |Δst| <= MB_LIFT_TOL). The breakpoint-spanning ALT hit then keeps +# a sub-interval AT the twin's position -> they group -> MAPQ recovered. +# +# Cases: +# (fwd) forward .alt block: read maps forward to its primary twin. RED before +# the fix (primary MAPQ 0); GREEN after (primary MAPQ>0, ALT secondary). +# (rev) reverse .alt block (FLAG 0x10): read maps reverse to its primary twin. +# Exercises the reverse-fold lift paths. Same RED->GREEN assertion. +# (par) PARALOG SAFETY: two distinct primary loci 7kb apart (same contig/strand, +# overlapping query) + an ALT twin co-located with ONE of them. The OTHER +# primary stays a distinct competing group -> MAPQ MUST remain 0. Proves +# the multi-interval change does NOT merge two far-apart primary loci. +# Baseline: chrM (no .alt) — mb_any_alt gate => pass never runs => byte-identical. +# +# Usage: test/altlg/test-svbreak.sh [] +set -eu +. "$(dirname "$0")/lib.sh" + +MDIR="${1:-$(cd "$(dirname "$0")/../.." && pwd)}" +MINIBWA="$MDIR/minibwa" + +# Shared no-.alt baseline check (see lib-baseline.sh). +. "$(dirname "$0")/lib-baseline.sh" +MK_SV="$MDIR/test/altlg/mkfixture-svbreak.sh" + +TMPD=$(mktemp -d /tmp/altlg-svbreak.XXXXXX) +trap 'rm -rf "$TMPD"' EXIT + + +# primary_flag/_mapq -> the SAM-primary record (no 0x100, no 0x800) +# secondary present on a given contig (0x100 set)? prints the FLAG of the first. +alt_secondary_flag() { + mawk -v q="$2" -v c="$3" '$1==q && $3==c { f=int($2); + if (int(f/256)%2==1) { print $2; exit } }' "$1" +} + +# ========================================================================= +echo "[test-svbreak] building svbreak fixture ..." +/bin/sh "$MK_SV" "$TMPD" 2>/dev/null + +# ---------------- forward variant ---------------- +echo "== case (fwd): breakpoint-spanning ALT groups with forward primary twin ==" +"$MINIBWA" index "$TMPD/ref.fa" 2>/dev/null +"$MINIBWA" map --outn=50 "$TMPD/ref.fa" "$TMPD/reads.fq" 2>/dev/null > "$TMPD/fwd.full.sam" +mawk '$1 !~ /^@/' "$TMPD/fwd.full.sam" > "$TMPD/fwd.sam" +[ -s "$TMPD/fwd.sam" ] || fail "(fwd) no alignments emitted" +echo "----- fwd SAM -----" +mawk '{printf "%s flag=%d %s pos=%s mapq=%s cig=%s\n",$1,$2,$3,$4,$5,$6}' "$TMPD/fwd.sam" +echo "-------------------" +fwd_flag=$(primary_flag "$TMPD/fwd.sam" r-sv-fwd) +fwd_mapq=$(primary_mapq "$TMPD/fwd.sam" r-sv-fwd) +[ -n "$fwd_mapq" ] || fail "(fwd) no SAM-primary record for r-sv-fwd" +# The SAM-primary must be the primary-contig twin (chrP), forward strand. +fwd_rname=$(mawk -v q=r-sv-fwd '$1==q { f=int($2); + if (int(f/256)%2==0 && int(f/2048)%2==0) { print $3; exit } }' "$TMPD/fwd.sam") +[ "$fwd_rname" = "chrP" ] || fail "(fwd) SAM-primary on $fwd_rname, expected chrP twin" +ok "(fwd) SAM-primary is chrP twin (flag=$fwd_flag)" +[ "$fwd_mapq" -gt 0 ] || fail "(fwd) chrP twin MAPQ=$fwd_mapq; expected >0 (breakpoint ALT must group, not compete)" +ok "(fwd) chrP twin MAPQ=$fwd_mapq > 0 (grouped with breakpoint-spanning ALT)" +fwd_alt=$(alt_secondary_flag "$TMPD/fwd.sam" r-sv-fwd chrP_altV) +[ -n "$fwd_alt" ] || fail "(fwd) chrP_altV not present as secondary; expected grouped/demoted" +ok "(fwd) chrP_altV ALT twin demoted to secondary (flag=$fwd_alt)" + +# ---------------- reverse variant ---------------- +echo "== case (rev): breakpoint-spanning ALT groups with reverse primary twin ==" +"$MINIBWA" index "$TMPD/rev/ref.fa" 2>/dev/null +"$MINIBWA" map --outn=50 "$TMPD/rev/ref.fa" "$TMPD/rev/reads.fq" 2>/dev/null > "$TMPD/rev.full.sam" +mawk '$1 !~ /^@/' "$TMPD/rev.full.sam" > "$TMPD/rev.sam" +[ -s "$TMPD/rev.sam" ] || fail "(rev) no alignments emitted" +echo "----- rev SAM -----" +mawk '{printf "%s flag=%d %s pos=%s mapq=%s cig=%s\n",$1,$2,$3,$4,$5,$6}' "$TMPD/rev.sam" +echo "-------------------" +rev_flag=$(primary_flag "$TMPD/rev.sam" r-sv-rev) +rev_mapq=$(primary_mapq "$TMPD/rev.sam" r-sv-rev) +[ -n "$rev_mapq" ] || fail "(rev) no SAM-primary record for r-sv-rev" +rev_rname=$(mawk -v q=r-sv-rev '$1==q { f=int($2); + if (int(f/256)%2==0 && int(f/2048)%2==0) { print $3; exit } }' "$TMPD/rev.sam") +[ "$rev_rname" = "chrP" ] || fail "(rev) SAM-primary on $rev_rname, expected chrP twin" +ok "(rev) SAM-primary is chrP twin (flag=$rev_flag)" +# confirm the twin maps on the reverse strand (0x10 set) +rev_is_rev=$(mawk -v f="$rev_flag" 'BEGIN{ print (int(int(f)/16)%2==1)?1:0 }') +[ "$rev_is_rev" = "1" ] || fail "(rev) chrP twin not reverse-strand (flag=$rev_flag); fixture/lift mismatch" +ok "(rev) chrP twin is reverse-strand (flag=$rev_flag, 0x10 set)" +[ "$rev_mapq" -gt 0 ] || fail "(rev) chrP twin MAPQ=$rev_mapq; expected >0 (reverse breakpoint ALT must group)" +ok "(rev) chrP twin MAPQ=$rev_mapq > 0 (grouped via reverse-folded sub-interval)" +rev_alt=$(alt_secondary_flag "$TMPD/rev.sam" r-sv-rev chrP_altV) +[ -n "$rev_alt" ] || fail "(rev) chrP_altV not present as secondary; expected grouped/demoted" +ok "(rev) chrP_altV ALT twin demoted to secondary (flag=$rev_alt)" + +# ========================================================================= +# --- case (par): paralog safety — two primary loci 7kb apart + ALT --- +echo "== case (par): two primary loci 7kb apart + ALT must STAY MAPQ 0 ==" +PARD="$TMPD/par"; mkdir -p "$PARD" +python3 - "$PARD" <<'PY' +import random, sys +d = sys.argv[1] +def gen(seed, n): + random.seed(seed); return ''.join(random.choice('ACGT') for _ in range(n)) +COPY = gen(2, 150) # identical paralog COPY +PAD0 = gen(1, 200) +GAP = gen(3, 7000) # the two primary loci are 7kb apart +PADZ = gen(5, 200) +CHRP = PAD0 + COPY + GAP + COPY + PADZ +open(d + "/ref.fa", "w").write(">chrP\n%s\n>chrP_altA\n%s\n" % (CHRP, COPY)) +# ALT co-locates with COPY1 (1-based POS 201). COPY2 (7kb away) stays distinct. +open(d + "/ref.fa.alt", "w").write("chrP_altA\t0\tchrP\t201\t60\t150M\t*\t0\t0\t*\t*\n") +open(d + "/reads.fq", "w").write("@r-par7k\n%s\n+\n%s\n" % (COPY, "I" * 150)) +PY +"$MINIBWA" index "$PARD/ref.fa" 2>/dev/null +"$MINIBWA" map --outn=50 "$PARD/ref.fa" "$PARD/reads.fq" 2>/dev/null > "$PARD/par.full.sam" +mawk '$1 !~ /^@/' "$PARD/par.full.sam" > "$PARD/par.sam" +[ -s "$PARD/par.sam" ] || fail "(par) no alignments emitted" +echo "----- par SAM -----" +mawk '{printf "%s flag=%d %s pos=%s mapq=%s cig=%s\n",$1,$2,$3,$4,$5,$6}' "$PARD/par.sam" +echo "-------------------" +# Sanity: the read must produce >= 2 primary-contig hits (the two paralog loci) so +# the multi-mapper verdict is genuinely about distinct primaries, not a missed hit. +n_chrP=$(mawk -v q=r-par7k '$1==q && $3=="chrP"{c++} END{print c+0}' "$PARD/par.sam") +[ "$n_chrP" -ge 2 ] || fail "(par) only $n_chrP chrP hits; fixture must expose two distinct primary loci" +ok "(par) two distinct chrP primary loci present ($n_chrP hits)" +par_mapq=$(primary_mapq "$PARD/par.sam" r-par7k) +[ -n "$par_mapq" ] || fail "(par) no SAM-primary record for r-par7k" +[ "$par_mapq" = "0" ] || fail "(par) r-par7k MAPQ=$par_mapq; expected 0 (distinct primary loci must NOT merge)" +ok "(par) r-par7k MAPQ=$par_mapq (paralogs not merged; multi-interval change is locus-safe)" + +# ========================================================================= +# --- baseline: chrM (no .alt) — mb_any_alt gate => byte-identical --- +echo "[test-svbreak] chrM baseline (no .alt): byte-identical to stock ..." +chrm_baseline "(BASELINE) chrM" se --outn=5 + +echo "[test-svbreak] PASS" +exit 0