Add verkko-fillet 0.1.25 to bio compiled software

ansible/roles/compiled/README.md

@@ -31,10 +31,10 @@ Software Categories
 
 ### Bioinformatics (`bio/`)
 - **Analysis Tools**: SAMtools, BCFtools, GATK, VEP, Picard
-- **Aligners**: BWA, Bowtie2, STAR, Minimap2, HISAT2
+- **Aligners**: BWA, Bowtie2, STAR, Minimap2, HISAT2, MashMap
 - **Quality Control**: FastQC, MultiQC, TrimGalore
 - **Variant Calling**: Manta, ExpansionHunter, CNVkit
-- **Assembly**: Hifiasm, vg, wfmash
+- **Assembly**: Hifiasm, vg, wfmash, verkko-fillet
 - **Population Genetics**: PLINK, Shapeit5, IMPUTE5, GLIMPSE
 
 ### Common Software (`common/`)

ansible/roles/compiled/files/uv/bio/verkko-fillet/0.1.25/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "ilifu-verkko-fillet"
+version = "0.1.0"
+# Upper bound pins the python3.12 path segment that the Lmod module prepends to
+# PATH (verkkofillet ships its CLI scripts inside site-packages, not as entry
+# points, so the interpreter version is baked into that path).
+requires-python = ">=3.12,<3.13"
+# verkkofillet declares an empty install_requires and has no entry points, so
+# every runtime dependency below was derived by grepping src/verkkofillet for
+# third-party imports. Do not use the upstream requirements.txt - it is a Sphinx
+# docs list concatenated with a pip freeze of the author's whole environment.
+dependencies = [
+    "verkkofillet==0.1.25",
+    "biopython",
+    "ipykernel",
+    "ipython",
+    "matplotlib",
+    "natsort",
+    "networkx",
+    "numpy",
+    "packaging",
+    "pandas",
+    "pillow",
+    "plotly",
+    "scikit-learn",
+    "seaborn",
+    "session-info",
+    "tqdm",
+]

ansible/roles/compiled/tasks/bio/main.yml

@@ -623,6 +623,9 @@
       - version_number: "2.24"
         url: "https://github.com/lh3/minimap2/releases/download/v2.24/minimap2-2.24_x64-linux.tar.bz2"
         checksum: "sha512:5e27233f75d0ae776707c4b56d43925c0f0f2a25b868e18d43d1a33b67fb730013df04ca881949859d22667aba803681ee0da1c95b01ce0d3dcb7458b12603d9"
+      - version_number: "2.28"
+        url: "https://github.com/lh3/minimap2/releases/download/v2.28/minimap2-2.28_x64-linux.tar.bz2"
+        checksum: "sha256:51f2cf0e486d0f9f88ace1aa58fdc56571382a676ea0889ae607301c60693377"
   ansible.builtin.include_tasks:
     file: "minimap2.yaml"
   loop: "{{ versions }}"
@@ -880,6 +883,59 @@
     - never
     - seqkit
 
+- name: Install seqtk
+  vars:
+    versions:
+      - version_number: "1.5"
+        url: "https://github.com/lh3/seqtk/archive/refs/tags/v1.5.tar.gz"
+        checksum: "sha256:384aa1e3cecf4f70403839d586cbb29d469b7c6f773a64bc5af48a6e4b8220a6"
+  ansible.builtin.include_tasks:
+    file: "seqtk.yaml"
+  loop: "{{ versions }}"
+  tags:
+    - never
+    - seqtk
+
+- name: Install mashmap
+  vars:
+    versions:
+      - version_number: "3.1.3"
+        url: "https://github.com/marbl/MashMap/archive/refs/tags/v3.1.3.tar.gz"
+        checksum: "sha256:7719dd6b3c25e650e16218252eaae7dbf424a10890d717ec3ad0920b102fd05a"
+        cmake_dir: "{{ common_dir }}/cmake/3.27.0"
+        gsl_dir: "{{ common_dir }}/libgsl/2.7.1"
+        gsl_version: "2.7.1"
+        htslib_dir: "{{ bio_dir }}/htslib/1.22.1"
+        htslib_version: "1.22.1"
+  ansible.builtin.include_tasks:
+    file: "mashmap.yaml"
+  loop: "{{ versions }}"
+  tags:
+    - never
+    - mashmap
+
+- name: Install verkko-fillet
+  vars:
+    versions:
+      - version_number: "0.1.25"
+        python_location: "/software/common/python/3.12.9"
+        # Must match the interpreter above - it is the site-packages path
+        # segment the module prepends to PATH.
+        python_abi: "python3.12"
+        samtools_module: "samtools/1.22.1"
+        htslib_module: "htslib/1.22.1"
+        seqtk_module: "seqtk/1.5"
+        mashmap_module: "mashmap/3.1.3"
+        bedtools_module: "bedtools/2.31.1"
+        meryl_module: "meryl/1.4.1"
+        minimap2_module: "minimap2/2.28"
+  ansible.builtin.include_tasks:
+    file: "verkko-fillet.yaml"
+  loop: "{{ versions }}"
+  tags:
+    - never
+    - verkko-fillet
+
 - name: Install chopper
   vars:
     versions:

ansible/roles/compiled/tasks/bio/mashmap.yaml

@@ -0,0 +1,79 @@
+---
+- name: Install mashmap {{ item.version_number }}
+  vars:
+    mashmap_dir: "{{ bio_dir }}/mashmap"
+    mashmap_module_dir: "{{ bio_modules }}/mashmap"
+    url: "{{ item.url }}"
+    version_number: "{{ item.version_number }}"
+    checksum: "{{ item.checksum }}"
+    install_dir: "{{ mashmap_dir }}/{{ version_number }}"
+    unzip_dir: "/dev/shm/MashMap-{{ version_number }}"
+    cmake_bin: "{{ item.cmake_dir }}/bin/cmake"
+    gsl: "{{ item.gsl_dir }}"
+    htslib: "{{ item.htslib_dir }}"
+  tags:
+    - never
+    - mashmap{{ item.version_number }}
+
+  block:
+    - name: Download and install mashmap {{ version_number }}
+      block:
+        - name: Download mashmap {{ version_number }} from {{ url }}
+          ansible.builtin.get_url:
+            url: "{{ url }}"
+            dest: "{{ bio_src }}/mashmap-{{ version_number }}.tar.gz"
+            checksum: "{{ checksum }}"
+            mode: u=rw,g=r,o=r
+        - name: Ensure mashmap bin dir exists {{ install_dir }}
+          ansible.builtin.file:
+            path: "{{ install_dir }}/bin"
+            state: directory
+            mode: u=rwx,g=rwx,o=rx
+        - name: Uncompress mashmap to {{ unzip_dir }}
+          ansible.builtin.unarchive:
+            src: "{{ bio_src }}/mashmap-{{ version_number }}.tar.gz"
+            dest: "/dev/shm"
+            creates: "{{ unzip_dir }}"
+        # MashMap needs C++17, CMake >= 3.21, GSL, zlib and htslib. The GSL and
+        # htslib module trees are injected below rather than relying on system
+        # copies, so the build is reproducible on any host.
+        - name: Run cmake for mashmap {{ version_number }}
+          ansible.builtin.shell: "{{ cmake_bin }} -H. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX:PATH={{ install_dir }} -Bbuild ."
+          args:
+            chdir: "{{ unzip_dir }}"
+            creates: "{{ unzip_dir }}/build/Makefile"
+          environment: &mashmap_build_env
+            LD_LIBRARY_PATH: "{{ gsl }}/lib:{{ htslib }}/lib:{{ ansible_env.LD_LIBRARY_PATH | default('') }}"
+            LIBRARY_PATH: "{{ gsl }}/lib:{{ htslib }}/lib:{{ ansible_env.LIBRARY_PATH | default('') }}"
+            INCLUDE: "{{ gsl }}/include:{{ htslib }}/include:{{ ansible_env.INCLUDE | default('') }}"
+            C_INCLUDE_PATH: "{{ gsl }}/include:{{ htslib }}/include:{{ ansible_env.C_INCLUDE_PATH | default('') }}"
+            CPATH: "{{ gsl }}/include:{{ htslib }}/include:{{ ansible_env.CPATH | default('') }}"
+            PKG_CONFIG_PATH: "{{ gsl }}/lib/pkgconfig:{{ htslib }}/lib/pkgconfig:{{ ansible_env.PKG_CONFIG_PATH | default('') }}"
+        - name: Run cmake build for mashmap {{ version_number }}
+          ansible.builtin.shell: "{{ cmake_bin }} --build build -- -j {{ ansible_processor_vcpus }}"
+          args:
+            chdir: "{{ unzip_dir }}"
+            creates: "{{ unzip_dir }}/build/bin/mashmap"
+          environment: *mashmap_build_env
+        # MashMap's CMakeLists has no reliable install target; upstream INSTALL.txt
+        # documents build/bin/mashmap as the product, so copy it out directly.
+        - name: Copy mashmap {{ version_number }} binary to {{ install_dir }}
+          ansible.builtin.copy:
+            src: "{{ unzip_dir }}/build/bin/mashmap"
+            dest: "{{ install_dir }}/bin/mashmap"
+            remote_src: true
+            force: true
+            mode: preserve
+
+    - name: Install mashmap module {{ version_number }}
+      block:
+        - name: Check module dir {{ mashmap_module_dir }}
+          ansible.builtin.file:
+            path: "{{ mashmap_module_dir }}"
+            state: directory
+            mode: u=rwx,g=rwx,o=rx
+        - name: Install mashmap module file {{ version_number }}
+          ansible.builtin.template:
+            src: bio/mashmap.lua
+            dest: "{{ mashmap_module_dir }}/{{ version_number }}.lua"
+            mode: u=rw,g=r,o=r

ansible/roles/compiled/tasks/bio/seqtk.yaml

@@ -0,0 +1,60 @@
+---
+- name: Install seqtk {{ item.version_number }}
+  vars:
+    seqtk_dir: "{{ bio_dir }}/seqtk"
+    seqtk_module_dir: "{{ bio_modules }}/seqtk"
+    url: "{{ item.url }}"
+    version_number: "{{ item.version_number }}"
+    checksum: "{{ item.checksum }}"
+    install_dir: "{{ seqtk_dir }}/{{ version_number }}"
+    unzip_dir: "/dev/shm/seqtk-{{ version_number }}"
+  tags:
+    - never
+    - seqtk{{ item.version_number }}
+
+  block:
+    - name: Download and install seqtk {{ version_number }}
+      block:
+        - name: Download seqtk {{ version_number }} from {{ url }}
+          ansible.builtin.get_url:
+            url: "{{ url }}"
+            dest: "{{ bio_src }}/seqtk-{{ version_number }}.tar.gz"
+            checksum: "{{ checksum }}"
+            mode: u=rw,g=r,o=r
+        - name: Ensure seqtk bin dir exists {{ install_dir }}
+          ansible.builtin.file:
+            path: "{{ install_dir }}/bin"
+            state: directory
+            mode: u=rwx,g=rwx,o=rx
+        - name: Uncompress seqtk to {{ unzip_dir }}
+          ansible.builtin.unarchive:
+            src: "{{ bio_src }}/seqtk-{{ version_number }}.tar.gz"
+            dest: "/dev/shm"
+            creates: "{{ unzip_dir }}"
+        # seqtk ships a bare Makefile with no configure and no install target,
+        # so the binary is built in place and copied out by hand.
+        - name: Make seqtk {{ version_number }}
+          community.general.make:
+            chdir: "{{ unzip_dir }}"
+          environment:
+            MAKEFLAGS: "-j {{ ansible_processor_vcpus }}"
+        - name: Copy seqtk {{ version_number }} binary to {{ install_dir }}
+          ansible.builtin.copy:
+            src: "{{ unzip_dir }}/seqtk"
+            dest: "{{ install_dir }}/bin/seqtk"
+            remote_src: true
+            force: true
+            mode: preserve
+
+    - name: Install seqtk module {{ version_number }}
+      block:
+        - name: Check module dir {{ seqtk_module_dir }}
+          ansible.builtin.file:
+            path: "{{ seqtk_module_dir }}"
+            state: directory
+            mode: u=rwx,g=rwx,o=rx
+        - name: Install seqtk module file {{ version_number }}
+          ansible.builtin.template:
+            src: bio/seqtk.lua
+            dest: "{{ seqtk_module_dir }}/{{ version_number }}.lua"
+            mode: u=rw,g=r,o=r

ansible/roles/compiled/tasks/bio/verkko-fillet.yaml

@@ -0,0 +1,54 @@
+---
+- name: Install verkko-fillet
+  vars:
+    verkko_fillet_dir: "{{ bio_dir }}/verkko-fillet"
+    verkko_fillet_module_dir: "{{ bio_modules }}/verkko-fillet"
+    install_dir: "{{ verkko_fillet_dir }}/{{ item.version_number }}"
+    python_location: "{{ item.python_location }}"
+    python_abi: "{{ item.python_abi }}"
+    version_number: "{{ item.version_number }}"
+  tags:
+    - never
+    - verkko-fillet{{ item.version_number }}
+
+  block:
+    - name: Create verkko-fillet directory {{ install_dir }}
+      ansible.builtin.file:
+        path: "{{ install_dir }}"
+        state: directory
+        mode: "ug=rwx,o=rx"
+        recurse: true
+
+    - name: Copy uv files
+      ansible.builtin.copy:
+        src: "uv/bio/verkko-fillet/{{ version_number }}/{{ uv_file }}"
+        dest: "{{ install_dir }}/{{ uv_file }}"
+        mode: u=rw,g=r,o=r
+      with_items:
+        - "uv.lock"
+        - "pyproject.toml"
+      loop_control:
+        loop_var: uv_file
+
+    - name: Initialise virtual environment
+      ansible.builtin.command: "{{ python_location }}/bin/uv venv"
+      args:
+        chdir: "{{ install_dir }}"
+        creates: "{{ install_dir }}/.venv"
+
+    - name: Synchronise Virtual Environment
+      ansible.builtin.command: "{{ python_location }}/bin/uv sync --frozen"
+      args:
+        chdir: "{{ install_dir }}"
+
+    - name: Create verkko-fillet module directory {{ verkko_fillet_module_dir }}
+      ansible.builtin.file:
+        path: "{{ verkko_fillet_module_dir }}"
+        state: directory
+        mode: "ug=rwx,o=rx"
+
+    - name: Install verkko-fillet module file
+      ansible.builtin.template:
+        src: bio/verkko-fillet.lua
+        dest: "{{ verkko_fillet_module_dir }}/{{ version_number }}.lua"
+        mode: u=rw,g=r,o=r

ansible/roles/compiled/templates/bio/mashmap.lua

@@ -0,0 +1,17 @@
+-- -*- lua -*-
+help([[
+This module configures mashmap {{ version_number }} for use
+
+MashMap is a fast approximate long-read and assembly mapper.
+]])
+whatis("Version: {{ version_number }}")
+whatis("Keywords: mashmap, alignment, mapping")
+whatis("Description: mashmap {{ version_number }} - fast approximate sequence mapper")
+
+-- mashmap links against the shared GSL and htslib libraries it was built with.
+depends_on("libgsl/{{ item.gsl_version }}", "htslib/{{ item.htslib_version }}")
+
+local mashmap_dir = "{{ install_dir }}"
+local bin_dir = pathJoin(mashmap_dir, "bin")
+
+prepend_path('PATH', bin_dir)

ansible/roles/compiled/templates/bio/seqtk.lua

@@ -0,0 +1,12 @@
+-- -*- lua -*-
+help([[
+This module configures seqtk {{ version_number }} for use
+]])
+whatis("Version: {{ version_number }}")
+whatis("Keywords: seqtk, fasta, fastq")
+whatis("Description: seqtk {{ version_number }} - toolkit for processing FASTA/Q sequences")
+
+local seqtk_dir = "{{ install_dir }}"
+local bin_dir = pathJoin(seqtk_dir, "bin")
+
+prepend_path('PATH', bin_dir)

ansible/roles/compiled/templates/bio/verkko-fillet.lua

@@ -0,0 +1,42 @@
+-- -*- lua -*-
+help([[
+This module configures verkko-fillet {{ version_number }} for use
+
+verkko-fillet is a toolkit for cleaning Verkko genome assemblies. It is driven
+primarily from Python (typically in Jupyter):
+
+    import verkkofillet as vf
+
+The external tools it shells out to (mashmap, samtools, bgzip, seqtk, bedtools,
+meryl, minimap2) are loaded automatically with this module.
+
+Known limitations on this cluster:
+  * screenAssembly() will not work - it needs a full verkko installation on
+    PATH (for lib/verkko/scripts and its bundled contaminant references), and
+    verkko is not currently packaged here.
+  * graphAlign() needs GraphAligner, which is not packaged here. Its location is
+    a runtime parameter (GraphAligner_path), so an external build can be passed in.
+  * Dot plots need gnuplot, which is only available as a container here. The call
+    site checks for it and degrades gracefully.
+]])
+whatis("Version: {{ version_number }}")
+whatis("Keywords: verkko-fillet, verkko, assembly, genomics")
+whatis("Description: verkko-fillet {{ version_number }} - toolkit for cleaning Verkko assemblies")
+
+-- verkkofillet declares no dependencies of its own and shells out to these at
+-- runtime. bgzip comes from htslib, not from the samtools tarball.
+depends_on("{{ item.samtools_module }}",
+           "{{ item.htslib_module }}",
+           "{{ item.seqtk_module }}",
+           "{{ item.mashmap_module }}",
+           "{{ item.bedtools_module }}",
+           "{{ item.meryl_module }}",
+           "{{ item.minimap2_module }}")
+
+local venv_dir = pathJoin("{{ install_dir }}", ".venv")
+-- verkkofillet ships no console entry points; its CLI scripts live inside the
+-- installed package, so that directory goes on PATH explicitly.
+local script_dir = pathJoin(venv_dir, "lib", "{{ python_abi }}", "site-packages", "verkkofillet", "bin")
+
+prepend_path('PATH', pathJoin(venv_dir, "bin"))
+prepend_path('PATH', script_dir)