feature branch: nice-to-have features

.github/workflows/linting.yml

@@ -34,7 +34,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
       - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v2
+        uses: nf-core/setup-nextflow@6c2e22b4d901f0c42ca66c5069f8026df026d165 # v2
 
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:

.github/workflows/twistgp_ci.yml

@@ -38,13 +38,13 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Nextflow
-        uses: nf-core/setup-nextflow@v2
+        uses: nf-core/setup-nextflow@6c2e22b4d901f0c42ca66c5069f8026df026d165 # v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
       - name: Set up Apptainer
         if: matrix.profile == 'singularity'
-        uses: eWaterCycle/setup-apptainer@main
+        uses: eWaterCycle/setup-apptainer@3f706d898c9db585b1d741b4692e66755f3a1b40 # main
 
       - name: Set up Singularity
         if: matrix.profile == 'singularity'

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added GATK4 FilterMutectCalls after Mutect2 variant calling
 - Added BCFTOOLS_VIEW pre-filtering step prior to TMB calculation
 - Added `--tmb_popaf_cutoff` and `--tmb_vaf_cutoff` parameters
+- Added `--skip_cnv`, `--skip_msi`, and `--skip_tmb` parameters to allow skipping CNV calling, MSI analysis, and TMB calculation respectively
 
 ## 1.1.0dev
 

README.md

@@ -183,10 +183,35 @@ Prior to TMB calculation, annotated variants are pre-filtered using `bcftools vi
 The following parameters control these thresholds:
 
 - `--tmb_popaf_cutoff` (default: `3.0`): Minimum POPAF value (negative log10 of population allele frequency) to include a variant. The default of `3.0` corresponds to a population allele frequency of ≤ 0.001 (0.1%), excluding common germline variants that are unlikely to be somatic. This value is derived from the Mutect2 `POPAF` INFO field.
-- `--tmb_vaf_cutoff` (default: `0.05`): Minimum variant allele frequency (FORMAT/AF) to include a variant. The default of `0.05` (5%) excludes very low frequency variants that may represent sequencing artifacts or sub-clonal noise, consistent with the [Friends of Cancer Research TMB Harmonization Project](https://friendsofcancerresearch.org/publication/in-silico-assessment-of-variation-in-tmb-quantification-across-diagnostic-platforms-phase-1-of-the-friends-of-cancer-research-harmonization-project/) recommendations.
+- `--tmb_vaf_cutoff` (default: `0.10`): Minimum variant allele frequency (FORMAT/AF) to include a variant. The [Friends of Cancer Research TMB Harmonization Project](https://friendsofcancerresearch.org/publication/in-silico-assessment-of-variation-in-tmb-quantification-across-diagnostic-platforms-phase-1-of-the-friends-of-cancer-research-harmonization-project/) recommends a minimum of 0.05 (5%). The default of 0.10 (10%) provides additional stringency to reduce sub-clonal noise in tumor-only analyses.
 
 </details>
 
+### Skipping Analysis Steps
+
+Individual analysis steps can be skipped using the following parameters:
+
+| Parameter        | Description                                                         |
+| ---------------- | ------------------------------------------------------------------- |
+| `--skip_cnv`     | Skip CNV calling with CNVkit                                        |
+| `--skip_msi`     | Skip microsatellite instability analysis (MSIsensor2/MSIsensor-pro) |
+| `--skip_tmb`     | Skip tumor mutational burden calculation (pyTMB)                    |
+| `--skip_civicpy` | Skip CIViCpy variant annotation                                     |
+
+For example, to run the pipeline without MSI and TMB:
+
+```console
+nextflow run twistcgp/main.nf \
+   -profile docker \
+   --input samplesheet.csv \
+   --fasta hg38_giab.fa \
+   --baits baits.bed \
+   --targets targets.bed \
+   --outdir results \
+   --skip_msi \
+   --skip_tmb
+```
+
 ### Variant Filtering with FilterMutectCalls
 
 Following variant calling with Mutect2, this pipeline applies [`FilterMutectCalls`](https://gatk.broadinstitute.org/hc/en-us/articles/360036856831-FilterMutectCalls) to annotate variant quality, consistent with [GATK Best Practices for somatic variant discovery](https://www.biorxiv.org/content/biorxiv/early/2019/12/02/861054/DC1/embed/media-1.pdf?download=true) (Benjamin et al., 2019).

assets/pytmb_vep.yml

@@ -89,12 +89,13 @@ polymDb:
   gnomad:
     - gnomADe_AF
     - gnomADe_AFR_AF
-    - gnomADe_AMR_AF
+    - gnomAD_AMR_AF
     - gnomADe_ASJ_AF
     - gnomADe_EAS_AF
     - gnomADe_FIN_AF
     - gnomADe_MID_AF
     - gnomADe_NFE_AF
+    - gnomADe_REMAINING_AF
     - gnomADe_SAS_AF
     - gnomADg_AF
     - gnomADg_AFR_AF

conf/modules.config

@@ -27,6 +27,7 @@ process {
     }
 
     withName: BCFTOOLS_VIEW {
+        ext.when = { params.skip_tmb == false }
         // NB: ext.args must include --write-index=tbi for downstream processes to run successfully
         ext.args = {
             [
@@ -53,8 +54,15 @@ process {
             saveAs: { filename -> filename != 'versions.yml' && params.save_reference ? filename : null },
         ]
     }
-    withName: CIVICPY {
-        ext.when = { params.skip_civicpy == false }
+
+    withName: CIVICPY_UPDATE_CACHE {
+        publishDir = [
+            path: { params.save_reference ? "${params.outdir}/reference" : params.outdir },
+            mode: params.publish_dir_mode,
+            saveAs: { filename -> filename != 'versions.yml' && params.save_reference ? filename : null },
+        ]
+    }
+    withName: CIVICPY_ANNOTATE_VCF {
         publishDir = [
             path: { "${params.outdir}/${meta.id}" },
             mode: params.publish_dir_mode,
@@ -63,6 +71,7 @@ process {
     }
 
     withName: CNVKIT_BATCH {
+        ext.when = { params.skip_cnv == false }
         // hmm-tumor performs well for relatively pure, high coverage tumor samples
         // alternative segmentation methods can be explored here:
         // https://cnvkit.readthedocs.io/en/stable/pipeline.html#segmentation-methods
@@ -85,13 +94,13 @@ process {
     }
 
     withName: 'ENSEMBLVEP_VEP' {
-        ext.args = "--vcf --af_gnomade --af_1kg" + (params.cosmic_vcf ? " --custom ${params.cosmic_vcf},COSMIC,vcf,exact,0,ID" : "")
+        ext.args = "--format vcf --vcf --af_gnomade --af_1kg" + (params.cosmic_vcf ? " --custom ${params.cosmic_vcf},COSMIC,vcf,exact,0,ID" : "")
         ext.prefix = { "${meta.id}.vep" }
         publishDir = [
             [
                 mode: params.publish_dir_mode,
                 path: { "${params.outdir}/${meta.id}/" },
-                pattern: "*{gz,html}",
+                pattern: "*{gz,gz.tbi,html}",
             ]
         ]
     }
@@ -106,8 +115,9 @@ process {
     }
 
     withName: GATK4_FILTERMUTECTCALLS {
+        ext.prefix = { "${meta.id}.labeled" }
         publishDir = [
-            path: { "${params.outdir}/${meta.id}.labeled" },
+            path: { "${params.outdir}/${meta.id}" },
             mode: params.publish_dir_mode,
             pattern: "*{vcf.gz,vcf.gz.tbi,filteringStats.tsv}",
         ]
@@ -148,6 +158,7 @@ process {
     }
 
     withName: MSISENSOR2_MSI {
+        ext.when = { params.skip_msi == false }
         // NB: The module outputs the summary file as "${prefix}"
         ext.prefix = { "${meta.id}.msi" }
         publishDir = [
@@ -158,6 +169,7 @@ process {
     }
 
     withName: MSISENSORPRO_PRO {
+        ext.when = { params.skip_msi == false }
         // NB: The module outputs the summary file as "${prefix}"
         ext.prefix = { "${meta.id}.msi" }
         publishDir = [
@@ -266,8 +278,11 @@ process {
     }
 
     withName: TMB {
+        ext.when = { params.skip_tmb == false }
+        errorStrategy = { task.attempt <= task.maxRetries ? 'retry' : 'finish' }
+        maxRetries = 3
         // VAF, minDepth, and minAltDepth recommended by: https://friendsofcancerresearch.org/publication/in-silico-assessment-of-variation-in-tmb-quantification-across-diagnostic-platforms-phase-1-of-the-friends-of-cancer-research-harmonization-project/
-        ext.args = "--polymDb gnomad --filterPolym --vaf 0.05 --minDepth 25 --minAltDepth 3 --filterLowQual --filterIndels --filterNonCoding --filterSyn --maf 0.01"
+        ext.args = "--polymDb gnomad --filterPolym --vaf ${params.tmb_vaf_cutoff} --minDepth 25 --minAltDepth 3 --filterLowQual --filterIndels --filterNonCoding --filterSyn --maf 0.01"
         publishDir = [
             path: { "${params.outdir}/${meta.id}" },
             mode: params.publish_dir_mode,
@@ -284,7 +299,15 @@ process {
         ]
     }
 
-    withName: TABIX_BGZIPTABIX {
+    withName: '.*VCF_ANNOTATE_ENSEMBLVEP:TABIX_TABIX' {
+        publishDir = [
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/${meta.id}/" },
+            pattern: "*.tbi",
+        ]
+    }
+
+    withName: '.*VCF_ANNOTATE_SNPEFF:TABIX_BGZIPTABIX' {
         ext.prefix = { "${meta.id}.snpeff" }
         publishDir = [
             mode: params.publish_dir_mode,
@@ -293,8 +316,8 @@ process {
         ]
     }
 
-    withName: TABIX_TABIX {
-        ext.prefix = { "${meta.id}.snpeff" }
+    withName: 'TWISTCGP:TABIX_BGZIPTABIX' {
+        ext.prefix = { "${meta.id}.civic" }
         publishDir = [
             mode: params.publish_dir_mode,
             path: { "${params.outdir}/${meta.id}/" },

conf/test.config

@@ -42,22 +42,15 @@ process {
         // FilterMutectCalls fails with the minimal test data
         ext.when = { false }
     }
-    withName: "TMB" {
-        // This process runs after SNPEFF_SNPEFF, which we are also skipping
-        ext.when = { false }
-    }
-    withName: "MSISENSOR2_MSI" {
-        // TODO: Create small test data to use in integration testing
-        ext.when = { false }
-    }
-    withName: "MSISENSORPRO_PRO" {
-        // This process fails with the empty output from MSISENSOR2_SCAN
-        // TODO: Create small test data to use in integration testing
+    withName: "CIVICPY_UPDATE_CACHE" {
+        // Downloading the CIViC cache is too slow for CI
         ext.when = { false }
     }
 }
 
 params {
+    skip_tmb = true
+    skip_msi = true
     config_profile_name = 'Test profile'
     config_profile_description = 'Minimal test dataset to check pipeline function'
 

modules/local/civicpy/main.nf → modules/local/civicpy/annotate/main.nf

@@ -1,15 +1,16 @@
-process CIVICPY {
+process CIVICPY_ANNOTATE_VCF {
     tag "${meta.id}"
     label 'process_single'
 
     conda "${moduleDir}/environment.yml"
     container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container
-        ? 'https://depot.galaxyproject.org/singularity/civicpy:5.1.0--pyhdfd78af_0'
-        : 'docker.io/griffithlab/civicpy:5.1.0' }"
+        ? 'https://depot.galaxyproject.org/singularity/civicpy:5.2.0--pyhdfd78af_0'
+        : 'docker.io/griffithlab/civicpy:v5.2.0' }"
 
     input:
     tuple val(meta), path(vcf), path(tbi)
     val annotation_genome_version
+    path cache
 
     output:
     tuple val(meta), path("*.vcf"), emit: vcf
@@ -23,12 +24,13 @@ process CIVICPY {
     def prefix = task.ext.prefix ?: "${meta.id}.civic"
 
     """
-    export CIVICPY_CACHE_FILE=\$PWD/.civicpy
+    export CIVICPY_CACHE_FILE=\$PWD/${cache}
 
     civicpy annotate-vcf --input-vcf ${vcf} \\
         --output-vcf ${prefix}.vcf \\
         --reference ${annotation_genome_version} \\
-        --include-status accepted
+        --include-status accepted \\
+        ${args}
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

modules/local/civicpy/update_cache/main.nf

@@ -0,0 +1,37 @@
+process CIVICPY_UPDATE_CACHE {
+    label 'process_single'
+
+    conda "${moduleDir}/../annotate/environment.yml"
+    container "${workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container
+        ? 'https://depot.galaxyproject.org/singularity/civicpy:5.2.0--pyhdfd78af_0'
+        : 'docker.io/griffithlab/civicpy:v5.2.0' }"
+
+    output:
+    path "civicpy_cache.pkl", emit: cache
+    path "versions.yml",      emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    """
+    export CIVICPY_CACHE_FILE=\$PWD/civicpy_cache.pkl
+
+    civicpy update
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        civicpy: \$(civicpy --version | sed 's/.*version //')
+    END_VERSIONS
+    """
+
+    stub:
+    """
+    touch civicpy_cache.pkl
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        civicpy: \$(civicpy --version | sed 's/.*version //')
+    END_VERSIONS
+    """
+}

modules/local/civicpy/update_cache/meta.yml

@@ -0,0 +1,38 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json
+name: "civicpy_update_cache"
+
+description: Downloads the CIViC nightly cache for use with civicpy annotation
+keywords:
+  - clinical interpretation
+  - variant annotation
+  - genomics
+tools:
+  - "civicpy":
+      description: "CIViC variant knowledgebase analysis toolkit."
+      homepage: "https://docs.civicpy.org/en/latest/"
+      documentation: "https://docs.civicpy.org/en/latest/"
+      tool_dev_url: "https://github.com/griffithlab/civicpy"
+      doi: "10.1200/CCI.19.00127"
+      licence: ["MIT"]
+      identifier: biotools:CIViCpy
+
+input: []
+
+output:
+  cache:
+    - "civicpy_cache.pkl":
+        type: file
+        description: CIViC database cache file
+        pattern: "*.pkl"
+
+  versions_yml:
+    - "versions.yml":
+        type: file
+        description: File containing software versions
+        pattern: "versions.yml"
+
+authors:
+  - "@emmcauley"
+maintainers:
+  - "@emmcauley"

modules/local/tmb/main.nf

@@ -37,7 +37,7 @@ process TMB {
         > ${prefix}.tmb.log
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        tmb: \$(echo \$(pyTMB.py --version 2>&1) | sed 's/^.*pyTMB.py //; s/.*\$//' | sed 's|[()]||g')
+        tmb: \$(pyTMB.py --version 2>&1 | sed 's/pyTMB.py (//; s/)//')
     END_VERSIONS
     """
 
@@ -50,7 +50,7 @@ process TMB {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        tmb: \$(echo \$(pyTMB.py --version 2>&1) | sed 's/^.*pyTMB.py //; s/.*\$//' | sed 's|[()]||g')
+        tmb: \$(pyTMB.py --version 2>&1 | sed 's/pyTMB.py (//; s/)//')
     END_VERSIONS
     """
 }

nextflow.config

@@ -60,12 +60,15 @@ params {
 
     // TMB options
     tmb_popaf_cutoff = 3.0
-    tmb_vaf_cutoff = 0.05
+    tmb_vaf_cutoff = 0.10
     tmb_vep_config = "${projectDir}/assets/pytmb_vep.yml"
     tmb_mutect2_config = "${projectDir}/assets/pytmb_mutect2.yml"
 
-    //CIVICpy
+    // Skip options
     skip_civicpy = false
+    skip_cnv = false
+    skip_msi = false
+    skip_tmb = false
 
     // MultiQC options
     multiqc_config = null