mskcc · gongyixiao · Mar 31, 2022 · Mar 31, 2022 · Mar 31, 2022 · Mar 31, 2022
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,57 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    env:
+      NXF_VER: "25.04.7"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Clean up Disk space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+
+      - name: Set up Java (OpenJDK 17)
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Ensure pip is up-to-date
+        run: python -m pip install --upgrade pip
+
+      - name: Set up Nextflow
+        uses: nf-core/setup-nextflow@v2
+        with:
+          version: "${{ env.NXF_VER }}"
+
+      - name: Download and extract test-data
+        # If your data is public this will work. If it's private, see notes below.
+        run: |
+          wget -O test-data.tar.gz --no-check-certificate 'https://drive.usercontent.google.com/download?export=download&confirm=no_antivirus&id=13zUVw4BZ0_5QAW7zmdCsZ_CZDHbNxyMV'
+          tar -xzvf test-data.tar.gz
+
+      - name: Install Python deps (if requirements.txt exists)
+        run: |
+          if [ -f requirements.txt ]; then
+            pip install -r requirements.txt
+          fi
+
+      - name: Run tests
+        run: |
+          python3 tests/test_runner.py tests/tests.json
+
diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,10 @@ nf.*
 *.gz
 *.bam.*
 Result/
+Results/
+result/
+results/
 *.Rhistory
 *.tsv
 docs/node_modules
+*.pyc
diff --git a/.nvmrc b/.nvmrc
@@ -0,0 +1 @@
+18
diff --git a/.travis.yml b/.travis.yml
diff --git a/README.md b/README.md
@@ -1,12 +1,75 @@
-[![Build Status](https://travis-ci.com/mskcc/tempo.svg?token=DokCkCiDp43sqzeuXUHD&branch=master)](https://travis-ci.com/mskcc/tempo)
 
-# <img id="logo" src="./docs/tempoLogo.png" height="42" align="left"/> TEMPO
+# Time-Efficient Mutational Profiling in Oncology (Tempo)
 
-Tempo is a CMO Computational Sciences (CCS) research pipeline processing WES & WGS tumor-normal pairs using the [Nextflow framework](https://www.nextflow.io/). Currently the pipeline is composed of alignment and QC, and detection of both somatic alterations and germline variants. Users can begin with inputs of either paired-end FASTQs or BAMs, and process these via the command line. 
+Tempo is a computational pipeline for processing data of paired-end whole-exome (WES) and whole-genome sequencing (WGS) of human cancer samples with matched normals. Its components are containerized and the pipeline runs on the [Juno high-performance computing cluster](http://mskcchpc.org/display/CLUS/Juno+Cluster+Guide) at Memorial Sloan Kettering Cancer Center and on [Amazon Web Services (AWS)](https://aws.amazon.com). The pipeline was written by members of the [Center for Molecular Oncology](https://www.mskcc.org/research-programs/molecular-oncology).
 
-For further details of how to begin processing data with Tempo, please view our [documentation](https://cmotempo.netlify.com/). For contributing to this project, please make a pull request as detailed [here](https://cmotempo.netlify.com/contributing-to-tempo.html).
+These pages contain instructions on how to run the Tempo pipeline. It also contains documentation on the bioinformatic components in the pipeline, some motivation for various parameter choices, plus an outline describing the reference resources used. 
+
+If there are any questions or comments, you are welcome to [raise an issue](https://github.com/mskcc/tempo/issues/new?title=[User%20question]).
+
+<small>Note: Tempo currently only supports human samples. The pipeline has only been tested for exome and genome sequencing experiments, and all reference files are in build GRCh37 of the human genome.</small>
+
+---
+
+## Table of Contents
+
+### 1. Getting Started
+
+#### 1.1. Setup
+* [Installation](docs/installation.md)
+* [Setup on Juno](docs/juno-setup.md)
+* [Setup on AWS](docs/aws-setup.md)
+
+#### 1.2. Usage
+* [Running the Pipeline](docs/running-the-pipeline.md)
+    * [Overview](docs/running-the-pipeline.md#overview)
+    * [Input Files](docs/running-the-pipeline.md#input-files)
+    * [Execution Mode](docs/running-the-pipeline.md#execution-mode)
+    * [Modifying or Resuming Pipeline Run](docs/running-the-pipeline.md#modifying-or-resuming-pipeline-run)
+    * [After Successful Run](docs/running-the-pipeline.md#after-successful-run)
+* [Nextflow Basics](docs/nextflow-basics.md)
+* [Working With Containers](docs/working-with-containers.md)
+
+#### 1.3 Outputs
+* [BAM Files](docs/outputs.md#bam-files)
+* [QC Outputs](docs/outputs.md#qc-outputs)
+* [Somatic Data](docs/outputs.md#somatic-data)
+* [Germline Data](docs/outputs.md#germline-data)
+* [Cohort Level Outputs](docs/outputs.md#cohort-level-outputs)
+
+### 2. Pipeline contents
+
+#### 2.1. Bioinformatic Components
+* [Read Alignment](docs/bioinformatic-components.md#read-alignment)
+* [Somatic Analyses](docs/bioinformatic-components.md#somatic-analyses)
+* [Germline Analyses](docs/bioinformatic-components.md#germline-analyses)
+* [Quality Control](docs/bioinformatic-components.md#quality-control)
+
+#### 2.2. Reference Resources
+* [Genome Assembly](docs/reference-files.md#genome-assembly)
+* [Genomic Intervals](docs/reference-files.md#genomic-intervals)
+* [RepeatMasker and Mappability Blacklist](docs/reference-files.md#repeatmasker-and-mappability-blacklist)
+* [Preferred Transcript Isoforms](docs/reference-files.md#preferred-transcript-isoforms)
+* [Hotspot Annotation](docs/reference-files.md#hotspot-annotation.md)
+* [OncoKB Annotation](docs/reference-files.md#oncokb.md)
+* [gnomAD](docs/gnomad.md)
+* [Panel of Normals for Exomes](docs/wes-panel-of-normals.md)
+
+#### 2.3. Variant Annotation and Filtering
+* [Somatic SNVs and Indels](docs/variant-annotation-and-filtering.md#somatic-snvs-and-indels)
+* [Germline SNVs and Indels](docs/variant-annotation-and-filtering.md#germline-snvs-and-indels)
+* [Somatic and Germline SVs](docs/variant-annotation-and-filtering.md#somatic-and-germline-svs)
+
+### 3. Help and Other Resources
+* [Troubleshooting](docs/troubleshooting.md)
+* [AWS Glossary](docs/aws-glossary.md)
+
+### 4. Contributing
+* [Contributing to Tempo](docs/contributing-to-tempo.md)
+
+### 5. Acknowledgements
+* [Acknowledgements](docs/acknowledgements.md)
 
-The inspiration for this project derives from [Sarek](https://github.com/SciLifeLab/Sarek), developed at [SciLifeLab](https://github.com/SciLifeLab).
 
 ## Pipeline Flowchart
 <p align="center">
@@ -22,3 +85,4 @@ The inspiration for this project derives from [Sarek](https://github.com/SciLife
 <p align="center">
   <img src="./docs/brandenburg5_allegro.jpg">
 </p>
+---
diff --git a/conf/awsbatch.config.template b/conf/awsbatch.config.template
@@ -20,8 +20,8 @@ executor {
 process {
   queue = <AWS-BATCH-QUEUE-ARN>
   scratch = true
-  errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' }
   maxRetries = 3
+  errorStrategy = { task.attempt <= process.maxRetries ? 'retry' : 'ignore' }
 }
 
 params {

diff --git a/conf/containers.config b/conf/containers.config
@@ -24,29 +24,29 @@
 
 //------------------- Somatic pipeline
 
-  withName:SomaticDellyCall {
-    container = "cmopipeline/delly-bcftools:0.0.1"
-  }
   withName:RunMutect2 {
     container = "broadinstitute/gatk:4.1.0.0"
   }
   withName:SomaticCombineMutect2Vcf {
     container = "cmopipeline/bcftools-vt:1.2.0"
   }
-  withName:SomaticRunManta {
-    container = "cmopipeline/strelka2_manta:latest" 
+  withName:SomaticMergeSVs {
+    container = "cmopipeline/bcftools-vt-mergesvvcf:0.0.1"
   }
-  withName:SomaticMergeDellyAndManta {
-    container = "cmopipeline/bcftools-vt:1.2.0"
+  withName:SomaticSVVcf2Bedpe {
+    container = "cmopipeline/svtools:0.0.3"
+  }
+  withName:SomaticAnnotateSVBedpe {
+    container = "cmopipeline/iannotatesv:0.0.2" 
   }
   withName:"SomaticRunStrelka2.*" {
-    container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.0"
+    container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.1"
   }
   withName:SomaticCombineChannel {
     container = "cmopipeline/bcftools-vt:1.2.3"
   }
   withName:SomaticAnnotateMaf {
-    container = "cmopipeline/vcf2maf:vep88_1.2.7"
+    container = "cmopipeline/vcf2maf:vep88_1.3.0"
   }
   withName:DoFacets {
     container = "cmopipeline/facets-suite-preview-htstools:0.0.1"
@@ -70,12 +70,47 @@
     container = "cmopipeline/facets-suite-preview-htstools:0.0.1"
   } 
   withName:RunNeoantigen {
-    container = "cmopipeline/neoantigen:0.3.2-hotfix"
+    container = "cmopipeline/neoantigen:0.3.3"
   }
   withName:MetaDataParser {
     container = "cmopipeline/metadataparser:0.5.9"
   }
-
+  withName:SomaticDellyCall {
+    container = "cmopipeline/delly-bcftools:0.0.1"
+  }
+  withName:'.*DellyCombine' {
+    container = "cmopipeline/delly-bcftools:0.0.1"
+  }
+  withName:SomaticRunManta {
+    container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.1"
+  }
+  withName: '.*RunSvABA' {
+    container = "cmopipeline/svaba:0.0.1"
+  }
+  withName: 'runBRASS.*' {
+    container = "cmopipeline/brass:0.0.2"
+  }
+  withName:generateBasFile {
+    container = "quay.io/wtsicgp/pcap-core:5.5.0"
+  }
+  withLabel:ascat {
+    container = "quay.io/wtsicgp/ascatNgs:4.4.0"
+  }
+  withName:HRDetect {
+    container = "cmopipeline/signaturetoolslib:0.0.1"
+  }
+  withName:RunSVSignatures {
+    container = "cmopipeline/signaturetoolslib:0.0.1"
+  }
+  withName:SomaticRunSVclone {
+    container = "cmopipeline/svclone:0.0.1"
+  }
+  withName:SomaticRunClusterSV {
+    container = "cmopipeline/clustersv:0.0.1"
+  }
+  withName:SomaticRunSVCircos {
+    container = "cmopipeline/biocircos:0.0.1"
+  }
 
 //------------------- Germline pipeline
 
@@ -89,10 +124,10 @@
     container = "cmopipeline/bcftools-vt:1.1.1"
   }
   withName:GermlineRunManta {
-    container = "cmopipeline/strelka2_manta:latest" 
+    container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.1"
   }
   withName:"GermlineRunStrelka2" {
-    container = "cmopipeline/strelka2_manta:latest"
+    container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.1"
   }
   withName:GermlineCombineChannel {
     container = "cmopipeline/bcftools-vt:1.2.2"
@@ -103,10 +138,15 @@
   withName:GermlineFacetsAnnotation {
     container = "cmopipeline/facets-suite-preview-htstools:0.0.1"
   } 
-  withName:GermlineMergeDellyAndManta {
-    container = "cmopipeline/bcftools-vt:1.1.1"
+  withName:GermlineMergeSVs {
+    container = "cmopipeline/bcftools-vt-mergesvvcf:0.0.1"
+  }
+  withName:GermlineSVVcf2Bedpe {
+    container = "cmopipeline/svtools:0.0.3"
+  }
+  withName:GermlineAnnotateSVBedpe {
+    container = "cmopipeline/iannotatesv:0.0.2"
   }
-
 
 
 //------------------- Quality Control

diff --git a/conf/genome.config b/conf/genome.config
@@ -36,4 +36,7 @@ params {
     minCoverageFilter = 10
   }
   scatterCount = 1000
+  ascat {
+    alleleCountLimit = ["GRCh37","smallGRCh37","GRCh38"].contains(params.genome) ? 48 : 1
+  }
 }
diff --git a/conf/juno.config b/conf/juno.config
@@ -18,15 +18,14 @@ process {
   clusterOptions = ""
   scratch = true
   beforeScript = "module load singularity/3.1.1; unset R_LIBS; catch_term () { echo 'caught USR2/TERM signal'; set +e; false; on_exit ; } ; trap catch_term USR2 TERM"
+  maxRetries = 3
+  errorStrategy = { task.attempt <= process.maxRetries ? 'retry' : 'ignore' }
 }
 
-process.errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' }
-process.maxRetries = 3
-
 params {
   max_memory = "128.GB"
   mem_per_core = true
-  reference_base = "/juno/work/taylorlab/cmopipeline" 
+  reference_base = "/juno/work/tempo/cmopipeline"
   // targets_base   = "/juno/work/ccs/resources/tempo/${params.genome}"
   targets_base   = "${reference_base}/mskcc-igenomes/${params.genome.toLowerCase()}/tempo_targets"
   genome_base = params.genome == 'GRCh37' ? "${reference_base}/mskcc-igenomes/igenomes/Homo_sapiens/GATK/GRCh37" : params.genome == 'GRCh38' ? "${reference_base}/mskcc-igenomes/igenomes/Homo_sapiens/GATK/GRCh38" : "${reference_base}/mskcc-igenomes/igenomes/smallGRCh37"

diff --git a/conf/references.config b/conf/references.config
@@ -35,14 +35,35 @@ params {
       knownIndelsIndex = "${params.genome_base}/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.small.vcf.idx"  
       msiSensorList    = "${params.genome_base}/small.msi.list"
       snpeffDb         = "GRCh37.75"
-      vepCacheVersion  = "95"
+      vepCache = "${params.reference_base}/vep"
+      vepCacheVersion  = "88"
+      facetsVcf = "${params.genome_base}/dbsnp_138.b37.small.vcf"
       svCallingExcludeRegions = "${params.genome_base}/human.hg19.excl.tsv"
       svCallingIncludeRegions = "${params.genome_base}/b37.test.bed.gz"
       svCallingIncludeRegionsIndex = "${svCallingIncludeRegions}.tbi"
+      repeatMasker = "${params.reference_base}/annotation/rmsk_mod.bed.gz"
+      repeatMaskerIndex = "${repeatMasker}.tbi"
+      mapabilityBlacklist = "${params.reference_base}/annotation/wgEncodeDacMapabilityConsensusExcludable.bed.gz"
+      mapabilityBlacklistIndex = "${mapabilityBlacklist}.tbi"
+      isoforms = "${params.reference_base}/annotation/isoforms"
       exomePoN = "${params.genome_base}/pon_test.vcf.gz"
       exomePoNIndex = "${exomePoN}.tbi"
       wgsPoN = "${params.genome_base}/pon_test.1.vcf.gz"
       wgsPoNIndex = "${wgsPoN}.tbi"
+      snpGcCorrections = "${params.genome_base}/SnpGcCorrections.small.tsv"
+      spliceSites = "${params.reference_base}/annotation/splice_sites.small.bed"
+      gnomadWesVcf = "${params.reference_base}/gnomad/gnomad.exomes.r2.1.1.sites.non_cancer.vcf.gz"
+      gnomadWesVcfIndex = "${gnomadWesVcf}.tbi"
+      gnomadWgsVcf = "${params.reference_base}/gnomad/gnomad.genome.r2.1.1.sites.minimal.vcf.gz"
+      gnomadWgsVcfIndex = "${gnomadWgsVcf}.tbi"
+      hlaFasta = "${params.reference_base}/hla/abc_complete.fasta"
+      hlaDat = "${params.reference_base}/hla/hla.dat"
+      neoantigenCDNA = "${params.reference_base}/neoantigen/Homo_sapiens.GRCh37.75.cdna.all.fa.gz"
+      neoantigenCDS = "${params.reference_base}/neoantigen/Homo_sapiens.GRCh37.75.cds.all.fa.gz"
+      svBlacklistBed   = "${params.genome_base}/sv_calling/pcawg6_blacklist.slop.bed.gz"
+      svBlacklistBedpe = "${params.genome_base}/sv_calling/pcawg6_blacklist.slop.trunc.bedpe.gz"
+      svBlacklistFoldbackBedpe = "${params.genome_base}/sv_calling/pcawg6_blacklist_foldback_artefacts.slop.trunc.bedpe.gz"
+      svBlacklistTEBedpe       = "${params.genome_base}/sv_calling/pcawg6_blacklist_TE_pseudogene.bedpe.gz"
     }
     'GRCh37' {
       acLoci      = "${params.genome_base}/Annotation/ASCAT/1000G_phase3_20130502_SNP_maf0.3.loci"
@@ -80,6 +101,15 @@ params {
       hlaDat = "${params.reference_base}/mskcc-igenomes/grch37/hla/hla.dat"
       neoantigenCDNA = "${params.reference_base}/mskcc-igenomes/grch37/neoantigen/Homo_sapiens.GRCh37.75.cdna.all.fa.gz"
       neoantigenCDS = "${params.reference_base}/mskcc-igenomes/grch37/neoantigen/Homo_sapiens.GRCh37.75.cds.all.fa.gz"
+      snpGcCorrections = "${params.reference_base}/mskcc-igenomes/grch37/ascat/SnpGcCorrections.tsv"
+      spliceSites = "${params.reference_base}/mskcc-igenomes/grch37/splice_sites/splice_sites.bed"
+      brassRefDir      = "${params.reference_base}/mskcc-igenomes/grch37/brass"
+      vagrentRefDir    = "${params.reference_base}/mskcc-igenomes/grch37/vagrent"
+      // svBlacklist* source: https://bitbucket.org/weischenfeldt/pcawg_sv_merge/src/docker/data/blacklist_files/
+      svBlacklistBed   = "${params.reference_base}/mskcc-igenomes/grch37/sv_calling/pcawg6_blacklist.slop.bed.gz"
+      svBlacklistBedpe = "${params.reference_base}/mskcc-igenomes/grch37/sv_calling/pcawg6_blacklist.slop.bedpe.gz"
+      svBlacklistFoldbackBedpe = "${params.reference_base}/mskcc-igenomes/grch37/sv_calling/pcawg6_blacklist_foldback_artefacts.slop.bedpe.gz"
+      svBlacklistTEBedpe       = "${params.reference_base}/mskcc-igenomes/grch37/sv_calling/pcawg6_blacklist_TE_pseudogene.bedpe.gz"
     } 
     'GRCh38' {
       acLoci           = "${params.genome_base}/Annotation/ASCAT/1000G_phase3_GRCh38_maf0.3.loci"
@@ -101,6 +131,7 @@ params {
       //AF_indexes    = "${params.genome_base}/{00-All.dbsnp_151.hg38.CAF.TOPMED.alternate.allele.freq,hapmap_3.3_grch38_pop_stratified_af.HMAF,SweGen_hg38_stratified.SWAF}.vcf.idx"
       hlaFasta = "${params.reference_base}/mskcc-igenomes/grch38/hla/abc_complete.fasta"   
       hlaDat = "${params.reference_base}/mskcc-igenomes/grch38/hla/hla.dat"
-      }
+      snpGcCorrections = "${params.reference_base}/mskcc-igenomes/grch38/ascat/SnpGcCorrections.tsv"
+    }
   }
 }