Feature/extract align

install_data.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+
+# Test data is hosted on Google Drive at:
+# https://drive.google.com/file/d/1GtT8jsBGwRoQC-5wHh06r8RFkiFBuirp/view?usp=sharing
+
+fileid=1GtT8jsBGwRoQC-5wHh06r8RFkiFBuirp
+
+filename=test_nucleo.tar.gz
+foldername=test_nucleo
+
+# Skip if already have test data
+[[ -f $filename ]] && exit 0
+[[ -d $foldername ]] && exit 0
+
+curl -c ./cookie -s -k -L "https://drive.google.com/uc?export=download&id=$fileid" > /dev/null
+
+curl -k -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=${fileid}" -o ${filename}
+
+# Suppress linux warnings for MacOS tar.gz files
+if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    tar --warning=no-unknown-keyword -xzvf $filename
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+    tar -xzvf $filename
+fi
+
+rm $filename

subworkflows/nf-core/fastq_alignsort_bwa_picard/main.nf

@@ -0,0 +1,63 @@
+// import
+// bwa2 for extra alignment option
+include { BWA_MEM      } from '../../../modules/nf-core/bwa/mem/main'
+include { BWA_INDEX      } from '../../../modules/nf-core/bwa/index/main'
+include { BWAMEM2_MEM      } from '../../../modules/nf-core/bwamem2/mem/main'
+include { BWAMEM2_INDEX      } from '../../../modules/nf-core/bwamem2/index/main'
+include { PICARD_ADDORREPLACEREADGROUPS      } from '../../../modules/nf-core/picard/addorreplacereadgroups/main'
+
+workflow FASTQ_ALIGNSORT_BWA_PICARD {
+
+    take:
+    fastqs // channel: [ val(meta), [ bam ] ]
+    reference
+    bwa
+
+    main:
+
+    versions = Channel.empty()
+
+    // switch statement to determine which bwa to use, this is a passed parameter
+    switch(bwa){
+        case 1:
+            BWA_INDEX ( reference )
+            // MEM
+            aligned_bam = BWA_MEM ( fastqs, BWA_INDEX.out.index, true ).bam.map {
+                meta, bam ->
+                    new_id = 'aligned_bam'
+                    [[id: new_id], bam ]
+            }
+            versions = versions.mix(BWA_MEM.out.versions)
+            break
+        case 2:
+            // INDEX
+            BWAMEM2_INDEX (reference)
+            versions = versions.mix(BWAMEM2_INDEX.out.versions)
+            // BWA MEM2
+            aligned_bam = BWAMEM2_MEM ( fastqs, BWAMEM2_INDEX.out.index, true ).bam.map {
+                meta, bam ->
+                    new_id = 'aligned_bam'
+                    [[id: new_id], bam ]
+            }
+            versions = versions.mix(BWAMEM2_MEM.out.versions)
+            break
+        default:
+            throw new Exception("The argument bwa must be either 1 or 2, not ${bwa}.")
+    }
+
+    // Picard add and replace
+    PICARD_ADDORREPLACEREADGROUPS(aligned_bam).bam.map {
+        meta, bam ->
+            new_id = 'grouped_aligned_bam'
+            [[id: new_id], bam ]
+    }.set {grouped_bam}
+    versions = versions.mix(PICARD_ADDORREPLACEREADGROUPS.out.versions)
+
+    // final output
+    emit:
+
+    bam      = PICARD_ADDORREPLACEREADGROUPS.out.bam           // channel: [ val(meta), [ bam ] ]
+
+    versions = versions                     // channel: [ versions.yml ]
+}
+

subworkflows/nf-core/fastq_alignsort_bwa_picard/meta.yml

@@ -0,0 +1,48 @@
+name: "alignment"
+## TODO nf-core: Add a description of the subworkflow and list keywords
+description: Sort SAM/BAM/CRAM file
+keywords:
+  - sort
+  - bam
+  - sam
+  - cram
+## TODO nf-core: Add a list of the modules used in the subworkflow
+modules:
+  - samtools/sort
+  - samtools/index
+## TODO nf-core: List all of the variables used as input, including their types and descriptions
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test' ]
+  - bam:
+      type: file
+      description: BAM/CRAM/SAM file
+      pattern: "*.{bam,cram,sam}"
+## TODO nf-core: List all of the variables used as output, including their types and descriptions
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test' ]
+  - bam:
+      type: file
+      description: Sorted BAM/CRAM/SAM file
+      pattern: "*.{bam,cram,sam}"
+  - bai:
+      type: file
+      description: BAM/CRAM/SAM samtools index
+      pattern: "*.{bai,crai,sai}"
+  - csi:
+      type: file
+      description: CSI samtools index
+      pattern: "*.csi"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+authors:
+  - "@buehlere"

subworkflows/nf-core/fastq_extractumi_fgbio_picard_gatk4_fastp/main.nf

@@ -0,0 +1,68 @@
+// TODO nf-core: If in doubt look at other nf-core/subworkflows to see how we are doing things! :)
+//               https://github.com/nf-core/modules/tree/master/subworkflows
+//               You can also ask for help via your pull request or on the #subworkflows channel on the nf-core Slack workspace:
+//               https://nf-co.re/join
+// TODO nf-core: A subworkflow SHOULD import at least two modules
+
+include { FGBIO_FASTQTOBAM } from '../../../modules/nf-core/fgbio/fastqtobam/main'
+include { PICARD_MERGESAMFILES } from '../../../modules/nf-core/picard/mergesamfiles/'
+include { GATK4_SAMTOFASTQ } from '../../../modules/nf-core/gatk4/samtofastq/main'
+include { FASTP } from '../../../modules/nf-core/fastp/main'
+
+workflow FASTQ_EXTRACTUMI_FGBIO_PICARD_GATK4_FASTP {
+
+    take:
+    // TODO nf-core: edit input (take) channels
+    ch_fastq // channel: [ val(meta), [ bam ] ]
+
+    main:
+
+    ch_versions = Channel.empty()
+
+    // FGBIO_FASTQTOBAM: get unmerged bams
+    // ch_fastq is a channel, which enables parallel
+    // channels enable parallel: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel
+    FGBIO_FASTQTOBAM (
+        ch_fastq
+    )
+    FGBIO_FASTQTOBAM.out.bam.map{
+        meta, bam ->
+            [bam]
+    }.collect().map{
+        bams ->
+        [[id: 'unmerged_bams'], bams ]
+    }.set{unmerged_bams}
+    ch_versions = ch_versions.mix(FGBIO_FASTQTOBAM.out.versions) //write out versioning
+
+    // PICARD_MERGESAMFILES: merge bams files
+    PICARD_MERGESAMFILES (
+        unmerged_bams
+    ).bam.map {
+        meta, bam ->
+            new_id = 'merged_bam'
+            [[id: new_id], bam ]
+    }.set {merged_bam}
+    ch_versions = ch_versions.mix(PICARD_MERGESAMFILES.out.versions)
+    // GATK4_SAMTOFASTQ: get fastqs from merged bam
+    GATK4_SAMTOFASTQ (
+        merged_bam
+    ).fastq.map {
+        meta, fastq ->
+            new_id = 'merged_fastq'
+            [[id: new_id], fastq ]
+    }.set {merged_fastq}
+    ch_versions = ch_versions.mix(GATK4_SAMTOFASTQ.out.versions)
+
+    // GATK4_SAMTOFASTQ: Run fastp on fastqs
+    FASTP (
+        merged_fastq, [], false, false
+    )
+    ch_versions = ch_versions.mix(FASTP.out.versions)
+    // final emit
+    emit:
+    // TODO nf-core: edit emitted channels
+    fastq = FASTP.out.reads
+
+    versions = ch_versions                     // channel: [ versions.yml ]
+
+}

subworkflows/nf-core/fastq_extractumi_fgbio_picard_gatk4_fastp/meta.yml

@@ -0,0 +1,48 @@
+name: "fastq_extractumi_fgbio_picard_gatk4_fastp"
+## TODO nf-core: Add a description of the subworkflow and list keywords
+description: Sort SAM/BAM/CRAM file
+keywords:
+  - sort
+  - bam
+  - sam
+  - cram
+## TODO nf-core: Add a list of the modules used in the subworkflow
+modules:
+  - samtools/sort
+  - samtools/index
+## TODO nf-core: List all of the variables used as input, including their types and descriptions
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test' ]
+  - bam:
+      type: file
+      description: BAM/CRAM/SAM file
+      pattern: "*.{bam,cram,sam}"
+## TODO nf-core: List all of the variables used as output, including their types and descriptions
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test' ]
+  - bam:
+      type: file
+      description: Sorted BAM/CRAM/SAM file
+      pattern: "*.{bam,cram,sam}"
+  - bai:
+      type: file
+      description: BAM/CRAM/SAM samtools index
+      pattern: "*.{bai,crai,sai}"
+  - csi:
+      type: file
+      description: CSI samtools index
+      pattern: "*.csi"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+authors:
+  - "@buehlere"

tests/config/pytest_modules.yml

@@ -3696,6 +3696,10 @@ subworkflows/fasta_newick_epang_gappa:
   - subworkflows/nf-core/fasta_newick_epang_gappa/**
   - tests/subworkflows/nf-core/fasta_newick_epang_gappa/**
 
+subworkflows/fastq_alignsort_bwa_picard:
+  - subworkflows/nf-core/fastq_alignsort_bwa_picard/**
+  - tests/subworkflows/nf-core/fastq_alignsort_bwa_picard/**
+
 subworkflows/fastq_align_bowtie2:
   - subworkflows/nf-core/fastq_align_bowtie2/**
   - tests/subworkflows/nf-core/fastq_align_bowtie2/**
@@ -3736,6 +3740,10 @@ subworkflows/fastq_download_prefetch_fasterqdump_sratools:
   - subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/**
   - tests/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/**
 
+subworkflows/fastq_extractumi_fgbio_picard_gatk4_fastp:
+  - subworkflows/nf-core/fastq_extractumi_fgbio_picard_gatk4_fastp/**
+  - tests/subworkflows/nf-core/fastq_extractumi_fgbio_picard_gatk4_fastp/**
+
 subworkflows/fastq_fastqc_umitools_fastp:
   - subworkflows/nf-core/fastq_fastqc_umitools_fastp/**
   - tests/subworkflows/nf-core/fastq_fastqc_umitools_fastp/**

tests/subworkflows/nf-core/fastq_alignsort_bwa_picard/main.nf

@@ -0,0 +1,37 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { FASTQ_EXTRACTUMI_FGBIO_PICARD_GATK4_FASTP } from '../../../../subworkflows/nf-core/fastq_extractumi_fgbio_picard_gatk4_fastp/main.nf'
+include { FASTQ_ALIGNSORT_BWA_PICARD } from '../../../../subworkflows/nf-core/fastq_alignsort_bwa_picard/main.nf'
+
+workflow test_fastq_alignsort_bwa_picard {
+    // load test data
+    def bashScriptFile = new File('install_data.sh')
+
+    def processBuilder = new ProcessBuilder('bash', bashScriptFile.toString())
+    processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT)
+    processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT)
+
+    def process = processBuilder.start()
+    process.waitFor()
+
+    // run extract umi
+    fastq = [
+    [[id:'gene1', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ad.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ad.fastq.gz')]],
+    [[id:'gene2', single_end:false], [file('test_nucleo/fastq/seracare_0-5_R1_001ae.fastq.gz'), file('test_nucleo/fastq/seracare_0-5_R2_001ae.fastq.gz')]]
+    ]
+    fastq = ch_fastq = Channel.fromList(fastq)
+    FASTQ_EXTRACTUMI_FGBIO_PICARD_GATK4_FASTP ( fastq )
+
+
+
+    // channels enable parallel: https://www.nextflow.io/docs/latest/faq.html?highlight=parallel
+    // test data
+    reference = [
+        [id:'reference'],
+        file('test_nucleo/reference/chr14_chr16.fasta')
+    ]
+    // workflow
+     FASTQ_ALIGNSORT_BWA_PICARD ( FASTQ_EXTRACTUMI_FGBIO_PICARD_GATK4_FASTP.out.fastq, reference, 1)
+}

tests/subworkflows/nf-core/fastq_alignsort_bwa_picard/nextflow.config

@@ -0,0 +1,12 @@
+executor.cpus          = 12
+executor.memory        = 15.GB
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+    withName:PICARD_ADDORREPLACEREADGROUPS{
+        ext.args = "--RGID 4 --RGLB 'lib1' --RGPL 'ILLUMINA' --RGPU 'unit1' --RGSM 20"
+    }
+    withName: BWA_MEM {
+        ext.args2 = { sort_bam ? "" : "-bh" }
+    }
+}

tests/subworkflows/nf-core/fastq_alignsort_bwa_picard/test.yml

@@ -0,0 +1,45 @@
+- name: fastq_alignsort_bwa_picard test_fastq_alignsort_bwa_picard
+  command: nextflow run ./tests/subworkflows/nf-core/fastq_alignsort_bwa_picard -entry test_fastq_alignsort_bwa_picard -c ./tests/config/nextflow.config
+  tags:
+    - bwa
+    - bwa/index
+    - bwa/mem
+    - bwamem2
+    - bwamem2/index
+    - bwamem2/mem
+    - picard
+    - picard/addorreplacereadgroups
+    - subworkflows
+    - subworkflows/fastq_alignsort_bwa_picard
+  files:
+    - path: output/bwa/bwa/chr14_chr16.amb
+      md5sum: 00fb74627e074db6238dcd9bc08dc48a
+    - path: output/bwa/bwa/chr14_chr16.ann
+      md5sum: d8825e2fcb3cd372cd61ededfe283025
+    - path: output/bwa/bwa/chr14_chr16.bwt
+      md5sum: 45637ec2c011d0f73cac6c470c5b5d2b
+    - path: output/bwa/bwa/chr14_chr16.pac
+      md5sum: 46f856371d59e859295497c967478d31
+    - path: output/bwa/bwa/chr14_chr16.sa
+      md5sum: 466dbbbce2fb9528e760477ccdc2ea5b
+    - path: output/bwa/merged_fastq.bam
+      md5sum: cb3f0b70c106c737ccd21eb590611bab
+    - path: output/fastp/merged_fastq.fastp.html
+    - path: output/fastp/merged_fastq.fastp.json
+      md5sum: 8c9cfed703141dd32818ce8abe9d2465
+    - path: output/fastp/merged_fastq.fastp.log
+    - path: output/fastp/merged_fastq_1.fastp.fastq.gz
+      md5sum: fdba450358c761a3e4264164da8b60b1
+    - path: output/fastp/merged_fastq_2.fastp.fastq.gz
+      md5sum: ca18410c7ebc49c778fe3dd059e8fee9
+    - path: output/fgbio/gene1.bam
+      md5sum: fb9749c10bbf3917dbf739de827276f7
+    - path: output/fgbio/gene2.bam
+      md5sum: 4596c2c2a8f94d770410246fb03f3baa
+    - path: output/gatk4/merged_bam_1.fastq.gz
+      md5sum: 8d87d76a1a07892a4bd0b2709338831e
+    - path: output/gatk4/merged_bam_2.fastq.gz
+      md5sum: 94c76d2f77a71d81ac26adcf94720ecd
+    - path: output/picard/aligned_bam.bam
+      md5sum: 975d449b09f28bc48456b0c6942f39d4
+    - path: output/picard/unmerged_bams.bam