From 226a8d7ded99ca1d3c23f50d1dad2906d02e780a Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 3 Feb 2023 12:20:58 -0500 Subject: [PATCH 1/9] added custom/splitfastqbylane module --- .../nf-core/custom/splitfastqbylane/main.nf | 35 +++++++++++++++ .../nf-core/custom/splitfastqbylane/meta.yml | 43 +++++++++++++++++++ .../templates/split_lanes_awk.sh | 36 ++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ .../nf-core/custom/splitfastqbylane/main.nf | 27 ++++++++++++ .../custom/splitfastqbylane/nextflow.config | 5 +++ .../nf-core/custom/splitfastqbylane/test.yml | 31 +++++++++++++ 7 files changed, 181 insertions(+) create mode 100644 modules/nf-core/custom/splitfastqbylane/main.nf create mode 100644 modules/nf-core/custom/splitfastqbylane/meta.yml create mode 100644 modules/nf-core/custom/splitfastqbylane/templates/split_lanes_awk.sh create mode 100644 tests/modules/nf-core/custom/splitfastqbylane/main.nf create mode 100644 tests/modules/nf-core/custom/splitfastqbylane/nextflow.config create mode 100644 tests/modules/nf-core/custom/splitfastqbylane/test.yml diff --git a/modules/nf-core/custom/splitfastqbylane/main.nf b/modules/nf-core/custom/splitfastqbylane/main.nf new file mode 100644 index 000000000000..b266f000bf7c --- /dev/null +++ b/modules/nf-core/custom/splitfastqbylane/main.nf @@ -0,0 +1,35 @@ +process CUSTOM_SPLITFASTQBYLANE { + tag "$meta.id" + label 'process_single' + + conda "anaconda::gawk=5.1.0" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/gawk:5.1.0': + 'quay.io/biocontainers/gawk:5.1.0' }" + + input: + tuple val(meta), path(reads) + + output: + tuple val(meta), path("*.split.fastq.gz"), emit: reads + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + shell: + args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + read1 = [reads].flatten()[0] + read2 = [reads].flatten().size() > 1 ? reads[1] : null + template 'split_lanes_awk.sh' + + stub: + """ + touch out.split.fastq.gz + cat <<-END_VERSIONS > versions.yml + "${task.process}": + gawk: \$(awk -Wversion | sed '1!d; s/.*Awk //; s/,.*//') + END_VERSIONS + """ +} diff --git a/modules/nf-core/custom/splitfastqbylane/meta.yml b/modules/nf-core/custom/splitfastqbylane/meta.yml new file mode 100644 index 000000000000..4a1086058753 --- /dev/null +++ b/modules/nf-core/custom/splitfastqbylane/meta.yml @@ -0,0 +1,43 @@ +name: "custom_splitfastqbylane" +description: | + Splits fastq files with multiple or unknown number of merged flowcell+lane sources into separate fastq files, each with single flowcell+line sources. +keywords: + - splitlanesbyfastq + - awk +tools: + - "gawk": + description: "The awk utility interprets a special-purpose programming language that makes it easy to handle simple data-reformatting jobs." + homepage: "https://www.gnu.org/software/gawk/manual/gawk.html" + documentation: "https://www.gnu.org/software/gawk/manual/gawk.html" + tool_dev_url: "http://savannah.gnu.org/projects/gawk/" + doi: "" + licence: "['GPL v3']" + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: List of input FastQ files of size 1 and 2 for single-end and paired-end data, respectively. + pattern: "*.{fastq,fastq.gz}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - reads: + type: file + description: List of output FastQ files of size of multiples of 1 and 2, for single-end and paired-end data, respectively. + pattern: "*.split.fastq.gz" + +authors: + - "@anoronh4" diff --git a/modules/nf-core/custom/splitfastqbylane/templates/split_lanes_awk.sh b/modules/nf-core/custom/splitfastqbylane/templates/split_lanes_awk.sh new file mode 100644 index 000000000000..d6d7a65ea537 --- /dev/null +++ b/modules/nf-core/custom/splitfastqbylane/templates/split_lanes_awk.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +if [[ "!{read1}" == *gz ]] ; then + cat_="zcat" +else + cat_="cat" +fi + +function a() { + awk \ + -v prefix=!{prefix} \ + -v readnumber=$1 \ + ' + BEGIN {FS = ":"} + { + lane=$(NF-3) + flowcell=$(NF-4) + outfastq=prefix"@"flowcell"_L00"lane"_R"readnumber".split.fastq.gz" + print | "gzip > "outfastq + for (i = 1; i <= 3; i++) { + getline + print | "gzip > "outfastq + } + } + ' <( eval "$cat_ $2") +} + +a 1 !{read1} +if [ ! -z !{read2} ] ; then + a 2 !{read2} +fi + +cat <<-END_VERSIONS > versions.yml +"!{task.process}": + gawk: $(awk -Wversion | sed '1!d; s/.*Awk //; s/,.*//') +END_VERSIONS diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 0cc64cac0948..340c6c385942 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -715,6 +715,10 @@ custom/matrixfilter: - modules/nf-core/custom/matrixfilter/** - tests/modules/nf-core/custom/matrixfilter/** +custom/splitfastqbylane: + - modules/nf-core/custom/splitfastqbylane/** + - tests/modules/nf-core/custom/splitfastqbylane/** + custom/sratoolsncbisettings: - modules/nf-core/custom/sratoolsncbisettings/** - tests/modules/nf-core/custom/sratoolsncbisettings/** diff --git a/tests/modules/nf-core/custom/splitfastqbylane/main.nf b/tests/modules/nf-core/custom/splitfastqbylane/main.nf new file mode 100644 index 000000000000..6cc2165e3293 --- /dev/null +++ b/tests/modules/nf-core/custom/splitfastqbylane/main.nf @@ -0,0 +1,27 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { CUSTOM_SPLITFASTQBYLANE } from '../../../../../modules/nf-core/custom/splitfastqbylane/main.nf' + +workflow test_custom_splitfastqbylane { + + input = [ + [ id:'test', single_end:false ], // meta map + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) + ] + ] + + CUSTOM_SPLITFASTQBYLANE ( input ) +} + +workflow test_custom_splitfastqbylane_single_end { + + input = [ + [ id:'test', single_end:true ], + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] + CUSTOM_SPLITFASTQBYLANE ( input ) +} diff --git a/tests/modules/nf-core/custom/splitfastqbylane/nextflow.config b/tests/modules/nf-core/custom/splitfastqbylane/nextflow.config new file mode 100644 index 000000000000..50f50a7a3579 --- /dev/null +++ b/tests/modules/nf-core/custom/splitfastqbylane/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} \ No newline at end of file diff --git a/tests/modules/nf-core/custom/splitfastqbylane/test.yml b/tests/modules/nf-core/custom/splitfastqbylane/test.yml new file mode 100644 index 000000000000..9798fc496a39 --- /dev/null +++ b/tests/modules/nf-core/custom/splitfastqbylane/test.yml @@ -0,0 +1,31 @@ +- name: custom splitfastqbylane test_custom_splitfastqbylane + command: nextflow run ./tests/modules/nf-core/custom/splitfastqbylane -entry test_custom_splitfastqbylane -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/custom/splitfastqbylane/nextflow.config + tags: + - custom + - custom/splitfastqbylane + files: + - path: output/custom/test@HK3MMAFX2_L001_R1.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L001_R2.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L002_R1.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L002_R2.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L003_R1.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L003_R2.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L004_R1.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L004_R2.split.fastq.gz + - path: output/custom/versions.yml + contains: + - "gawk" + +- name: custom splitfastqbylane test_custom_splitfastqbylane_single_end + command: nextflow run ./tests/modules/nf-core/custom/splitfastqbylane -entry test_custom_splitfastqbylane_single_end -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/custom/splitfastqbylane/nextflow.config + tags: + - custom + - custom/splitfastqbylane + files: + - path: output/custom/test@HK3MMAFX2_L001_R1.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L002_R1.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L003_R1.split.fastq.gz + - path: output/custom/test@HK3MMAFX2_L004_R1.split.fastq.gz + - path: output/custom/versions.yml + contains: + - "gawk" From 1811877191f60cbe35731412adb1a52ca12fefed Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 3 Feb 2023 12:44:35 -0500 Subject: [PATCH 2/9] fix trailing spaces and amount of left-padding spaces --- .../templates/split_lanes_awk.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/nf-core/custom/splitfastqbylane/templates/split_lanes_awk.sh b/modules/nf-core/custom/splitfastqbylane/templates/split_lanes_awk.sh index d6d7a65ea537..6ca616154b53 100644 --- a/modules/nf-core/custom/splitfastqbylane/templates/split_lanes_awk.sh +++ b/modules/nf-core/custom/splitfastqbylane/templates/split_lanes_awk.sh @@ -1,16 +1,16 @@ #!/bin/bash -if [[ "!{read1}" == *gz ]] ; then - cat_="zcat" -else - cat_="cat" +if [[ "!{read1}" == *gz ]] ; then + cat_="zcat" +else + cat_="cat" fi function a() { awk \ - -v prefix=!{prefix} \ - -v readnumber=$1 \ - ' + -v prefix=!{prefix} \ + -v readnumber=$1 \ + ' BEGIN {FS = ":"} { lane=$(NF-3) @@ -25,8 +25,8 @@ function a() { ' <( eval "$cat_ $2") } -a 1 !{read1} -if [ ! -z !{read2} ] ; then +a 1 !{read1} +if [ ! -z !{read2} ] ; then a 2 !{read2} fi From f941b21d569305cd65de923304a4dd15e31dabae Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 7 Jul 2023 12:43:35 -0400 Subject: [PATCH 3/9] added custom/splitfastqbylane module From 605e4c3deb4eb8624bddd90611ab291bd3ac260b Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 7 Jul 2023 12:59:49 -0400 Subject: [PATCH 4/9] add keywords in meta.yml --- modules/nf-core/custom/splitfastqbylane/meta.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/nf-core/custom/splitfastqbylane/meta.yml b/modules/nf-core/custom/splitfastqbylane/meta.yml index 4a1086058753..359560be0772 100644 --- a/modules/nf-core/custom/splitfastqbylane/meta.yml +++ b/modules/nf-core/custom/splitfastqbylane/meta.yml @@ -4,6 +4,10 @@ description: | keywords: - splitlanesbyfastq - awk + - custom + - fastq + - split + - lanes tools: - "gawk": description: "The awk utility interprets a special-purpose programming language that makes it easy to handle simple data-reformatting jobs." From ba45de604e6a7f13e46ded9d132741c9debc9b73 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 7 Jul 2023 13:04:29 -0400 Subject: [PATCH 5/9] fixed quay container --- modules/nf-core/custom/splitfastqbylane/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/custom/splitfastqbylane/main.nf b/modules/nf-core/custom/splitfastqbylane/main.nf index b266f000bf7c..5844fd8dd3bb 100644 --- a/modules/nf-core/custom/splitfastqbylane/main.nf +++ b/modules/nf-core/custom/splitfastqbylane/main.nf @@ -4,8 +4,8 @@ process CUSTOM_SPLITFASTQBYLANE { conda "anaconda::gawk=5.1.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/gawk:5.1.0': - 'quay.io/biocontainers/gawk:5.1.0' }" + 'https://depot.galaxyproject.org/singularity/gawk:5.1.0' : + 'biocontainers/gawk:5.1.0' }" input: tuple val(meta), path(reads) From 5eb40695b1b34f0429cb9f0b5e2534d5cb66cec0 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 7 Jul 2023 13:22:14 -0400 Subject: [PATCH 6/9] Removed empty doi attribute to resolve linting error --- modules/nf-core/custom/splitfastqbylane/meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/nf-core/custom/splitfastqbylane/meta.yml b/modules/nf-core/custom/splitfastqbylane/meta.yml index 359560be0772..307b1c29cfcd 100644 --- a/modules/nf-core/custom/splitfastqbylane/meta.yml +++ b/modules/nf-core/custom/splitfastqbylane/meta.yml @@ -14,7 +14,6 @@ tools: homepage: "https://www.gnu.org/software/gawk/manual/gawk.html" documentation: "https://www.gnu.org/software/gawk/manual/gawk.html" tool_dev_url: "http://savannah.gnu.org/projects/gawk/" - doi: "" licence: "['GPL v3']" input: From 72481bde1438a8d53a4b8b1defe56a51cd4fd225 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 7 Jul 2023 13:47:57 -0400 Subject: [PATCH 7/9] Empty commit From 018862c7c72ef53118ea00f43b707419492cf781 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 10 Jul 2023 13:23:18 -0400 Subject: [PATCH 8/9] Empty commit From 42bed69a8f482c3b70880ceb605911f3d7695bf9 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 10 Jul 2023 15:17:17 -0400 Subject: [PATCH 9/9] Empty commit