From 06b27e0e25a894eff1418ed760b89cc49fe7dae8 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 15:48:17 -0400 Subject: [PATCH 01/33] Create sync-fork.yml --- .github/workflows/sync-fork.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/sync-fork.yml diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml new file mode 100644 index 000000000000..1b68e43e34cb --- /dev/null +++ b/.github/workflows/sync-fork.yml @@ -0,0 +1,26 @@ +name: Sync Fork + +on: + push: + branches: + - master + workflow_dispatch: # on button click + +jobs: + merge: + runs-on: ubuntu-latest + steps: + - name: Checkout branch + uses: actions/checkout@v2 + with: + ref: master + - name: Fetch changes from upstream + run: git fetch upstream + env: + upstream: https://github.com/nf-core/modules.git + - name: Merge changes from upstream + run: git merge upstream/master --no-edit + - name: Push changes to fork + uses: ad-m/github-push-action@master + with: + branch: master \ No newline at end of file From f2c759d6718aef653b77913f5719bdb0fcced7e2 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 15:52:56 -0400 Subject: [PATCH 02/33] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 1b68e43e34cb..7dc04722c82a 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -16,8 +16,6 @@ jobs: ref: master - name: Fetch changes from upstream run: git fetch upstream - env: - upstream: https://github.com/nf-core/modules.git - name: Merge changes from upstream run: git merge upstream/master --no-edit - name: Push changes to fork From 7ff8837dbc105be340bbf2778ce71c6a1aa27ef4 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 15:53:39 -0400 Subject: [PATCH 03/33] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 7dc04722c82a..dd5ed4308692 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -15,7 +15,7 @@ jobs: with: ref: master - name: Fetch changes from upstream - run: git fetch upstream + run: git fetch upstream https://github.com/nf-core/modules.git - name: Merge changes from upstream run: git merge upstream/master --no-edit - name: Push changes to fork From 03ede3c6daff68325b0e4e24b7274c3b40a2d910 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:01:11 -0400 Subject: [PATCH 04/33] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index dd5ed4308692..746034b4e4dd 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -10,12 +10,12 @@ jobs: merge: runs-on: ubuntu-latest steps: + - name: Fetch changes from upstream + run: git fetch upstream https://github.com/nf-core/modules.git - name: Checkout branch uses: actions/checkout@v2 with: ref: master - - name: Fetch changes from upstream - run: git fetch upstream https://github.com/nf-core/modules.git - name: Merge changes from upstream run: git merge upstream/master --no-edit - name: Push changes to fork From 879199a56c9a7668bf0f4a87f30f03121ae902b8 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:03:42 -0400 Subject: [PATCH 05/33] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 746034b4e4dd..389cf743afb2 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -9,16 +9,23 @@ on: jobs: merge: runs-on: ubuntu-latest - steps: - - name: Fetch changes from upstream - run: git fetch upstream https://github.com/nf-core/modules.git - - name: Checkout branch + steps: + - name: Checkout repository uses: actions/checkout@v2 - with: - ref: master - - name: Merge changes from upstream - run: git merge upstream/master --no-edit - - name: Push changes to fork - uses: ad-m/github-push-action@master - with: - branch: master \ No newline at end of file + + - name: Add upstream remote + run: | + git remote add upstream https://github.com/nf-core/modules.git + + - name: Fetch upstream changes + run: | + git fetch upstream + + - name: Merge upstream master into forked master + run: | + git checkout master + git merge upstream/master --no-edit + + - name: Push changes to forked repository + run: | + git push origin master \ No newline at end of file From c21e35b5ef6bc7a6e60e5f5a4cc44215af73767b Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:06:46 -0400 Subject: [PATCH 06/33] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 389cf743afb2..e35dcaa527cd 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -1,10 +1,8 @@ name: Sync Fork on: - push: - branches: - - master - workflow_dispatch: # on button click + schedule: + - cron: "0 0 * * *" jobs: merge: From cd0957950e9336f4674806e2916072a1a4efec57 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:10:12 -0400 Subject: [PATCH 07/33] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index e35dcaa527cd..232e5bfaa234 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -7,7 +7,7 @@ on: jobs: merge: runs-on: ubuntu-latest - steps: + steps: - name: Checkout repository uses: actions/checkout@v2 @@ -26,4 +26,5 @@ jobs: - name: Push changes to forked repository run: | - git push origin master \ No newline at end of file + git push origin master + \ No newline at end of file From 31331e4e80efe1a7f872b912033994f1a0bb436d Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 9 May 2023 16:13:37 -0400 Subject: [PATCH 08/33] Update sync-fork.yml --- .github/workflows/sync-fork.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml index 232e5bfaa234..eef323780b7a 100644 --- a/.github/workflows/sync-fork.yml +++ b/.github/workflows/sync-fork.yml @@ -27,4 +27,3 @@ jobs: - name: Push changes to forked repository run: | git push origin master - \ No newline at end of file From e0c8587417c69e1435ef0bc8d0625d77e53db2b5 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 10 May 2023 15:38:38 -0400 Subject: [PATCH 09/33] Create sync-action.yml --- .github/workflows/sync-action.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/sync-action.yml diff --git a/.github/workflows/sync-action.yml b/.github/workflows/sync-action.yml new file mode 100644 index 000000000000..3b68c7e2169d --- /dev/null +++ b/.github/workflows/sync-action.yml @@ -0,0 +1,17 @@ +name: Sync Fork + +on: + schedule: + - cron: "0 0 * * *" # run once a night at midnight + workflow_dispatch: # on button click + +jobs: + sync: + runs-on: ubuntu-latest + + steps: + - uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: master + head: master From 7215646e120cdb93f1d7dbf0b952523725370060 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:23:10 -0400 Subject: [PATCH 10/33] Update sync-action.yml --- .github/workflows/sync-action.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sync-action.yml b/.github/workflows/sync-action.yml index 3b68c7e2169d..f20440aa0395 100644 --- a/.github/workflows/sync-action.yml +++ b/.github/workflows/sync-action.yml @@ -10,8 +10,15 @@ jobs: runs-on: ubuntu-latest steps: - - uses: tgymnich/fork-sync@v1.8 - with: - token: ${{ secrets.PERSONAL_TOKEN }} - base: master - head: master + - name: Sync master branch + uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: master + head: master + - name: Sync develop branch + uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: develop + head: develop From ab7ba9828e92e9aa8d3db3095c5856b85b0e80bf Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:28:04 -0400 Subject: [PATCH 11/33] Delete sync-fork.yml --- .github/workflows/sync-fork.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/sync-fork.yml diff --git a/.github/workflows/sync-fork.yml b/.github/workflows/sync-fork.yml deleted file mode 100644 index eef323780b7a..000000000000 --- a/.github/workflows/sync-fork.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Sync Fork - -on: - schedule: - - cron: "0 0 * * *" - -jobs: - merge: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Add upstream remote - run: | - git remote add upstream https://github.com/nf-core/modules.git - - - name: Fetch upstream changes - run: | - git fetch upstream - - - name: Merge upstream master into forked master - run: | - git checkout master - git merge upstream/master --no-edit - - - name: Push changes to forked repository - run: | - git push origin master From 35baf68d990c5aa31fcc1330e823d2ac5aacf282 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:37:06 -0400 Subject: [PATCH 12/33] Update sync-action.yml --- .github/workflows/sync-action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-action.yml b/.github/workflows/sync-action.yml index f20440aa0395..67f9c9624f56 100644 --- a/.github/workflows/sync-action.yml +++ b/.github/workflows/sync-action.yml @@ -20,5 +20,5 @@ jobs: uses: tgymnich/fork-sync@v1.8 with: token: ${{ secrets.PERSONAL_TOKEN }} - base: develop + base: master head: develop From f2f4e041381c0388e718e38877d695c59f1a1667 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:37:40 -0400 Subject: [PATCH 13/33] Update sync-action.yml --- .github/workflows/sync-action.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sync-action.yml b/.github/workflows/sync-action.yml index 67f9c9624f56..a1f36450ec76 100644 --- a/.github/workflows/sync-action.yml +++ b/.github/workflows/sync-action.yml @@ -10,15 +10,15 @@ jobs: runs-on: ubuntu-latest steps: - - name: Sync master branch - uses: tgymnich/fork-sync@v1.8 - with: - token: ${{ secrets.PERSONAL_TOKEN }} - base: master - head: master - - name: Sync develop branch - uses: tgymnich/fork-sync@v1.8 - with: - token: ${{ secrets.PERSONAL_TOKEN }} - base: master - head: develop + - name: Sync master branch + uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: master + head: master + - name: Sync develop branch + uses: tgymnich/fork-sync@v1.8 + with: + token: ${{ secrets.PERSONAL_TOKEN }} + base: master + head: develop From 5912f2909a2274b926971ae3146a92b17d3d3064 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 17 May 2023 13:43:03 -0400 Subject: [PATCH 14/33] Revert "Merge pull request #16 from nf-core/master" This reverts commit 13d45de834825776f339977b9e0ef34304b14128, reversing changes made to ab7ba9828e92e9aa8d3db3095c5856b85b0e80bf. --- .github/workflows/pytest-workflow.yml | 2 - modules/nf-core/bcftools/query/main.nf | 6 +- modules/nf-core/tabix/bgziptabix/main.nf | 6 +- modules/nf-core/tabix/bgziptabix/meta.yml | 5 - .../vcf_annotate_ensemblvep_snpeff/main.nf | 191 --------- .../vcf_annotate_ensemblvep_snpeff/meta.yml | 73 ---- tests/config/pytest_modules.yml | 4 - tests/modules/nf-core/bcftools/query/test.yml | 10 +- .../modules/nf-core/tabix/bgziptabix/main.nf | 7 +- .../nf-core/tabix/bgziptabix/nextflow.config | 6 +- .../modules/nf-core/tabix/bgziptabix/test.yml | 6 +- .../vcf_annotate_ensemblvep_snpeff/main.nf | 170 -------- .../nextflow.config | 21 - .../vcf_annotate_ensemblvep_snpeff/test.yml | 384 ------------------ 14 files changed, 14 insertions(+), 877 deletions(-) delete mode 100644 subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf delete mode 100644 subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/meta.yml delete mode 100644 tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf delete mode 100644 tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/nextflow.config delete mode 100644 tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/test.yml diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index a5c7979835c1..76eec0da2eb4 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -136,8 +136,6 @@ jobs: tags: gatk4/determinegermlinecontigploidy - profile: "conda" tags: subworkflows/bcl_demultiplex - - profile: "conda" - tags: subworkflows/vcf_annotate_ensemblvep_snpeff - profile: "conda" tags: subworkflows/vcf_annotate_ensemblvep - profile: "conda" diff --git a/modules/nf-core/bcftools/query/main.nf b/modules/nf-core/bcftools/query/main.nf index 0a775ebebb27..dbc00e00b16e 100644 --- a/modules/nf-core/bcftools/query/main.nf +++ b/modules/nf-core/bcftools/query/main.nf @@ -1,6 +1,6 @@ process BCFTOOLS_QUERY { tag "$meta.id" - label 'process_single' + label 'process_medium' conda "bioconda::bcftools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? @@ -28,12 +28,12 @@ process BCFTOOLS_QUERY { def samples_file = samples ? "--samples-file ${samples}" : "" """ bcftools query \\ + --output ${prefix}.txt \\ $regions_file \\ $targets_file \\ $samples_file \\ $args \\ - $vcf \\ - > ${prefix}.txt + $vcf cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tabix/bgziptabix/main.nf b/modules/nf-core/tabix/bgziptabix/main.nf index fef528fcdf87..fe8160bae4cd 100644 --- a/modules/nf-core/tabix/bgziptabix/main.nf +++ b/modules/nf-core/tabix/bgziptabix/main.nf @@ -5,14 +5,13 @@ process TABIX_BGZIPTABIX { conda "bioconda::tabix=1.11" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0' : - 'quay.io/biocontainers/tabix:1.11--hdfd78af_0' }" + 'biocontainers/tabix:1.11--hdfd78af_0' }" input: tuple val(meta), path(input) output: - tuple val(meta), path("*.gz"), path("*.tbi"), optional: true, emit: gz_tbi - tuple val(meta), path("*.gz"), path("*.csi"), optional: true, emit: gz_csi + tuple val(meta), path("*.gz"), path("*.tbi"), emit: gz_tbi path "versions.yml" , emit: versions when: @@ -37,7 +36,6 @@ process TABIX_BGZIPTABIX { """ touch ${prefix}.${input.getExtension()}.gz touch ${prefix}.${input.getExtension()}.gz.tbi - touch ${prefix}.${input.getExtension()}.gz.csi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/tabix/bgziptabix/meta.yml b/modules/nf-core/tabix/bgziptabix/meta.yml index 2761e27183bb..49c032894157 100644 --- a/modules/nf-core/tabix/bgziptabix/meta.yml +++ b/modules/nf-core/tabix/bgziptabix/meta.yml @@ -37,14 +37,9 @@ output: type: file description: tabix index file pattern: "*.{gz.tbi}" - - csi: - type: file - description: tabix alternate index file - pattern: "*.{gz.csi}" - versions: type: file description: File containing software versions pattern: "versions.yml" authors: - "@maxulysse" - - "@DLBPointon" diff --git a/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf b/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf deleted file mode 100644 index 91cace4d1d9f..000000000000 --- a/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf +++ /dev/null @@ -1,191 +0,0 @@ -// -// Run VEP and/or SNPEFF to annotate VCF files -// - -include { ENSEMBLVEP_VEP } from '../../../modules/nf-core/ensemblvep/vep/main' -include { SNPEFF_SNPEFF } from '../../../modules/nf-core/snpeff/snpeff/main' -include { TABIX_TABIX } from '../../../modules/nf-core/tabix/tabix/main' -include { BCFTOOLS_PLUGINSCATTER } from '../../../modules/nf-core/bcftools/pluginscatter/main' -include { BCFTOOLS_CONCAT } from '../../../modules/nf-core/bcftools/concat/main' -include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort/main' - -workflow VCF_ANNOTATE_ENSEMBLVEP_SNPEFF { - take: - ch_vcf // channel: [ val(meta), path(vcf), path(tbi), [path(file1), path(file2)...] ] - ch_fasta // channel: [ val(meta2), path(fasta) ] (optional) - val_vep_genome // value: genome to use - val_vep_species // value: species to use - val_vep_cache_version // value: cache version to use - ch_vep_cache // channel: [ path(cache) ] (optional) - ch_vep_extra_files // channel: [ path(file1), path(file2)... ] (optional) - val_snpeff_db // value: the db version to use for snpEff - ch_snpeff_cache // channel: [ path(cache) ] (optional) - val_tools_to_use // value: a list of tools to use options are: ["ensemblvep", "snpeff"] - val_sites_per_chunk // value: the amount of variants per scattered VCF - - main: - ch_versions = Channel.empty() - - // Check if val_sites_per_chunk is set and scatter if it is - if(val_sites_per_chunk) { - // - // Prepare the input VCF channel for scattering (split VCFs from custom files) - // - - ch_input = ch_vcf - .multiMap { meta, vcf, tbi, custom_files -> - vcf: [ meta, vcf, tbi ] - custom: [ meta, custom_files ] - } - - // - // Scatter the input VCFs into multiple VCFs. These VCFs contain the amount of variants - // specified by `val_sites_per_chunk`. The lower this value is, the more files will be created - // - - BCFTOOLS_PLUGINSCATTER( - ch_input.vcf, - val_sites_per_chunk, - [], - [], - [], - [] - ) - ch_versions = ch_versions.mix(BCFTOOLS_PLUGINSCATTER.out.versions.first()) - - // - // Run the annotation with EnsemblVEP - // - - ch_scatter = BCFTOOLS_PLUGINSCATTER.out.scatter - .map { meta, vcfs -> - // This checks if multiple files were created using the scatter process - // If multiple files are created, a list will be made as output of the process - // So if the output isn't a list, there is always one file and if there is a list, - // the amount of files in the list gets counted by .size() - is_list = vcfs instanceof ArrayList - count = is_list ? vcfs.size() : 1 - [ meta, is_list ? vcfs : [vcfs], count ] - // Channel containing the list of VCFs and the size of this list - } - .transpose(by:1) // Transpose on the VCFs => Creates an entry for each VCF in the list - .combine(ch_input.custom, by: 0) // Re-add the sample specific custom files - .multiMap { meta, vcf, count, custom_files -> - // Define the new ID. The `_annotated` is to disambiguate the VEP output with its input - new_id = "${meta.id}${vcf.name.replace(meta.id,"").tokenize(".")[0]}_annotated" as String - new_meta = meta + [id:new_id] - - // Create channels: one with the VEP input and one with the original ID and count of scattered VCFs - input: [ new_meta, vcf, custom_files ] - count: [ new_meta, meta.id, count ] - } - - ch_vep_input = ch_scatter.input - } else { - // Use the normal input when no scattering has to be performed - ch_vep_input = ch_vcf.map { meta, vcf, tbi, files -> [ meta, vcf, files ] } - } - - // Annotate with ensemblvep if it's part of the requested tools - if("ensemblvep" in val_tools_to_use){ - ENSEMBLVEP_VEP( - ch_vep_input, - val_vep_genome, - val_vep_species, - val_vep_cache_version, - ch_vep_cache, - ch_fasta, - ch_vep_extra_files - ) - ch_versions = ch_versions.mix(ENSEMBLVEP_VEP.out.versions.first()) - - ch_vep_output = ENSEMBLVEP_VEP.out.vcf - ch_vep_reports = ENSEMBLVEP_VEP.out.report - } else { - ch_vep_output = ch_vep_input.map { meta, vcf, files -> [ meta, vcf ] } - ch_vep_reports = Channel.empty() - } - - // Annotate with snpeff if it's part of the requested tools - if("snpeff" in val_tools_to_use){ - SNPEFF_SNPEFF( - ch_vep_output, - val_snpeff_db, - ch_snpeff_cache - ) - ch_versions = ch_versions.mix(SNPEFF_SNPEFF.out.versions.first()) - - ch_snpeff_output = SNPEFF_SNPEFF.out.vcf - ch_snpeff_reports = SNPEFF_SNPEFF.out.report - ch_snpeff_html = SNPEFF_SNPEFF.out.summary_html - ch_snpeff_genes = SNPEFF_SNPEFF.out.genes_txt - } else { - ch_snpeff_output = ch_vep_output - ch_snpeff_reports = Channel.empty() - ch_snpeff_html = Channel.empty() - ch_snpeff_genes = Channel.empty() - } - - // Gather the files back together if they were scattered - if(val_sites_per_chunk) { - // - // Concatenate the VCFs back together with bcftools concat - // - - ch_concat_input = ch_snpeff_output - .join(ch_scatter.count, failOnDuplicate:true, failOnMismatch:true) - .map { meta, vcf, id, count -> - new_meta = meta + [id:id] - [ groupKey(new_meta, count), vcf ] - } - .groupTuple() // Group the VCFs which need to be concatenated - .map { it + [[]] } - - BCFTOOLS_CONCAT( - ch_concat_input - ) - ch_versions = ch_versions.mix(BCFTOOLS_CONCAT.out.versions.first()) - - // - // Sort the concatenate output (bcftools concat is unable to do this on its own) - // - - BCFTOOLS_SORT( - BCFTOOLS_CONCAT.out.vcf - ) - ch_versions = ch_versions.mix(BCFTOOLS_SORT.out.versions.first()) - - ch_ready_vcfs = BCFTOOLS_SORT.out.vcf - } else { - ch_ready_vcfs = ch_snpeff_output - } - - // - // Index the resulting bgzipped VCFs - // - - ch_tabix_input = ch_ready_vcfs - .branch { meta, vcf -> - // Split the bgzipped VCFs from the unzipped VCFs (only bgzipped VCFs should be indexed) - bgzip: vcf.extension == "gz" - unzip: true - return [ meta, vcf, [] ] - } - - TABIX_TABIX( - ch_tabix_input.bgzip - ) - ch_versions = ch_versions.mix(TABIX_TABIX.out.versions) - - ch_vcf_tbi = ch_tabix_input.bgzip - .join(TABIX_TABIX.out.tbi, failOnDuplicate: true, failOnMismatch: true) - .mix(ch_tabix_input.unzip) - - emit: - vcf_tbi = ch_vcf_tbi // channel: [ val(meta), path(vcf), path(tbi) ] - vep_reports = ch_vep_reports // channel: [ path(html) ] - snpeff_reports = ch_snpeff_reports // channel: [ path(csv) ] - snpeff_html = ch_snpeff_html // channel: [ path(html) ] - snpeff_genes = ch_snpeff_genes // channel: [ path(genes) ] - versions = ch_versions // channel: [ versions.yml ] -} diff --git a/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/meta.yml b/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/meta.yml deleted file mode 100644 index 211144330be6..000000000000 --- a/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/meta.yml +++ /dev/null @@ -1,73 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: vcf_annotate_ensemblvep_snpeff -description: | - Perform annotation with ensemblvep and/or snpeff and bgzip + tabix index the resulting VCF file. This subworkflow uses the scatter-gather method to run VEP/snpEff in parallel to increase throughput. The input VCF is split into multiple smaller VCFs of fixed size, which are annotated separately and concatenated back together to a single output file per sample. Only VCF/BCF outputs are currently supported. -keywords: - - vcf - - annotation - - ensemblvep - - snpeff -modules: - - ensemblvep/vep - - snpeff/snpeff - - tabix/tabix - - bcftools/pluginscatter - - bcftools/concat -input: - - ch_vcf: - description: | - vcf file to annotate - Structure: [ val(meta), path(vcf), path(tbi) ] - - ch_fasta: - description: | - Reference genome fasta file (optional) - Structure: [ val(meta2), path(fasta) ] - - val_vep_genome: - type: string - description: genome to use for ensemblvep - - val_vep_species: - type: string - description: species to use for ensemblvep - - val_vep_cache_version: - type: integer - description: cache version to use for ensemblvep - - ch_vep_cache: - description: | - the root cache folder for ensemblvep (optional) - Structure: [ path(cache) ] - - ch_vep_extra_files: - description: | - any extra files needed by plugins for ensemblvep (optional) - Structure: [ path(file1), path(file2)... ] - - val_snpeff_db: - type: string - description: database to use for snpeff - - ch_snpeff_cache: - description: | - the root cache folder for snpeff (optional) - Structure: [ path(cache) ] - - val_tools_to_use: - type: list - description: The tools to use. Options => '["ensemblvep", "snpeff"]' - - val_sites_per_chunk: - type: integer - description: | - The amount of variants per scattered VCF. - Set this value to `null`, `[]` or `false` to disable scattering. -output: - - vcf_tbi: - description: | - Compressed vcf file + tabix index - Structure: [ val(meta), path(vcf), path(tbi) ] - - reports: - type: file - description: html reports - pattern: "*.html" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@maxulysse" - - "@matthdsm" - - "@nvnieuwk" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index db13db14cb86..71c6735a89ab 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -3544,10 +3544,6 @@ subworkflows/vcf_annotate_ensemblvep: - subworkflows/nf-core/vcf_annotate_ensemblvep/** - tests/subworkflows/nf-core/vcf_annotate_ensemblvep/** -subworkflows/vcf_annotate_ensemblvep_snpeff: - - subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/** - - tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/** - subworkflows/vcf_annotate_snpeff: - subworkflows/nf-core/vcf_annotate_snpeff/** - tests/subworkflows/nf-core/vcf_annotate_snpeff/** diff --git a/tests/modules/nf-core/bcftools/query/test.yml b/tests/modules/nf-core/bcftools/query/test.yml index 7acf1880dbda..829ee630e4d9 100644 --- a/tests/modules/nf-core/bcftools/query/test.yml +++ b/tests/modules/nf-core/bcftools/query/test.yml @@ -1,19 +1,17 @@ -- name: bcftools query test_bcftools_query +- name: bcftools query command: nextflow run ./tests/modules/nf-core/bcftools/query -entry test_bcftools_query -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/bcftools/query/nextflow.config tags: - bcftools - bcftools/query files: - path: output/bcftools/out.txt - md5sum: 51d135de052f3bcef50dcd6b74806094 - - path: output/bcftools/versions.yml + md5sum: c32a6d28f185822d8fe1eeb7e42ec155 -- name: bcftools query test_bcftools_query_with_optional_files +- name: bcftools query with optional files command: nextflow run ./tests/modules/nf-core/bcftools/query -entry test_bcftools_query_with_optional_files -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/bcftools/query/nextflow.config tags: - bcftools - bcftools/query files: - path: output/bcftools/out.txt - md5sum: 1785d1957ba7206df852d0689b91753f - - path: output/bcftools/versions.yml + md5sum: 5a87e0865df2f0ab2884fc113ec2a70d diff --git a/tests/modules/nf-core/tabix/bgziptabix/main.nf b/tests/modules/nf-core/tabix/bgziptabix/main.nf index f4bc4754bf58..26e17101f8b9 100644 --- a/tests/modules/nf-core/tabix/bgziptabix/main.nf +++ b/tests/modules/nf-core/tabix/bgziptabix/main.nf @@ -2,15 +2,12 @@ nextflow.enable.dsl = 2 -include { TABIX_BGZIPTABIX as TABIX_BGZIPTABIX_TBI } from '../../../../../modules/nf-core/tabix/bgziptabix/main.nf' -include { TABIX_BGZIPTABIX as TABIX_BGZIPTABIX_CSI } from '../../../../../modules/nf-core/tabix/bgziptabix/main.nf' +include { TABIX_BGZIPTABIX } from '../../../../../modules/nf-core/tabix/bgziptabix/main.nf' workflow test_tabix_bgziptabix { input = [ [ id:'test' ], // meta map [ file(params.test_data['sarscov2']['illumina']['test_vcf'], checkIfExists: true) ] ] - TABIX_BGZIPTABIX_TBI ( input ) - TABIX_BGZIPTABIX_CSI ( input ) - + TABIX_BGZIPTABIX ( input ) } diff --git a/tests/modules/nf-core/tabix/bgziptabix/nextflow.config b/tests/modules/nf-core/tabix/bgziptabix/nextflow.config index 7830e9153461..041bfa6acc1c 100644 --- a/tests/modules/nf-core/tabix/bgziptabix/nextflow.config +++ b/tests/modules/nf-core/tabix/bgziptabix/nextflow.config @@ -2,12 +2,8 @@ process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - withName: TABIX_BGZIPTABIX_TBI { + withName: TABIX_BGZIPTABIX { ext.args2 = '-p vcf' } - withName: TABIX_BGZIPTABIX_CSI { - ext.args2 = '-p vcf --csi' - } - } diff --git a/tests/modules/nf-core/tabix/bgziptabix/test.yml b/tests/modules/nf-core/tabix/bgziptabix/test.yml index 6bbfb09c798d..78ce288b3218 100644 --- a/tests/modules/nf-core/tabix/bgziptabix/test.yml +++ b/tests/modules/nf-core/tabix/bgziptabix/test.yml @@ -5,8 +5,6 @@ - tabix files: - path: output/tabix/test.vcf.gz - md5sum: 02aff9ae43faf2594f28e2cf87b32ba9 + md5sum: fc178eb342a91dc0d1d568601ad8f8e2 - path: output/tabix/test.vcf.gz.tbi - md5sum: d22e5b84e4fcd18792179f72e6da702e - - path: output/tabix/test.vcf.gz.csi - md5sum: 6f733d80ee760fcc8fdbe504a03f2640 + md5sum: 36e11bf96ed0af4a92caa91a68612d64 diff --git a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf b/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf deleted file mode 100644 index d6af5c2c9998..000000000000 --- a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { VCF_ANNOTATE_ENSEMBLVEP_SNPEFF } from '../../../../subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/main.nf' - -workflow vcf_annotate_ensemblvep_snpeff_vep { - input = Channel.of([ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - [[],[]], - "WBcel235", - "caenorhabditis_elegans", - "108", - [], - [], - [], - [], - ["ensemblvep"], - 5 - ) -} - -workflow vcf_annotate_ensemblvep_snpeff_snpeff { - input = Channel.of([ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - [[],[]], - [], - [], - [], - [], - [], - "WBcel235.99", - [], - ["snpeff"], - 5 - ) -} - -workflow vcf_annotate_ensemblvep_snpeff_both { - input = Channel.of([ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - [[],[]], - "WBcel235", - "caenorhabditis_elegans", - "108", - [], - [], - "WBcel235.99", - [], - ["snpeff", "ensemblvep"], - 5 - ) -} - -workflow vcf_annotate_ensemblvep_snpeff_large_chunks { - input = Channel.of([ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - fasta = [ - [id:"fasta"], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - fasta, - "WBcel235", - "caenorhabditis_elegans", - "108", - [], - [], - [], - [], - ["ensemblvep"], - 100 - ) -} - -workflow vcf_annotate_ensemblvep_snpeff_no_scatter { - input = Channel.of([ - [ id:'test1' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_vcf_gz_tbi'], checkIfExists: true), - [] - ],[ - [ id:'custom_test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test2_vcf_gz_tbi'], checkIfExists: true), - [ - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test3_vcf_gz_tbi'], checkIfExists: true) - ] - ]) - - fasta = [ - [id:"fasta"], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] - - VCF_ANNOTATE_ENSEMBLVEP_SNPEFF ( - input, - fasta, - "WBcel235", - "caenorhabditis_elegans", - "108", - [], - [], - [], - [], - ["ensemblvep"], - [] - ) -} diff --git a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/nextflow.config b/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/nextflow.config deleted file mode 100644 index e48fa306d676..000000000000 --- a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/nextflow.config +++ /dev/null @@ -1,21 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - - withName: ENSEMBLVEP_VEP { - container = 'docker.io/nfcore/vep:108.2.WBcel235' - ext.args = {[ - "--vcf --offline", - meta.id.startsWith("custom_test") ? "--custom test3.vcf.gz,,vcf,exact,0,TOPMED" : "" - ].join(" ")} - } - - withName: SNPEFF_SNPEFF { - container = 'docker.io/nfcore/snpeff:5.0.WBcel235' - } - - withName: BCFTOOLS_CONCAT { - ext.prefix = { "${meta.id}_concat" } - } - -} diff --git a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/test.yml b/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/test.yml deleted file mode 100644 index 92fa202cb90d..000000000000 --- a/tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff/test.yml +++ /dev/null @@ -1,384 +0,0 @@ -- name: vcf_annotate_ensemblvep_snpeff vcf_annotate_ensemblvep_snpeff_vep - command: nextflow run ./tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff -entry vcf_annotate_ensemblvep_snpeff_vep -c ./tests/config/nextflow.config - tags: - - bcftools - - bcftools/concat - - bcftools/pluginscatter - - bcftools/sort - - ensemblvep - - ensemblvep/vep - - snpeff - - snpeff/snpeff - - subworkflows - - subworkflows/vcf_annotate_ensemblvep_snpeff - - tabix - - tabix/tabix - files: - - path: output/bcftools/custom_test.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test0.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test1.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test_concat.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test0.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test1.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test_concat.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/ensemblvep/custom_test0_annotated.summary.html - contains: - - "" - - "" - - path: output/ensemblvep/custom_test0_annotated.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/ensemblvep/custom_test1_annotated.summary.html - contains: - - "" - - "" - - path: output/ensemblvep/custom_test1_annotated.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/ensemblvep/test0_annotated.summary.html - contains: - - "" - - "" - - path: output/ensemblvep/test0_annotated.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/ensemblvep/test1_annotated.summary.html - contains: - - "" - - "" - - path: output/ensemblvep/test1_annotated.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/tabix/custom_test.vcf.gz.tbi - - path: output/tabix/test.vcf.gz.tbi -- name: vcf_annotate_ensemblvep_snpeff vcf_annotate_ensemblvep_snpeff_snpeff - command: nextflow run ./tests/subworkflows/nf-core/vcf_annotate_ensemblvep_snpeff -entry vcf_annotate_ensemblvep_snpeff_snpeff -c ./tests/config/nextflow.config - tags: - - bcftools - - bcftools/concat - - bcftools/pluginscatter - - bcftools/sort - - ensemblvep - - ensemblvep/vep - - snpeff - - snpeff/snpeff - - subworkflows - - subworkflows/vcf_annotate_ensemblvep_snpeff - - tabix - - tabix/tabix - files: - - path: output/bcftools/custom_test.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test0.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test1.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/custom_test_concat.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test0.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test1.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/bcftools/test_concat.vcf.gz - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/snpeff/custom_test0_annotated.ann.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/snpeff/custom_test0_annotated.csv - contains: - - "# Summary table" - - path: output/snpeff/custom_test0_annotated.genes.txt - md5sum: 130536bf0237d7f3f746d32aaa32840a - - path: output/snpeff/custom_test1_annotated.ann.vcf - contains: - - "##fileformat=VCFv4.2" - - '##FILTER=' - - path: output/snpeff/custom_test1_annotated.csv - contains: - - "# Summary table" - - path: output/snpeff/custom_test1_annotated.genes.txt - md5sum: 130536bf0237d7f3f746d32aaa32840a - - path: output/snpeff/snpEff_summary.html - contains: - - '