From 876a489e709edd874b51561332cad556edc472db Mon Sep 17 00:00:00 2001 From: cimendes Date: Sat, 8 Jun 2019 11:31:23 +0100 Subject: [PATCH 1/2] fix bug where reference files where being passed onto mafft via dengue_typing multiple times. --- changelog.md | 4 + flowcraft/flowcraft.py | 4 - flowcraft/generator/engine.py | 39 ++------ flowcraft/nextflow.config | 176 ---------------------------------- 4 files changed, 14 insertions(+), 209 deletions(-) delete mode 100644 flowcraft/nextflow.config diff --git a/changelog.md b/changelog.md index 2f7a1267..ac16d66f 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,10 @@ - `Concoct`: Clustering metagenomic assembled comtigs with coverage and composition - `MetaBAT2`: A robust statistical framework for reconstructing genomes from metagenomic data +### Minor/Other changes + +- added manifest information to the `nextflow.config` file to allow for remote execution + ## 1.4.1 ### New features diff --git a/flowcraft/flowcraft.py b/flowcraft/flowcraft.py index 48d30f63..69a5d5ff 100755 --- a/flowcraft/flowcraft.py +++ b/flowcraft/flowcraft.py @@ -279,10 +279,6 @@ def copy_project(path): # Copy bin scripts copy_tree(join(repo_dir, "bin"), join(target_dir, "bin")) - # Copy default config file - shutil.copy(join(repo_dir, "nextflow.config"), - join(target_dir, "nextflow.config")) - # Copy static profiles file shutil.copy(join(repo_dir, "profiles.config"), join(target_dir, "profiles.config")) diff --git a/flowcraft/generator/engine.py b/flowcraft/generator/engine.py index 59c4764c..f2ff2e32 100644 --- a/flowcraft/generator/engine.py +++ b/flowcraft/generator/engine.py @@ -150,10 +150,10 @@ class of the component. See :func:`NextflowGenerator._get_params_string` """ - self.manifest = "" + self.config = "" """ - str: Stores de manifest directives string for the nextflow pipeline. - See :func:`NextflowGenerator._get_manifest_string` + str: Stores de configuration for the nextflow pipeline. + See :func:`NextflowGenerator._get_config_string` """ self.user_config = "" @@ -1234,24 +1234,6 @@ def _get_merged_params_help(self): return help_list - def _get_manifest_string(self): - """Returns the nextflow manifest config string to include in the - config file from the information on the pipeline. - - Returns - ------- - str - Nextflow manifest configuration string - """ - - config_str = "" - - config_str += '\n\tname = "{}"'.format(self.pipeline_name) - config_str += '\n\tmainScript = "{}"'.format(self.nf_file) - - return config_str - - @staticmethod def _render_config(template, context): @@ -1277,7 +1259,7 @@ def _set_configurations(self): resources = "" containers = "" params = "" - manifest = "" + config = "" if self.merge_params: params += self._get_merged_params_string() @@ -1297,8 +1279,6 @@ def _set_configurations(self): resources += self._get_resources_string(p.directives, p.pid) containers += self._get_container_string(p.directives, p.pid) - manifest = self._get_manifest_string() - self.resources = self._render_config("resources.config", { "process_info": resources }) @@ -1308,8 +1288,9 @@ def _set_configurations(self): self.params = self._render_config("params.config", { "params_info": params }) - self.manifest = self._render_config("manifest.config", { - "manifest_info": manifest + self.config = self._render_config("nextflow.config", { + "pipeline_name": self.pipeline_name, + "nf_file": self.nf_file }) self.help = self._render_config("Helper.groovy", { "nf_file": basename(self.nf_file), @@ -1426,9 +1407,9 @@ def write_configs(self, project_root): with open(join(project_root, "params.config"), "w") as fh: fh.write(self.params) - # Write manifest config - with open(join(project_root, "manifest.config"), "w") as fh: - fh.write(self.manifest) + # Write nextflow config + with open(join(project_root, "nextflow.config"), "w") as fh: + fh.write(self.config) # Write user config if not present in the project directory if not exists(join(project_root, "user.config")): diff --git a/flowcraft/nextflow.config b/flowcraft/nextflow.config deleted file mode 100644 index e85f2bbf..00000000 --- a/flowcraft/nextflow.config +++ /dev/null @@ -1,176 +0,0 @@ -params { - platformHTTP = null - reportHTTP = null - - // Settings this option to true, will trigger the removal of temporary - // data (usually fastq reads) at particular checkpoint processes that - // modify that data. These checkpoint processes include 'trimmomatic', - // 'spades' and 'skesa'. - // WARNING: This will remove temporary fastq files that are not necessary - // for the completion of the pipeline but, consequently, will disable - // the resume functionality of the pipeline. However, it is often necessary - // for very large pipelines and whenever disk space is critical. - // More precisely, these checkpoint components will check whether the - // putative temporary files are inside the nextflow work directory by - // matching the regex: ".*/work/.{2}/.{30}/.*" - // If it is a match, then the file is assumed to be a temporary one and - // will be removed. - clearAtCheckpoint = false -} - -env { - PYTHONPATH = "$baseDir/templates:\$PYTHONPATH" - PATH = "$baseDir/templates:\$PATH" -} - -process { - cpus = 1 - memory = "1GB" - - errorStrategy = { task.attempt <= 7 ? "retry" : "ignore" } - maxRetries = 7 - container = "flowcraft/flowcraft_base:1.0.0-1" -} - -docker { - // Added default docker option to avoid docker permission errors. See issue - // #142 - runOptions = "-u \$(id -u):\$(id -g)" -} - - -executor { - $local { - cpus = 4 - } -} - -singularity { - cacheDir = "$HOME/.singularity_cache" - autoMounts = true -} - -trace { - enabled = true - file = "pipeline_stats.txt" - fields = "task_id,\ - hash,\ - process,\ - tag,\ - status,\ - exit,\ - start,\ - container,\ - cpus,\ - time,\ - disk,\ - memory,\ - duration,\ - realtime,\ - queue,\ - %cpu,\ - %mem,\ - rss,\ - vmem,\ - rchar,\ - wchar" -} - -// PROFILE OPTIONS // -/////////////////////////////////////////////////////////////////////////////// - -profiles { - - oneida { - - process.executor = "slurm" - docker.enabled = true - - process{ - - // MEMORY USAGE PER PROCESS // - // general memory usage - memory = "4GB" - - } - - } - - // INCD PROFILE - incd { - - process.executor = "slurm" - singularity.enabled = true - - singularity { - cacheDir = "/mnt/singularity_cache" - autoMounts = true - } - - // Error and retry strategies - process.errorStrategy = "retry" - maxRetries = 3 - - process.$chewbbaca.queue = "chewBBACA" - - process { - - // MEMORY USAGE PER PROCESS // - // general memory usage - memory = "4GB" - - } - - } - - // SLURM PROFILE - slurm { - - // Change executor for SLURM - process.executor = "slurm" - // Change container engine for Shifter - shifter.enabled = true - - process { - - clusterOptions = "--qos=oneida" - - errorStrategy = "retry" - maxRetries = 5 - - // MEMORY USAGE PER PROCESS // - // general memory usage - memory = "4GB" - - } - - } - - // SLURM PROFILE - slurmOneida { - - // Change executor for SLURM - process.executor = "slurm" - // Change container engine for Shifter - shifter.enabled = true - - process { - - clusterOptions = "--qos=oneida" - - // MEMORY USAGE PER PROCESS // - // general memory usage - memory = "4GB" - - // Set QOS for chewbbaca in order to run a single job - $chewbbaca.clusterOptions = "--qos=chewbbaca" - } - } -} - -includeConfig "profiles.config" -includeConfig "resources.config" -includeConfig "containers.config" -includeConfig "params.config" -includeConfig "manifest.config" -includeConfig "user.config" From 51f20e14be25188aae280e51afdb45f78fd6c482 Mon Sep 17 00:00:00 2001 From: cimendes Date: Sat, 8 Jun 2019 11:39:42 +0100 Subject: [PATCH 2/2] add nextflow.config template remove manifest.config template adjusted tests for changes --- flowcraft/generator/templates/manifest.config | 5 - flowcraft/generator/templates/nextflow.config | 180 ++++++++++++++++++ flowcraft/tests/test_assemblerflow.py | 2 +- 3 files changed, 181 insertions(+), 6 deletions(-) delete mode 100644 flowcraft/generator/templates/manifest.config create mode 100644 flowcraft/generator/templates/nextflow.config diff --git a/flowcraft/generator/templates/manifest.config b/flowcraft/generator/templates/manifest.config deleted file mode 100644 index bbdea919..00000000 --- a/flowcraft/generator/templates/manifest.config +++ /dev/null @@ -1,5 +0,0 @@ -manifest { - -{{ manifest_info }} - -} \ No newline at end of file diff --git a/flowcraft/generator/templates/nextflow.config b/flowcraft/generator/templates/nextflow.config new file mode 100644 index 00000000..a0d0d32f --- /dev/null +++ b/flowcraft/generator/templates/nextflow.config @@ -0,0 +1,180 @@ +manifest { + name = "{{ pipeline_name }}" + mainScript = "{{ nf_file }}" +} + +params { + platformHTTP = null + reportHTTP = null + + // Settings this option to true, will trigger the removal of temporary + // data (usually fastq reads) at particular checkpoint processes that + // modify that data. These checkpoint processes include 'trimmomatic', + // 'spades' and 'skesa'. + // WARNING: This will remove temporary fastq files that are not necessary + // for the completion of the pipeline but, consequently, will disable + // the resume functionality of the pipeline. However, it is often necessary + // for very large pipelines and whenever disk space is critical. + // More precisely, these checkpoint components will check whether the + // putative temporary files are inside the nextflow work directory by + // matching the regex: ".*/work/.{2}/.{30}/.*" + // If it is a match, then the file is assumed to be a temporary one and + // will be removed. + clearAtCheckpoint = false +} + +env { + PYTHONPATH = "$baseDir/templates:\$PYTHONPATH" + PATH = "$baseDir/templates:\$PATH" +} + +process { + cpus = 1 + memory = "1GB" + + errorStrategy = { task.attempt <= 7 ? "retry" : "ignore" } + maxRetries = 7 + container = "flowcraft/flowcraft_base:1.0.0-1" +} + +docker { + // Added default docker option to avoid docker permission errors. See issue + // #142 + runOptions = "-u \$(id -u):\$(id -g)" +} + + +executor { + $local { + cpus = 4 + } +} + +singularity { + cacheDir = "$HOME/.singularity_cache" + autoMounts = true +} + +trace { + enabled = true + file = "pipeline_stats.txt" + fields = "task_id,\ + hash,\ + process,\ + tag,\ + status,\ + exit,\ + start,\ + container,\ + cpus,\ + time,\ + disk,\ + memory,\ + duration,\ + realtime,\ + queue,\ + %cpu,\ + %mem,\ + rss,\ + vmem,\ + rchar,\ + wchar" +} + +// PROFILE OPTIONS // +/////////////////////////////////////////////////////////////////////////////// + +profiles { + + oneida { + + process.executor = "slurm" + docker.enabled = true + + process{ + + // MEMORY USAGE PER PROCESS // + // general memory usage + memory = "4GB" + + } + + } + + // INCD PROFILE + incd { + + process.executor = "slurm" + singularity.enabled = true + + singularity { + cacheDir = "/mnt/singularity_cache" + autoMounts = true + } + + // Error and retry strategies + process.errorStrategy = "retry" + maxRetries = 3 + + process.$chewbbaca.queue = "chewBBACA" + + process { + + // MEMORY USAGE PER PROCESS // + // general memory usage + memory = "4GB" + + } + + } + + // SLURM PROFILE + slurm { + + // Change executor for SLURM + process.executor = "slurm" + // Change container engine for Shifter + shifter.enabled = true + + process { + + clusterOptions = "--qos=oneida" + + errorStrategy = "retry" + maxRetries = 5 + + // MEMORY USAGE PER PROCESS // + // general memory usage + memory = "4GB" + + } + + } + + // SLURM PROFILE + slurmOneida { + + // Change executor for SLURM + process.executor = "slurm" + // Change container engine for Shifter + shifter.enabled = true + + process { + + clusterOptions = "--qos=oneida" + + // MEMORY USAGE PER PROCESS // + // general memory usage + memory = "4GB" + + // Set QOS for chewbbaca in order to run a single job + $chewbbaca.clusterOptions = "--qos=chewbbaca" + } + } +} + +includeConfig "profiles.config" +includeConfig "resources.config" +includeConfig "containers.config" +includeConfig "params.config" +includeConfig "user.config" diff --git a/flowcraft/tests/test_assemblerflow.py b/flowcraft/tests/test_assemblerflow.py index 402b584b..684b6ed6 100644 --- a/flowcraft/tests/test_assemblerflow.py +++ b/flowcraft/tests/test_assemblerflow.py @@ -53,7 +53,7 @@ def test_build_file_2(tmp): assert sorted(os.listdir(tmp)) == [".forkTree.json", ".treeDag.json", "containers.config", - "lib", "manifest.config", "params.config", + "lib", "nextflow.config", "params.config", "resources.config", "teste.html", "teste.nf", "user.config"]