Merge pull request #173 from DennisSchmitz/dev

Request memory resources

bin/includes/Userconfig_CM

@@ -57,7 +57,7 @@ if [ "$cm_installeranswers_compmode" == "U" ]; then
                 echo -e "installeranswers:\n    compmode: G\n   queue: ${queueanswer}" > "${HOME}"/.jovian_installchoice_compmode
                 sed -i '\|drmaa|d' config/config.yaml
                 sed -i "s/.*local-cores:.*/${gridmode_localcores}/g" config/config.yaml
-                echo -e "drmaa: \" -q ${queueanswer} -n {threads} -R \\\"span[hosts=1]\\\"\"" >> config/config.yaml
+                echo -e "drmaa: \" -q ${queueanswer} -n {threads} -R 'span[hosts=1]' -R 'rusage[mem={resources.memory}]'\"" >> config/config.yaml
                 echo -e "drmaa-log-dir: logs/drmaa" >> config/config.yaml
                 break
             done
@@ -101,6 +101,6 @@ elif [ "${cm_installeranswers_compmode}" == "G" ]; then
     echo -e "Jovian is set to run in computing mode: Grid.\n"
     sed -i '\|drmaa|d' config/config.yaml
     sed -i "s/.*local-cores:.*/${gridmode_localcores}/g" config/config.yaml
-    echo -e "drmaa: \" -q ${cm_installeranswers_queue} -n {threads} -R \\\"span[hosts=1]\\\"\"" >> config/config.yaml
+    echo -e "drmaa: \" -q ${cm_installeranswers_queue} -n {threads} -R 'span[hosts=1]' -R 'rusage[mem={resources.memory}]'\"" >> config/config.yaml
     echo -e "drmaa-log-dir: logs/drmaa" >> config/config.yaml
 fi

bin/rules/BG_removal_1.smk

@@ -21,6 +21,8 @@ rule HuGo_removal_pt1_alignment:
     benchmark:
         f"{logdir + bench}" + "HuGo_removal_pt1_alignment_{sample}.txt"
     threads: config["threads"]["HuGo_removal"]
+    resources: 
+        memory = (config["threads"]["HuGo_removal"] * 12) * 1024
     params:
         alignment_type  =   config["Global"]["HuGo_removal_method"]
     shell:

bin/rules/BG_removal_2.smk

@@ -6,18 +6,20 @@
 
 rule HuGo_removal_pt2_extract_paired_unmapped_reads:
     input:
-         bam        =   rules.HuGo_removal_pt1_alignment.output.sorted_bam,
-         bam_index  =   rules.HuGo_removal_pt1_alignment.output.sorted_bam_index
+        bam        =   rules.HuGo_removal_pt1_alignment.output.sorted_bam,
+        bam_index  =   rules.HuGo_removal_pt1_alignment.output.sorted_bam_index
     output:
-         fastq_R1   =   f"{datadir + cln}" + "{sample}_pR1.fq",
-         fastq_R2   =   f"{datadir + cln}" + "{sample}_pR2.fq"
+        fastq_R1   =   f"{datadir + cln}" + "{sample}_pR1.fq",
+        fastq_R2   =   f"{datadir + cln}" + "{sample}_pR2.fq"
     conda:
         f"{conda_envs}HuGo_removal.yaml"
     log:
         f"{logdir}" + "HuGo_removal_pt2_extract_paired_unmapped_reads_{sample}.log"
     benchmark:
         f"{logdir + bench}" + "HuGo_removal_pt2_extract_paired_unmapped_reads_{sample}.txt"
     threads: config["threads"]["HuGo_removal"]
+    resources: 
+        memory = (config["threads"]["HuGo_removal"] * 12) * 1024
     shell:
         """
 samtools view -@ {threads} -b -f 1 -f 4 -f 8 {input.bam} 2> {log} |\

bin/rules/BG_removal_3.smk

@@ -17,6 +17,8 @@ rule HuGo_removal_pt3_extract_unpaired_unmapped_reads:
     benchmark:
         f"{logdir + bench}" + "HuGo_removal_pt3_extract_unpaired_unmapped_reads_{sample}.txt"
     threads: config["threads"]["HuGo_removal"]
+    resources: 
+        memory = (config["threads"]["HuGo_removal"] * 12) * 1024
     shell:
         """
 samtools view -@ {threads} -b -F 1 -f 4 {input.bam} 2> {log} |\

bin/rules/Blast.smk

@@ -16,6 +16,8 @@ rule Scaffold_classification:
     benchmark:
         f"{logdir + bench}" + "logs/benchmark/Scaffold_classification_{sample}.txt"
     threads: config["threads"]["Classification_of_scaffolds"]
+    resources: 
+        memory = (config["threads"]["Classification_of_scaffolds"] * 12) * 1024
     params:
         outfmt          =   "6 std qseqid sseqid staxids sscinames stitle",
         evalue          =   config["Illumina_meta"]["Classification"]["e_value"],

bin/rules/CleanData.smk

@@ -21,6 +21,8 @@ rule Clean_the_data:
     benchmark:
         f"{logdir + bench}" + "Clean_the_data_{sample}.txt"
     threads: config["threads"]["Clean_the_data"]
+    resources:
+        memory = (config["threads"]["Clean_the_data"] * 4) * 1024
     params:
         adapter_removal_config  =   config["Illumina"]["Clean"]["adapter_removal_config"],
         quality_trimming_config =   config["Illumina"]["Clean"]["quality_trimming_config"],

bin/rules/Concat_SNPs.smk

@@ -18,6 +18,8 @@ rule Concat_filtered_SNPs:
     benchmark:
         f"{logdir + bench}Concat_filtered_SNPs.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     params:
         vcf_folder_glob =   f"{datadir + scf_filt}/\*_filtered.vcf"
     shell:

bin/rules/Concat_files.smk

@@ -22,6 +22,8 @@ rule Concat_files:
     benchmark:
         f"{logdir + bench}Concat_files.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     params:
         search_folder       =   f"{datadir + tbl}",
         classified_glob     =   "*_taxClassified.tsv",

bin/rules/Concat_reads.smk

@@ -18,6 +18,8 @@ rule concatenate_read_counts:
     benchmark:
         f"{logdir + bench}concatenate_read_counts.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     shell:
         """
         bin/scripts/concatenate_mapped_read_counts.py \

bin/rules/Contig_metrics.smk

@@ -20,6 +20,8 @@ rule Generate_contigs_metrics:
     benchmark:
         f"{logdir + bench}" + "Generate_contigs_metrics_{sample}.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     shell: #! bbtools' pileup.sh counts every read, even those marked as duplicate upstream. Hence, upstream all duplicates are HARD removed.
         """
 pileup.sh in={input.bam} \

bin/rules/Count_reads.smk

@@ -16,6 +16,8 @@ rule count_mapped_reads:
     benchmark:
         f"{logdir + bench}" + "count_mapped_reads-{sample}.txt"
     threads: 1
+    resources:
+        memory = 12 * 1024
     shell:
         """
 bash bin/scripts/count_mapped_reads.sh {input} > {output} 2> {log}

bin/rules/GC_content.smk

@@ -19,6 +19,8 @@ rule Determine_GC_content:
     benchmark:
         f"{logdir + bench}" + "Determine_GC_content_{sample}.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params:
         window_size =   config["Global"]["GC_window_size"]
     shell:

bin/rules/Heatmaps.smk

@@ -25,6 +25,8 @@ rule draw_heatmaps:
     benchmark:
         f"{logdir + bench}draw_heatmaps.txt"
     threads: 1
+    resources:
+        memory = 12 * 1024
     shell:
         """
 python bin/scripts/draw_heatmaps.py -c {input.classified} -n {input.numbers} -sq {output.super_quantities} -st {output.stats} -vs {output.vir_stats} -ps {output.phage_stats} -bs {output.bact_stats} -s {output.super} -v {output.virus} -p {output.phage} -b {output.bact} > {log} 2>&1

bin/rules/IGVjs.smk

@@ -23,6 +23,8 @@ rule HTML_IGVJs_variable_parts:
     benchmark:
         f"{logdir + bench}" + "HTML_IGVJs_variable_parts_{sample}.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     shell:
         """
 bash bin/html/igvjs_write_tabs.sh {wildcards.sample} {output.tab_output}
@@ -54,6 +56,8 @@ rule HTML_IGVJs_generate_final:
     benchmark:
         f"{logdir + bench}benchmark/HTML_IGVJs_generate_final.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     params:
         tab_basename    =   f"{datadir + html}2_tab_",
         div_basename    =   f"{datadir + html}4_html_divs_",

bin/rules/ILM_Ref_GC_content.smk

@@ -13,6 +13,8 @@ rule Illumina_determine_GC_content:
     benchmark:
         f"{logdir + bench}" + "Illumina_determine_GC_content_{sample}.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params:
         window_size =   config["Global"]["GC_window_size"]
     shell:

bin/rules/ILM_Ref_ORF_analysis.smk

@@ -14,6 +14,8 @@ rule Illumina_reference_ORF_analysis:
     benchmark:
         f"{logdir + bench}" + "Illumina_reference_ORF_analysis_{sample}.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params: #? Currently it's using the same prodigal settings as the main workflow, I see no problems with it since it's both foremost intended for viruses.
         procedure       =   config["Global"]["ORF_procedure"],
         output_format   =   config["Global"]["ORF_output_format"]

bin/rules/ILM_Ref_QC_clean.smk

@@ -10,6 +10,8 @@ rule ILM_Ref_QC_clean_data:
     benchmark:
         f"{logdir + bench}" + "QC_clean_data_{sample}.txt"
     threads: 6
+    resources:
+        memory = 12 * 1024
     shell:
         """
 if [ -s "{input}" ] # If file exists and is NOT empty (i.e. filesize > 0) do...

bin/rules/ILM_Ref_RemovePrimers.smk

@@ -13,6 +13,8 @@ rule RemovePrimers_pt1:
     benchmark:
         f"{logdir + bench}" + "Illumina_RemovePrimers_pt1_{sample}.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     shell:
         """
 cat {input} > {output}
@@ -31,6 +33,8 @@ rule RemovePrimers_pt2:
     benchmark:
         f"{logdir + bench}" + "Illumina_RemovePrimers_pt2_{sample}.txt"
     threads: config["threads"]["Illumina_RemovePrimers"]
+    resources: 
+        memory = (config["threads"]["Illumina_RemovePrimers"] * 12) * 1024
     params:
         primer_status = prstatus
     shell:

bin/rules/ILM_Ref_align_to_ref_it1.smk

@@ -16,6 +16,8 @@ rule Illumina_align_to_reference_it1:
     benchmark:
         f"{logdir + bench}" + "Illumina_align_to_reference_it1_{sample}.txt"
     threads: config["threads"]["Illumina_align_to_reference"]
+    resources: 
+        memory = (config["threads"]["Illumina_align_to_reference"] * 12) * 1024
     params:
         aln_type        =   config["Illumina_ref"]["Alignment"]["Alignment_type"],
         remove_dups     =   config["Illumina_ref"]["Alignment"]["Duplicates"], #! Don't change this, see this gotcha with duplicate marked reads in bedtools genomecov (which is used downstream): https://groups.google.com/forum/#!msg/bedtools-discuss/wJNC2-icIb4/wflT6PnEHQAJ . bedtools genomecov is not able to filter them out and includes those dup-reads in it's coverage metrics. So the downstream BoC analysis and consensus at diff cov processes require dups to be HARD removed.

bin/rules/ILM_Ref_align_to_ref_it2.smk

@@ -17,6 +17,8 @@ rule Illumina_align_to_reference_it2:
     benchmark:
         f"{logdir + bench}" + "Illumina_align_to_reference_it2_{sample}.txt"
     threads: config["threads"]["Illumina_align_to_reference"]
+    resources: 
+        memory = (config["threads"]["Illumina_align_to_reference"] * 12) * 1024
     params:
         aln_type        =   config["Illumina_ref"]["Alignment"]["Alignment_type"],
         remove_dups     =   config["Illumina_ref"]["Alignment"]["Duplicates"], #! Don't change this, see this gotcha with duplicate marked reads in bedtools genomecov (which is used downstream): https://groups.google.com/forum/#!msg/bedtools-discuss/wJNC2-icIb4/wflT6PnEHQAJ . bedtools genomecov is not able to filter them out and includes those dup-reads in it's coverage metrics. So the downstream BoC analysis and consensus at diff cov processes require dups to be HARD removed.

bin/rules/ILM_Ref_boc_covs.smk

@@ -13,6 +13,8 @@ rule Illumina_determine_BoC_at_diff_cov_thresholds:
     benchmark:
         f"{logdir + bench}" + "Illumina_determine_BoC_at_diff_cov_thresholds_{sample}.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     shell:
         """
 bash bin/scripts/BoC_analysis.sh {wildcards.sample} {input.bedgraph} {input.reference} \

bin/rules/ILM_Ref_boc_mets.smk

@@ -18,6 +18,8 @@ rule Illumina_concat_BoC_metrics:
     benchmark:
         f"{logdir + bench}Illumina_concat_BoC_metrics.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     shell:
         """
 echo -e "Sample_name\tTotal_ref_size\tBoC_at_coverage_threshold_1\tBoC_at_coverage_threshold_5\tBoC_at_coverage_threshold_10\tBoC_at_coverage_threshold_30\tBoC_at_coverage_threshold_100" > {output.combined_BoC_int_tsv}

bin/rules/ILM_Ref_concat-snips.smk

@@ -12,6 +12,8 @@ rule SNP_table:
     benchmark:
         f"{logdir + bench}Concatenated_SNPs.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     shell:
         """
 echo -e "# The events below are incorporated into the consensus genome, depending on the INDELS versus the user-supplied reference these coordinates are not comparable to the generated consensus genome in the \"results/sequences/\" folder\nSample\tReference AccessionID\tPosition\tType\tReference\tAlternative\tQuality" > {output}

bin/rules/ILM_Ref_extract_clean_cons.smk

@@ -27,6 +27,8 @@ rule Illumina_extract_clean_consensus:
     benchmark:
         f"{logdir + bench}" + "Illumina_extract_clean_consensus_{sample}.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params:
         output_data_folder      =   f"{datadir + it2 + cons}",
         output_results_folder   =   f"{res + seqs}"

bin/rules/ILM_Ref_extract_raw_cons_it1.smk

@@ -17,6 +17,8 @@ rule Illumina_extract_raw_consensus_it1:
     benchmark:
         f"{logdir + bench}" + "Illumina_extract_raw_consensus_it1_{sample}.txt"
     threads: config["threads"]["Illumina_extract_raw_consensus"]
+    resources: 
+        memory = (config["threads"]["Illumina_extract_raw_consensus"] * 12) * 1024
     params:
         max_cov             = config["Illumina_ref"]["SNP"]["Max_coverage"],
     shell: # First codeblock, SNP and indel calling, filter majority SNPs from LoFreq output, zip/normalize/index it, call consensus based on these SNPs.

bin/rules/ILM_Ref_extract_raw_cons_it2.smk

@@ -20,6 +20,8 @@ rule Illumina_extract_raw_consensus_it2:
     benchmark:
         f"{logdir + bench}" + "Illumina_extract_raw_consensus_it2_{sample}.txt"
     threads: config["threads"]["Illumina_extract_raw_consensus"]
+    resources: 
+        memory = (config["threads"]["Illumina_extract_raw_consensus"] * 12) * 1024
     params:
         max_cov             = config["Illumina_ref"]["SNP"]["Max_coverage"],
         min_AF              = config["Illumina_ref"]["SNP"]["Minority_SNP_AF_lower_threshold"]

bin/rules/ILM_Ref_igv_combi.smk

@@ -17,6 +17,8 @@ rule Illumina_HTML_IGVJs_generate_final:
     benchmark:
         f"{logdir + bench}Illumina_HTML_IGVJs_generate_final.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     params:
         chunkpath       =   f"{fls + chunks}", 
         tab_basename    =   f"{datadir + html}2_tab_",

bin/rules/ILM_Ref_igv_vars.smk

@@ -17,6 +17,8 @@ rule Illumina_HTML_IGVJs_variable_parts:
     benchmark:
         f"{logdir + bench}" + "Illumina_HTML_IGVJs_variable_parts_{sample}.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     shell:
         """
 bash bin/html/igvjs_write_tabs.sh {wildcards.sample} {output.tab_output}

bin/rules/ILM_Ref_index.smk

@@ -13,6 +13,8 @@ rule Illumina_index_reference:
     benchmark:
         f"{logdir + bench}Illumina_index_reference.txt"
     threads: 4
+    resources:
+        memory = 12 * 1024
     shell: # The reference is copied to the hardcoded subdir to make it standardized and easily logged. Convert it to a two-line fasta for easier downstream processing, replace ambiguity nucleotides with an N to avoid errors downstream.
         """
 cat {input.reference} | seqtk seq - | sed '/^[^>]/s/[R|Y|W|S|M|K|H|B|V|D]/N/g' > {output.reference_copy} 2>> {log}

bin/rules/ILM_Ref_multiqc.smk

@@ -43,6 +43,8 @@ rule Illumina_MultiQC_report:
     benchmark:
         f"{logdir + bench}Illumina_MultiQC_report.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params:
         config_file =   f"{fls}multiqc_config.yaml",
         output_dir  =   f"{res}",

bin/rules/KronaChart.smk

@@ -20,6 +20,8 @@ rule Krona_chart_combine:
     benchmark:
         f"{logdir + bench}Krona_chart_combine.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params:
         krona_tax_db    =   config["databases"]["Krona_taxonomy"]
     shell:

bin/rules/Krona_LCA.smk

@@ -12,6 +12,8 @@ rule Krona_chart_and_LCA:
     benchmark:
         f"{logdir + bench}" + "Krona_chart_and_LCA_{sample}.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params:
         bitscoreDelta   =   config["Illumina_meta"]["LCA"]["bitscoreDelta"],
         krona_tax_db    =   config["databases"]["Krona_taxonomy"]

bin/rules/Merge_metrics.smk

@@ -24,6 +24,8 @@ rule Merge_all_metrics_into_single_tsv:
     benchmark:
         f"{logdir + bench}" + "Merge_all_metrics_into_single_tsv_{sample}.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     shell:
         """
 python bin/scripts/merge_data.py {wildcards.sample} \

bin/rules/MultiQC.smk

@@ -41,6 +41,8 @@ rule MultiQC_report:
     benchmark:
         f"{logdir + bench}MultiQC_report.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params:
         config_file =   f"{fls}multiqc_config.yaml"
     shell:

bin/rules/Nano_Ref_Cleanup.smk

@@ -12,6 +12,8 @@ rule Cleanup:
     benchmark:
         f"{logdir + bench}" + "Data_Cleanup_{sample}.txt"
     threads: config["threads"]["Nanopore_cleanup"]
+    resources: 
+        memory = (config["threads"]["Nanopore_cleanup"] * 10) * 1024
     params:
         QualityFilter   =   config["Nanopore_ref"]["Quality_score"]
     shell:

bin/rules/Nano_Ref_Cut-primers.smk

@@ -11,6 +11,8 @@ rule Cut_primers:
     benchmark:
         f"{logdir + bench}" + "Primer_removal_{sample}.txt"
     threads: config["threads"]["Nanopore_primer_removal"]
+    resources: 
+        memory = (config["threads"]["Nanopore_primer_removal"] * 12) * 1024
     shell:
         """
 python bin/scripts/RemoveONTPrimers.py --input {input.fastq} --reference {input.ref} --primers {input.primers} --threads {threads} --output {output}

bin/rules/Nano_Ref_GC-content.smk

@@ -15,6 +15,8 @@ rule determine_GC_content:
     benchmark:
         f"{logdir + bench}Determine_GC_content.txt"
     threads: 1
+    resources:
+        memory = 8 * 1024
     params:
         window_size =   config["Global"]["GC_window_size"]
     shell:

bin/rules/Nano_Ref_IGVjs.smk

@@ -19,6 +19,8 @@ rule HTML_IGVJs_variable_parts:
     benchmark:
         f"{logdir + bench}" + "HTML_IGVJs_variable_parts_{sample}.txt"
     threads: 1
+    resources:
+        memory = 4
     shell:
         """
 bash bin/html/igvjs_write_tabs.sh {wildcards.sample} {output.tab}
@@ -49,6 +51,8 @@ rule HTML_IGVJs_generate_file:
     benchmark:
         f"{logdir + bench}HTML_IGVJs_generate_file.txt"
     threads: 1
+    resources:
+        memory = 4 * 1024
     params:
         tab_basename    =   f"{datadir + chunks}2_tab_",
         div_basename    =   f"{datadir + chunks}4_html_divs_",