Merge pull request #167 from DennisSchmitz/dev

Jovian release v1.2.0
DennisSchmitz · Dec 11, 2020 · b348249 · b348249
2 parents cf25ee1 + bb49d0a
commit b348249
Show file tree

Hide file tree

Showing 40 changed files with 1,097 additions and 274 deletions.
diff --git a/.env b/.env
@@ -13,6 +13,9 @@ PATH_JOVIAN_MASTER_YAML="bin/envs/Jovian_master_environment.yaml"
 MASTER_NAME=$(head -n 1 ${PATH_JOVIAN_MASTER_YAML} | cut -f2 -d ' ') # Extract Conda environment name as specified in yaml file
 PATH_JOVIAN_HELPER_YAML="bin/envs/Jovian_helper_environment.yaml"
 HELPER_NAME=$(head -n 1 ${PATH_JOVIAN_HELPER_YAML} | cut -f2 -d ' ') # Extract Conda environment name as specified in yaml file
+PATH_JOVIAN_STATS_yaml="bin/envs/Jovian_coverage_statistics.yaml"
+STATS_NAME=$(head -n 1 ${PATH_JOVIAN_STATS_yaml} | cut -f2 -d ' ') # Extract Conda environment name as specified in yaml file
+
 
 ### Default values for CLI parameters
 INPUT_DIR="raw_data/"

diff --git a/Notebook_report.ipynb b/Notebook_report.ipynb
@@ -67,7 +67,8 @@
    "metadata": {
     "deleteable": false,
     "init_cell": true,
-    "scrolled": false
+    "scrolled": false,
+    "tags": []
    },
    "outputs": [],
    "source": [
@@ -259,13 +260,31 @@
     "'''\n",
     "\n",
     "\n",
-    "scf_viewer_cell  = '''\n",
+    "scf_viewer_cell_ILM_META  = '''\n",
     "___\n",
     "## Scaffold viewer:\n",
     "**Containing: SNPs and minority variants (quasispecies), predicted ORFs, depth of coverage graph, GC contents graph**\n",
     "___\n",
     "N.B. Depending on the depth of coverage of the selected contig it can be <b>(very) slow, or downright crash your browser</b>. This is a <b>client-sided</b> problem, meaning, your computer isn't powerful enough.  \n",
-    "Open the scaffold viewer as a dedicated page by clicking [here](results/igv.html).  \n",
+    "Open the scaffold viewer as a dedicated page by clicking [here](results/igv_meta.html).  \n",
+    "'''\n",
+    "\n",
+    "scf_viewer_cell_ILM_REF  = '''\n",
+    "___\n",
+    "## Alignment viewer:\n",
+    "**Containing: SNPs and minority variants (quasispecies), predicted ORFs, depth of coverage graph, GC contents graph**\n",
+    "___\n",
+    "N.B. Depending on the depth of coverage of the selected contig it can be <b>(very) slow, or downright crash your browser</b>. This is a <b>client-sided</b> problem, meaning, your computer isn't powerful enough.  \n",
+    "Open the scaffold viewer as a dedicated page by clicking [here](results/igv_ilr.html).  \n",
+    "'''\n",
+    "\n",
+    "scf_viewer_cell_NANO_REF  = '''\n",
+    "___\n",
+    "## Alignment viewer:\n",
+    "**Containing: SNPs, depth of coverage graph, GC contents graph**\n",
+    "___\n",
+    "N.B. Depending on the depth of coverage of the selected contig it can be <b>(very) slow, or downright crash your browser</b>. This is a <b>client-sided</b> problem, meaning, your computer isn't powerful enough.  \n",
+    "Open the scaffold viewer as a dedicated page by clicking [here](results/igv_nano.html).  \n",
     "'''\n",
     "\n",
     "snp_table_cell = '''\n",
@@ -360,10 +379,19 @@
     "            dp(md(goback))\n",
     "\n",
     "def scaffoldviewer():\n",
-    "    if os.path.exists(\"results/igv.html\"):\n",
-    "        dp(md(scf_viewer_cell))\n",
-    "        dp(fr('results/igv.html', '100%', '980px'))\n",
+    "    if os.path.exists(\"results/igv_meta.html\"):\n",
+    "        dp(md(scf_viewer_cell_ILM_META))\n",
+    "        dp(fr('results/igv_meta.html', '100%', '980px'))\n",
     "        dp(md(goback))\n",
+    "    if os.path.exists(\"results/igv_ilr.html\"):\n",
+    "        dp(md(scf_viewer_cell_ILM_REF))\n",
+    "        dp(fr('results/igv_ilr.html', '100%', '980px'))\n",
+    "        dp(md(goback))\n",
+    "    if os.path.exists(\"results/igv_nano.html\"):\n",
+    "        dp(md(scf_viewer_cell_NANO_REF))\n",
+    "        dp(fr('results/igv_nano.html', '100%', '980px'))\n",
+    "        dp(md(goback))\n",
+    "        \n",
     "\n",
     "\n",
     "def audit():\n",
@@ -719,7 +747,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.7"
+   "version": "3.7.8-final"
   },
   "toc": {
    "base_numbering": 1,
@@ -778,4 +806,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}
diff --git a/bin/Illumina_Meta.smk b/bin/Illumina_Meta.smk
@@ -151,7 +151,7 @@ rule all:
                                 'krona'
                                 ]
                 ), # HTML Reports
-        expand( "{p}igv.html",
+        expand( "{p}igv_meta.html",
                 p   =   f"{res}"
                 ) # IGVjs index
 

diff --git a/bin/Illumina_vir_Ref.smk b/bin/Illumina_vir_Ref.smk
@@ -191,7 +191,7 @@ rule all:
         f"{res}SNPs.tsv",
         f"{res}BoC_int.tsv", # Integer BoC overview in .tsv format
         f"{res}BoC_pct.tsv", # Percentage BoC overview in .tsv format
-        f"{res}igv.html", # IGVjs output html
+        f"{res}igv_ilr.html", # IGVjs output html
         f"{res}multiqc.html" # MultiQC report
 
 

diff --git a/bin/Nano_vir_Ref.smk b/bin/Nano_vir_Ref.smk
@@ -39,7 +39,16 @@ reference_basename  =   os.path.splitext(os.path.basename(reference))[0]
 
 rule all:
     input: 
-        expand( "{p}{sample}_cov_ge_{thresholds}.fa",
+        expand( "{p}{sample}_standard_cov_ge_{thresholds}.fa",
+                p               =   f"{res + seqs}",
+                sample          =   SAMPLES,
+                thresholds      =   [   '1',
+                                        '5', 
+                                        '10',
+                                        '30',
+                                        '100'
+                                        ]),
+        expand( "{p}{sample}_gap_corrected_cov_ge_{thresholds}.fa",
                 p               =   f"{res + seqs}",
                 sample          =   SAMPLES,
                 thresholds      =   [   '1',
@@ -56,9 +65,25 @@ rule all:
                 ext     =   [   '.zip',
                                 '.html'
                                 ]),
+        expand( "{p}concat_standard_cov_ge_{thr}.fasta",
+                p   =   f"{res + seqs}",
+                thr =   [   '1',
+                            '5',
+                            '10',
+                            '30',
+                            '100'
+                            ]),
+        expand( "{p}concat_gap_corrected_cov_ge_{thr}.fasta",
+                p   =   f"{res + seqs}",
+                thr =   [   '1',
+                            '5',
+                            '10',
+                            '30',
+                            '100'
+                            ]),
         f"{res}BoC_int.tsv",
         f"{res}BoC_pct.tsv",
-        f"{res}igv.html",
+        f"{res}igv_nano.html",
         f"{res}multiqc.html",
         f"{res}SNPs.tsv"
 
@@ -69,20 +94,17 @@ rule all:
 include: f"{rls}Nano_Ref_index.smk"
 include: f"{rls}Nano_Ref_ORF-analysis.smk"
 include: f"{rls}Nano_Ref_GC-content.smk"
+include: f"{rls}Nano_Ref_get_primers.smk"
 
 include: f"{rls}Nano_Ref_pre_qc.smk"
 
-include: f"{rls}Nano_Ref_adp_trim.smk"
-
-include: f"{rls}Nano_Ref_Cut-primers.smk"
+include: f"{rls}Nano_Ref_RemoveAdapters.smk"
 
 include: f"{rls}Nano_Ref_Cleanup.smk"
 
-include: f"{rls}Nano_Ref_post-qc.smk"
-
-include: f"{rls}Nano_Ref_HuGo_removal_pt1.smk"
+include: f"{rls}Nano_Ref_Cut-primers.smk"
 
-include: f"{rls}Nano_Ref_HuGo_removal_pt2.smk"
+include: f"{rls}Nano_Ref_post-qc.smk"
 
     #>############################################################################
     #>#### Alignments and concensus
@@ -94,6 +116,8 @@ include: f"{rls}Nano_Ref_alignment_pt2.smk"
 
 include: f"{rls}Nano_Ref_consensus.smk"
 
+include: f"{rls}Nano_Ref_concat_seqs.smk"
+
 include: f"{rls}Nano_Ref_coverage.smk"
 
     #>############################################################################

diff --git a/bin/envs/Jovian_coverage_statistics.yaml b/bin/envs/Jovian_coverage_statistics.yaml
@@ -0,0 +1,10 @@
+name: Jovian_coverage_statistics
+channels:
+  - bioconda
+  - conda-forge
+  - defaults
+dependencies:
+  - python==3.8.5
+  - samtools==1.10
+  - pandas
+  - pip
diff --git a/bin/envs/MultiQC_report.yaml b/bin/envs/MultiQC_report.yaml
@@ -4,4 +4,4 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-    - multiqc==1.8
+    - multiqc==1.9
diff --git a/bin/envs/Nano_clean.yaml b/bin/envs/Nano_clean.yaml
@@ -0,0 +1,21 @@
+name: Nano_clean_and_QC
+channels:
+  - bioconda
+  - conda-forge
+  - intel
+  - anaconda
+  - defaults
+dependencies:
+  - python=3.7
+  - minimap2==2.17
+  - samtools==1.11
+  - bedtools==2.29.2
+  - pysam==0.16.0.1
+  - pysamstats==1.1.2
+  - pandas==1.1.3
+  - mkl-service
+  - biopython==1.78
+  - openjdk==11.0.8
+  - fastqc==0.11.8
+  - fastp==0.20.1
+  - cutadapt==2.10
diff --git a/bin/envs/Nano_ref_alignment.yaml b/bin/envs/Nano_ref_alignment.yaml
@@ -12,4 +12,5 @@ dependencies:
     - bedtools==2.29.2
     - tabix==0.2.6
     - seqtk==1.2.0
+    - seqkit==0.14.0
     - pysam==0.16.0.1
diff --git a/bin/envs/Nano_ref_consensus.yaml b/bin/envs/Nano_ref_consensus.yaml
@@ -0,0 +1,15 @@
+name: Nano_Consensus
+channels:
+  - bioconda
+  - conda-forge
+  - intel
+  - anaconda
+  - defaults
+dependencies:
+  - python=3.7
+  - pysam==0.16.0.1
+  - pysamstats==1.1.2
+  - pandas==1.1.3
+  - pip
+  - pip:
+    - gffpandas==1.2.0
diff --git a/bin/globals.py b/bin/globals.py
@@ -11,6 +11,7 @@
 qc_post = "FastQC_posttrim/"
 cln = "cleaned_fastq/"
 hugo_no_rm = "fastq_without_HuGo_removal/"
+nohugo = "fastq_without_HuGo/"
 scf_raw = "scaffolds_raw/"
 scf_filt = "scaffolds_filtered/"
 taxclas = "taxonomic_classification/"
@@ -26,10 +27,12 @@
 it2 = "it2/"
 raw = "raw/"
 refdir = "reference/"
+prim = "primers/"
 aln = "alignment/"
 bf = "bam-files/"
 vf = "vcf-files/"
 cons = "consensus/"
+covs = "coverages/"
 boc = "BoC/"
 chunks = "html_chunks/"
 seqs = "sequences/"

diff --git a/bin/includes/Rebuild_archive b/bin/includes/Rebuild_archive
@@ -23,11 +23,20 @@ newhostname="http://${HOSTNAME}"
 
 
 ###### Modifying the HTML index file for IGVjs
-if [ -e results/igv.html ]; then
+if [ -e results/igv_meta.html ]; then
     echo -e "Modifying IGVjs index file..."
-    sed -i -e "s@${oldhostname}@${newhostname}@g" "results/igv.html"
+    sed -i -e "s@${oldhostname}@${newhostname}@g" "results/igv_meta.html"
 fi
 
+if [ -e results/igv_nano.html ]; then
+    echo -e "Modifying IGVjs index file..."
+    sed -i -e "s@${oldhostname}@${newhostname}@g" "results/igv_nano.html"
+fi
+
+if [ -e results/igv_ilr.html ]; then
+    echo -e "Modifying IGVjs index file..."
+    sed -i -e "s@${oldhostname}@${newhostname}@g" "results/igv_ilr.html"
+fi
 
 if [ -e config/variables.yaml ]; then
     echo -e "Resetting some of the variables..."

diff --git a/bin/includes/amplicon_cov b/bin/includes/amplicon_cov
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+
+primerfile=$1
+
+#start_spinner "Looking for average depth of coverage for each amplicon"
+#sleep 0.2
+
+python bin/scripts/amplicon_coverage.py "${primerfile}"
+
+#stop_spinner $?
diff --git a/bin/includes/amplicon_cov_wrapper b/bin/includes/amplicon_cov_wrapper
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+conda deactivate
+
+if [[ $PATH != *${STATS_NAME}* ]]; then # If helper env is not in your path (i.e. it is not currently active), do...
+    set +ue # Turn bash strict mode off because that breaks conda
+
+    if ! conda activate "${STATS_NAME}"; then # If exit statement is not 0, i.e. helper conda env hasn't been installed yet, do...
+        echo -e "\tInstalling Jovian coverage-statistics environment..."
+        conda env create -f ${PATH_JOVIAN_STATS_yaml} # Create the env from the specified yaml file
+        set -o allexport
+		conda activate "${STATS_NAME}"
+		set +o allexport
+	fi
+
+    set -ue # Turn bash strict mode on again
+fi
+
+bash bin/includes/amplicon_cov $1
+
+conda deactivate
diff --git a/bin/rules/IGVjs.smk b/bin/rules/IGVjs.smk
@@ -46,7 +46,7 @@ rule HTML_IGVJs_generate_final:
                 sample      =   SAMPLES
                 )
     output:
-        f"{res}igv.html"
+        f"{res}igv_meta.html"
     conda:
         f"{conda_envs}data_wrangling.yaml"
     log:

diff --git a/bin/rules/ILM_Ref_igv_combi.smk b/bin/rules/ILM_Ref_igv_combi.smk
@@ -9,7 +9,7 @@ rule Illumina_HTML_IGVJs_generate_final:
                 sample      = SAMPLES
                 )
     output:
-        f"{res}igv.html"
+        f"{res}igv_ilr.html"
     conda:
         f"{conda_envs}data_wrangling.yaml"
     log:

diff --git a/bin/rules/Nano_Ref_Cleanup.smk b/bin/rules/Nano_Ref_Cleanup.smk
@@ -1,16 +1,12 @@
-
-
-
-
 rule Cleanup:
     input:
-        fastq   =   rules.Cut_primers.output.cleaneddata_pt1
+        fastq   =   rules.Remove_Adapters_pt2.output
     output:
         qc_fastq    =   f"{datadir + cln + datadir}" + "{sample}.fastq",
         qc_html     =   f"{datadir + cln + html}" + "{sample}.fastp.html",
         qc_json     =   f"{datadir + cln + json}" + "{sample}.fastp.json"
     conda:
-        f"{conda_envs}QC_and_clean.yaml"
+        f"{conda_envs}Nano_clean.yaml"
     log:
         f"{logdir}" + "Data_Cleanup_{sample}.log"
     benchmark: