Correct checks and documentation for evaluation metrics (remove HK an…

…d HSS)
biomodhub · Jun 13, 2024 · 4164cd7 · 4164cd7
1 parent 8eaa324
commit 4164cd7
Show file tree

Hide file tree

Showing 25 changed files with 170 additions and 176 deletions.
diff --git a/R/BIOMOD_EnsembleForecasting.R b/R/BIOMOD_EnsembleForecasting.R
@@ -38,16 +38,16 @@
 ##' A \code{vector} containing evaluation metric names to be used to transform prediction values 
 ##' into binary values based on models evaluation scores obtained with the 
 ##' \code{\link{BIOMOD_Modeling}} function. Must be among \code{all} (same evaluation metrics than 
-##' those of \code{modeling.output}) or \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, 
-##' \code{BIAS}, \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, 
-##' \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' those of \code{modeling.output}) or \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, 
+##' \code{ACCURACY}, \code{BIAS}, \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, 
+##' \code{CSI}, \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' @param metric.filter (\emph{optional, default} \code{NULL}) \cr 
 ##' A \code{vector} containing evaluation metric names to be used to transform prediction values 
 ##' into filtered values based on models evaluation scores obtained with the 
 ##' \code{\link{BIOMOD_Modeling}} function. Must be among \code{all} (same evaluation metrics than 
-##' those of \code{modeling.output}) or \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, 
-##' \code{BIAS}, \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, 
-##' \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' those of \code{modeling.output}) or \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, 
+##' \code{ACCURACY}, \code{BIAS}, \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, 
+##' \code{CSI}, \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' 
 ##' @param compress (\emph{optional, default} \code{TRUE}) \cr 
 ##' A \code{logical} or a \code{character} value defining whether and how objects should be 

diff --git a/R/BIOMOD_EnsembleModeling.R b/R/BIOMOD_EnsembleModeling.R
@@ -26,27 +26,25 @@
 ##' \code{metric.select.thresh} to exclude single models based on their evaluation scores 
 ##' (for ensemble methods like probability weighted mean or committee averaging). Must be among  
 ##' \code{all} (same evaluation metrics than those of \code{bm.mod}), \code{user.defined} 
-##' (and defined through \code{metric.select.table}) or \code{ROC}, \code{TSS}, \code{KAPPA}, 
-##' \code{ACCURACY}, \code{BIAS}, \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, 
-##' \code{ETS}, \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' (and defined through \code{metric.select.table}) or \code{POD}, \code{FAR}, \code{POFD}, 
+##' \code{SR}, \code{ACCURACY}, \code{BIAS}, \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, 
+##' \code{ORSS}, \code{CSI}, \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' @param metric.select.thresh (\emph{optional, default} \code{NULL}) \cr 
 ##' A \code{vector} of \code{numeric} values corresponding to the minimum scores (one for each 
 ##' \code{metric.select}) below which single models will be excluded from the ensemble model 
 ##' building
-##' @param metric.select.table (\emph{optional, default} \code{NULL}) \cr If
-##'   \code{metric.select = 'user.defined'}, a \code{data.frame} containing
-##'   evaluation scores calculated for each single models and that will be
-##'   compared to \code{metric.select.thresh} values to exclude some of them
-##'   from the ensemble model building, with \code{metric.select} rownames, and
-##'   \code{models.chosen} colnames
-##' @param metric.select.dataset (\emph{optional, default} \code{'validation'}
-##'   \emph{if possible}). A character determining which dataset should be used
-##'   to filter and/or weigh the ensemble models should be among 'evaluation',
-##'   'validation' or 'calibration'.
+##' @param metric.select.table (\emph{optional, default} \code{NULL}) \cr If 
+##' \code{metric.select = 'user.defined'}, a \code{data.frame} containing evaluation scores 
+##' calculated for each single models and that will be compared to \code{metric.select.thresh} 
+##' values to exclude some of them from the ensemble model building, with \code{metric.select} 
+##' rownames, and \code{models.chosen} colnames
+##' @param metric.select.dataset (\emph{optional, default} \code{'validation'} 
+##' \emph{if possible}). A character determining which dataset should be used to filter and/or 
+##' weigh the ensemble models should be among 'evaluation', 'validation' or 'calibration'.
 ##' @param metric.eval a \code{vector} containing evaluation metric names to be used, must 
-##' be among \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, \code{BIAS}, \code{POD}, 
-##' \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, \code{HK}, \code{HSS}, \code{OR}, 
-##' \code{ORSS}
+##' be among  \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, \code{BIAS}, 
+##' \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, \code{ETS}, 
+##' \code{BOYCE}, \code{MPA}
 ##' @param var.import (\emph{optional, default} \code{NULL}) \cr 
 ##' An \code{integer} corresponding to the number of permutations to be done for each variable to 
 ##' estimate variable importance
@@ -961,9 +959,9 @@ BIOMOD_EnsembleModeling <- function(bm.mod,
 
   ## 7. Check metric.eval -----------------------------------------------------
   metric.eval <- unique(metric.eval)
-  avail.eval.meth.list <- c('TSS', 'KAPPA', 'ACCURACY', 'BIAS', 'POD', 'FAR', 'POFD'
-                            , 'SR', 'CSI', 'ETS', 'HK', 'HSS', 'OR', 'ORSS', 'ROC'
-                            , 'BOYCE', 'MPA')
+  avail.eval.meth.list <- c('POD', 'FAR', 'POFD', 'SR', 'ACCURACY', 'BIAS'
+                            , 'ROC', 'TSS', 'KAPPA', 'OR', 'ORSS', 'CSI'
+                            , 'ETS', 'BOYCE', 'MPA')
   .fun_testIfIn(TRUE, "metric.eval", metric.eval, avail.eval.meth.list)
 
   ## 8. Check selected EM algo ------------------------------------------------

diff --git a/R/BIOMOD_LoadModels.R b/R/BIOMOD_LoadModels.R
@@ -40,9 +40,9 @@
 ##' \code{MAXENT}, \code{MAXNET}, \code{RF}, \code{SRE}, \code{XGBOOST}, \code{mergedAlgo}
 ##' @param filtered.by (\emph{optional, default} \code{NULL}) \cr 
 ##' A \code{vector} containing evaluation metric selected to filter single models to build the 
-##' ensemble models, must be among \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, 
-##' \code{BIAS}, \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, 
-##' \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' ensemble models, must be among \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, 
+##' \code{ACCURACY}, \code{BIAS}, \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, 
+##' \code{ORSS}, \code{CSI}, \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' 
 ##' 
 ##' @return 

diff --git a/R/BIOMOD_Modeling.R b/R/BIOMOD_Modeling.R
@@ -79,9 +79,9 @@
 ##' A \code{numeric} between \code{0} and \code{1} corresponding to the species prevalence to 
 ##' build '\emph{weighted response weights}' (see Details)
 ##' @param metric.eval a \code{vector} containing evaluation metric names to be used, must 
-##' be among \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, \code{BIAS}, \code{POD}, 
-##' \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, \code{HK}, \code{HSS}, \code{OR}, 
-##' \code{ORSS}, \code{BOYCE}, \code{MPA}
+##' be among \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, \code{BIAS}, 
+##' \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, \code{ETS}, 
+##' \code{BOYCE}, \code{MPA}
 ##' @param var.import (\emph{optional, default} \code{NULL}) \cr 
 ##' An \code{integer} corresponding to the number of permutations to be done for each variable to 
 ##' estimate variable importance
@@ -744,9 +744,9 @@ BIOMOD_Modeling <- function(bm.format,
 
   ## 7. Check metric.eval arguments -------------------------------------------
   metric.eval <- unique(metric.eval)
-  avail.eval.meth.list <- c('TSS', 'KAPPA', 'ACCURACY', 'BIAS', 'POD', 'FAR', 'POFD'
-                            , 'SR', 'CSI', 'ETS', 'HK', 'HSS', 'OR', 'ORSS', 'ROC'
-                            , 'BOYCE', 'MPA')
+  avail.eval.meth.list <- c('POD', 'FAR', 'POFD', 'SR', 'ACCURACY', 'BIAS'
+                            , 'ROC', 'TSS', 'KAPPA', 'OR', 'ORSS', 'CSI'
+                            , 'ETS', 'BOYCE', 'MPA')
   .fun_testIfIn(TRUE, "metric.eval", metric.eval, avail.eval.meth.list)
 
 

diff --git a/R/BIOMOD_Projection.R b/R/BIOMOD_Projection.R
@@ -33,16 +33,16 @@
 ##' A \code{vector} containing evaluation metric names to be used to transform prediction values 
 ##' into binary values based on models evaluation scores obtained with the 
 ##' \code{\link{BIOMOD_Modeling}} function. Must be among \code{all} (same evaluation metrics than 
-##' those of \code{bm.mod}) or \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, 
-##' \code{BIAS}, \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, 
-##' \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' those of \code{bm.mod}) or \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, 
+##' \code{BIAS}, \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, 
+##' \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' @param metric.filter (\emph{optional, default} \code{NULL}) \cr 
 ##' A \code{vector} containing evaluation metric names to be used to transform prediction values 
 ##' into filtered values based on models evaluation scores obtained with the 
 ##' \code{\link{BIOMOD_Modeling}} function. Must be among \code{all} (same evaluation metrics than 
-##' those of \code{bm.mod}) or \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, 
-##' \code{BIAS}, \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, 
-##' \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' those of \code{bm.mod}) or \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, 
+##' \code{BIAS}, \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, 
+##' \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' 
 ##' @param compress (\emph{optional, default} \code{TRUE}) \cr 
 ##' A \code{logical} or a \code{character} value defining whether and how objects should be 

diff --git a/R/biomod2_classes_3.R b/R/biomod2_classes_3.R
@@ -69,27 +69,27 @@
 ##' \code{MAXENT}, \code{MAXNET}, \code{RF}, \code{SRE}, \code{XGBOOST}, \code{mergedAlgo}
 ##' @param filtered.by (\emph{optional, default} \code{NULL}) \cr 
 ##' A \code{vector} containing evaluation metric selected to filter single models to build the 
-##' ensemble models, must be among \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, 
-##' \code{BIAS}, \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, 
-##' \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' ensemble models, must be among \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, 
+##' \code{ACCURACY}, \code{BIAS}, \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, 
+##' \code{CSI}, \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' 
 ##' @param metric.eval (\emph{optional, default} \code{NULL}) \cr 
-##' A \code{vector} containing evaluation metric to be kept, must be among \code{ROC}, 
-##' \code{TSS}, \code{KAPPA}, \code{ACCURACY}, \code{BIAS}, \code{POD}, \code{FAR}, 
-##' \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' A \code{vector} containing evaluation metric to be kept, must be among \code{POD}, 
+##' \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, \code{BIAS}, \code{ROC}, \code{TSS}, 
+##' \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' @param expl.var (\emph{optional, default} \code{NULL}) \cr 
 ##' A \code{vector} containing explanatory variables to be kept, that can be obtained with the 
 ##' \code{\link{get_formal_data}(obj, subinfo = 'expl.var.names')} function
 ##' 
 ##' @param metric.binary (\emph{optional, default} \code{NULL}) \cr 
 ##' A \code{vector} containing evaluation metric selected to transform predictions into binary 
-##' values, must be among \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, \code{BIAS}, 
-##' \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, \code{HK}, 
-##' \code{HSS}, \code{OR}, \code{ORSS}
+##' values, must be among \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, 
+##' \code{BIAS}, \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, 
+##' \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' @param metric.filter (\emph{optional, default} \code{NULL}) \cr 
-##' A \code{vector} containing evaluation metric to filter predictions, must be among \code{ROC}, 
-##' \code{TSS}, \code{KAPPA}, \code{ACCURACY}, \code{BIAS}, \code{POD}, \code{FAR}, 
-##' \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, \code{HK}, \code{HSS}, \code{OR}, \code{ORSS}
+##' A \code{vector} containing evaluation metric to filter predictions, must be among \code{POD}, 
+##' \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, \code{BIAS}, \code{ROC}, \code{TSS}, 
+##' \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, \code{ETS}, \code{BOYCE}, \code{MPA}
 ##' 
 ##' @param model.as.col (\emph{optional, default} \code{FALSE}) \cr
 ##' A \code{boolean} given to \code{\link{get_predictions}}. If \code{TRUE} 

diff --git a/R/bm_PlotEvalMean.R b/R/bm_PlotEvalMean.R
@@ -16,9 +16,9 @@
 ##' object that can be obtained with the \code{\link{BIOMOD_Modeling}} or 
 ##' \code{\link{BIOMOD_EnsembleModeling}} functions
 ##' @param metric.eval a \code{vector} containing evaluation metric names to be used, must 
-##' be among \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, \code{BIAS}, \code{POD}, 
-##' \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, \code{HK}, \code{HSS}, \code{OR}, 
-##' \code{ORSS}
+##' be among \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, \code{BIAS}, 
+##' \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, \code{ETS}, 
+##' \code{BOYCE}, \code{MPA}
 ##' @param dataset a \code{character} corresponding to the dataset upon which evaluation metrics 
 ##' have been calculated and that is to be represented, must be among \code{calibration}, 
 ##' \code{validation}, \code{evaluation}

diff --git a/R/bm_RunModelsLoop.R b/R/bm_RunModelsLoop.R
@@ -29,9 +29,9 @@
 ##' @param bm.options a \code{\link{BIOMOD.models.options}} object returned by the  
 ##' \code{\link{bm_ModelingOptions}} function
 ##' @param metric.eval a \code{vector} containing evaluation metric names to be used, must 
-##' be among \code{ROC}, \code{TSS}, \code{KAPPA}, \code{ACCURACY}, \code{BIAS}, \code{POD}, 
-##' \code{FAR}, \code{POFD}, \code{SR}, \code{CSI}, \code{ETS}, \code{HK}, \code{HSS}, \code{OR}, 
-##' \code{ORSS}
+##' be among \code{POD}, \code{FAR}, \code{POFD}, \code{SR}, \code{ACCURACY}, \code{BIAS}, 
+##' \code{ROC}, \code{TSS}, \code{KAPPA}, \code{OR}, \code{ORSS}, \code{CSI}, \code{ETS}, 
+##' \code{BOYCE}, \code{MPA}
 ##' @param var.import (\emph{optional, default} \code{NULL}) \cr 
 ##' An \code{integer} corresponding to the number of permutations to be done for each variable to 
 ##' estimate variable importance
@@ -655,9 +655,9 @@ bm_RunModel <- function(model, run.name, dir.name = '.'
 
   ## 4. Check models.eval.meth arguments --------------------------------------
   metric.eval <- unique(metric.eval)
-  avail.eval.meth.list <- c('TSS', 'KAPPA', 'ACCURACY', 'BIAS', 'POD', 'FAR', 'POFD'
-                            , 'SR', 'CSI', 'ETS', 'HK', 'HSS', 'OR', 'ORSS', 'ROC'
-                            , 'BOYCE', 'MPA')
+  avail.eval.meth.list <- c('POD', 'FAR', 'POFD', 'SR', 'ACCURACY', 'BIAS'
+                            , 'ROC', 'TSS', 'KAPPA', 'OR', 'ORSS', 'CSI'
+                            , 'ETS', 'BOYCE', 'MPA')
   # .fun_testIfIn(TRUE, "metric.eval", metric.eval, avail.eval.meth.list)
   if (sum(!(metric.eval %in% avail.eval.meth.list)) > 0) {
     tmp = which(metric.eval %in% avail.eval.meth.list)

diff --git a/docs/reference/BIOMOD_EnsembleForecasting.html b/docs/reference/BIOMOD_EnsembleForecasting.html