Tree Based Classification Applications.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Application of Tree Based Classification Methods to Wisconsin Breast Cancer Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from time import time\n",
    "from functools import reduce\n",
    "from collections import deque\n",
    "from sklearn import tree, ensemble\n",
    "from metrix import accuracy, confusion_matrix, precision, recall, f1_score\n",
    "import scipy.stats as st\n",
    "\n",
    "from DecisionTree import DecisionTreeClassifier, DecisionTreeRegressor\n",
    "from RandomForestClassifier import RandomForestClassifier\n",
    "from AdaBoostClassifier import AdaBoostClassifier\n",
    "from GradientBoostingClassifier import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test_split(X, ratio=0.7):\n",
    "    X = X.sample(frac=1).reset_index(drop=True)\n",
    "    return X[:int(len(X) * ratio)], X[int(len(X) * ratio):]\n",
    "\n",
    "# Helper functions for grid search\n",
    "def cartesian_product(arr1, arr2):\n",
    "    product = []\n",
    "    for i in arr1:\n",
    "        for j in arr2:\n",
    "            if isinstance(i, tuple):\n",
    "                # (1, 2) and 5 -> (1, 2, 5)\n",
    "                product.append((*i,j))\n",
    "            else:\n",
    "                # 1 and 2 -> (1, 2)\n",
    "                product.append((i,j))\n",
    "    return product\n",
    "\n",
    "def all_possible_param_combinations(params):\n",
    "    return reduce(cartesian_product, map(lambda param_name: params[param_name], params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grid_search(model, params_to_optimize, X_train, y_train, X_test, y_test):\n",
    "    all_possibilities = all_possible_param_combinations(params_to_optimize)\n",
    "    best_accuracy = 0\n",
    "    best_model = None\n",
    "    for index, possibility in enumerate(all_possibilities):\n",
    "        model_i = model(*possibility)\n",
    "        a=time()\n",
    "        model_i.fit(X_train, y_train)\n",
    "        b=time()\n",
    "        print('model', index + 1)\n",
    "        print('trained in', b-a, 'seconds')\n",
    "        accuracy = model_i.score(X_test, y_test)\n",
    "        print('accuracy: ', accuracy)\n",
    "        if accuracy > best_accuracy:\n",
    "            best_accuracy = accuracy\n",
    "            best_model = model_i\n",
    "\n",
    "    return best_accuracy, best_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000025</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1002945</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1015425</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1016277</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1017023</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>694</th>\n",
       "      <td>776715</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>695</th>\n",
       "      <td>841769</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>888820</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>697</th>\n",
       "      <td>897471</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>897471</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>699 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0   1   2   3   4   5   6   7   8   9   10\n",
       "0    1000025   5   1   1   1   2   1   3   1   1   2\n",
       "1    1002945   5   4   4   5   7  10   3   2   1   2\n",
       "2    1015425   3   1   1   1   2   2   3   1   1   2\n",
       "3    1016277   6   8   8   1   3   4   3   7   1   2\n",
       "4    1017023   4   1   1   3   2   1   3   1   1   2\n",
       "..       ...  ..  ..  ..  ..  ..  ..  ..  ..  ..  ..\n",
       "694   776715   3   1   1   1   3   2   1   1   1   2\n",
       "695   841769   2   1   1   1   2   1   1   1   1   2\n",
       "696   888820   5  10  10   3   7   3   8  10   2   4\n",
       "697   897471   4   8   6   4   3   4  10   6   1   4\n",
       "698   897471   4   8   8   5   4   5  10   4   1   4\n",
       "\n",
       "[699 rows x 11 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('breast-cancer-wisconsin.data', header=None)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.8/site-packages/pandas/core/ops/array_ops.py:253: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  res_values = method(rvalues)\n"
     ]
    }
   ],
   "source": [
    "df = df.loc[(df != '?').all(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data = train_test_split(df.iloc[:, 1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train_data.iloc[:, :-1]\n",
    "y_train = train_data.iloc[:, -1]\n",
    "\n",
    "X_test = test_data.iloc[:, :-1]\n",
    "y_test = test_data.iloc[:, -1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Selection with Cross-Validation and Grid Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have different hyperparameters to consider: min_members is the minimum required examples to label a node as leaf. max_depth is the maximum depth of a feature. max_features is the number of features to consider when making a split. Those features are selected randomly. Now, let's search for the optimal parameters with grid search by evaluating model performance on validation set. First, we'll built single decision trees from our data and later we will compare their performance with some ensemble methods such as Random Forest, AdaBoost etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_to_optimize = {\n",
    "    'tol': [0.1],\n",
    "    'max_depth': [2, 3, 4],\n",
    "    'min_members': [10, 20, 50],\n",
    "    'criterion': ['gini'],\n",
    "    'split_method': ['binary'],\n",
    "    'max_features': [1, 2]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 1\n",
      "trained in 0.0024471282958984375 seconds\n",
      "accuracy:  0.8292682926829268\n",
      "model 2\n",
      "trained in 0.0025339126586914062 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 3\n",
      "trained in 0.0018148422241210938 seconds\n",
      "accuracy:  0.9219512195121952\n",
      "model 4\n",
      "trained in 0.0032830238342285156 seconds\n",
      "accuracy:  0.8926829268292683\n",
      "model 5\n",
      "trained in 0.0013821125030517578 seconds\n",
      "accuracy:  0.8292682926829268\n",
      "model 6\n",
      "trained in 0.002389669418334961 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 7\n",
      "trained in 0.0026226043701171875 seconds\n",
      "accuracy:  0.926829268292683\n",
      "model 8\n",
      "trained in 0.0029892921447753906 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 9\n",
      "trained in 0.0021331310272216797 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 10\n",
      "trained in 0.0028219223022460938 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 11\n",
      "trained in 0.0024251937866210938 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 12\n",
      "trained in 0.0026772022247314453 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 13\n",
      "trained in 0.002405881881713867 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 14\n",
      "trained in 0.003987789154052734 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 15\n",
      "trained in 0.0026166439056396484 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 16\n",
      "trained in 0.003480195999145508 seconds\n",
      "accuracy:  0.9170731707317074\n",
      "model 17\n",
      "trained in 0.003460407257080078 seconds\n",
      "accuracy:  0.9170731707317074\n",
      "model 18\n",
      "trained in 0.004156351089477539 seconds\n",
      "accuracy:  0.9170731707317074\n"
     ]
    }
   ],
   "source": [
    "best_dt_accuracy, best_dt_model = grid_search(DecisionTreeClassifier, params_to_optimize, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9414634146341463"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_dt_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 3, 1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_dt_model.min_members, best_dt_model.max_depth, best_dt_model.max_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "variables": {
     "best_dt_model.max_depth": "3",
     "best_dt_model.max_features": "1",
     "best_dt_model.min_members": "20",
     "np.round(best_dt_accuracy*100, 2).astype('int')": "94"
    }
   },
   "source": [
    "From the 18 different models we evaluated, the model with hyperparameters min_members={{best_dt_model.min_members}}, max_depth={{best_dt_model.max_depth}}, max_features={{best_dt_model.max_features}} seems to be the best one with the {{np.round(best_dt_accuracy*100, 2).astype('int')}}% accuracy on validation set. Let us further evaluate this model by looking at other metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 4, 4, 2, 2, 4, 2, 4, 2, 4, 4, 2, 2, 2, 2, 4, 4, 2, 2, 4, 2,\n",
       "       2, 2, 4, 4, 2, 4, 4, 2, 4, 2, 4, 2, 4, 4, 2, 2, 2, 2, 2, 4, 2, 4,\n",
       "       2, 4, 4, 2, 2, 2, 4, 2, 4, 2, 2, 2, 4, 4, 4, 4, 4, 2, 4, 4, 4, 2,\n",
       "       4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 2, 2, 2, 4,\n",
       "       2, 2, 2, 4, 2, 2, 4, 2, 2, 2, 4, 2, 4, 2, 4, 2, 2, 2, 2, 2, 2, 4,\n",
       "       2, 2, 2, 4, 2, 2, 4, 2, 4, 2, 2, 4, 4, 4, 2, 2, 2, 2, 2, 2, 4, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 2, 2, 2, 4, 2, 2, 4, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 4, 2, 2, 2, 2, 2, 2, 4, 2, 2, 2, 4, 2, 2, 4, 2, 2,\n",
       "       2, 2, 4, 4, 2, 4, 2, 2, 2, 4, 2, 2, 2, 4, 2, 2, 2, 2, 2, 4, 2, 2,\n",
       "       4, 4, 4, 4, 2, 2, 2])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_dt = best_dt_model.predict(X_test)\n",
    "pred_dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual Positive</th>\n",
       "      <th>Actual Negative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Predicted Positive</th>\n",
       "      <td>61</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predicted Negative</th>\n",
       "      <td>8</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Actual Positive  Actual Negative\n",
       "Predicted Positive               61                4\n",
       "Predicted Negative                8              132"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_matrix = confusion_matrix(y_test, pred_dt, classes=[2, 4])\n",
    "conf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_true_positive = conf_matrix.iloc[0, 0]\n",
    "dt_predicted_positive = conf_matrix.iloc[0, :].sum()\n",
    "dt_actual_positive = conf_matrix.iloc[:, 0].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "variables": {
     "dt_predicted_positive": "65",
     "dt_true_positive": "61"
    }
   },
   "source": [
    "We have {{dt_true_positive}} true positives out of {{dt_predicted_positive}} positively predicted examples. So, our precision rate is {{dt_true_positive}}/{{dt_predicted_positive}}:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9384615384615385"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_precision = precision(y_test, pred_dt, classes=[2, 4])\n",
    "dt_precision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "variables": {
     "dt_actual_positive": "69",
     "dt_true_positive": "61"
    }
   },
   "source": [
    "The model correctly classified {{dt_true_positive}} out of {{dt_actual_positive}} positive examples. So, the *recall* is:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8840579710144928"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_recall = recall(y_test, pred_dt, classes=[2, 4])\n",
    "dt_recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we calculate *f1 score* by $ \\frac{2pr}{p + r} $, where p is precision and r is recall:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9104477611940298"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_f1 = f1_score(y_test, pred_dt, classes=[2, 4])\n",
    "dt_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let us examine how well Random Forest will perform on our data. We have an extra hyperparameter, *n_trees*, which defines the number of trees to build."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_to_optimize = {\n",
    "    'n_trees': [100, 150, 200],\n",
    "    'tol': [0.1],\n",
    "    'max_depth': [3, 4],\n",
    "    'min_members': [10, 20],\n",
    "    'criterion': ['gini'],\n",
    "    'split_method': ['binary'],\n",
    "    'max_features': [1, 2],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 1\n",
      "trained in 0.1718606948852539 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 2\n",
      "trained in 0.2550930976867676 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 3\n",
      "trained in 0.1602160930633545 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 4\n",
      "trained in 0.253950834274292 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 5\n",
      "trained in 0.2288961410522461 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 6\n",
      "trained in 0.3246805667877197 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 7\n",
      "trained in 0.22014999389648438 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 8\n",
      "trained in 0.3346598148345947 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 9\n",
      "trained in 0.23746824264526367 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 10\n",
      "trained in 0.38608264923095703 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 11\n",
      "trained in 0.23241543769836426 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 12\n",
      "trained in 0.3595423698425293 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 13\n",
      "trained in 0.30576252937316895 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 14\n",
      "trained in 0.47082948684692383 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 15\n",
      "trained in 0.32668352127075195 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 16\n",
      "trained in 0.47914862632751465 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 17\n",
      "trained in 0.3142733573913574 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 18\n",
      "trained in 0.5217273235321045 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 19\n",
      "trained in 0.31964707374572754 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 20\n",
      "trained in 0.4905869960784912 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 21\n",
      "trained in 0.40297937393188477 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 22\n",
      "trained in 0.6124308109283447 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 23\n",
      "trained in 0.4217853546142578 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 24\n",
      "trained in 0.6403026580810547 seconds\n",
      "accuracy:  0.9658536585365853\n"
     ]
    }
   ],
   "source": [
    "best_rf_accuracy, best_rf_model = grid_search(RandomForestClassifier, params_to_optimize, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9658536585365853"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_rf_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 3, 1, 10)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_rf_model.n_trees, best_rf_model.max_depth, best_rf_model.max_features, best_rf_model.min_members"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "variables": {
     "best_rf_model.max_depth": "3",
     "best_rf_model.max_features": "1",
     "best_rf_model.min_members": "10",
     "best_rf_model.n_trees": "100",
     "np.round(best_rf_accuracy*100, 2).astype('int')": "96"
    }
   },
   "source": [
    "From the 24 different models we evaluated, the model with hyperparameters min_members={{best_rf_model.min_members}}, max_depth={{best_rf_model.max_depth}}, max_features={{best_rf_model.max_features}}, n_trees={{best_rf_model.n_trees}} performs best one with the {{np.round(best_rf_accuracy*100, 2).astype('int')}}% accuracy on validation set. Now, again, compute other metrics to see examine the performance of this model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_rf = best_rf_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual Positive</th>\n",
       "      <th>Actual Negative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Predicted Positive</th>\n",
       "      <td>67</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predicted Negative</th>\n",
       "      <td>2</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Actual Positive  Actual Negative\n",
       "Predicted Positive               67                5\n",
       "Predicted Negative                2              131"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, pred_rf, classes=[2, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9305555555555556"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_precision = precision(y_test, pred_rf, classes=[2, 4])\n",
    "rf_precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9710144927536232"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_recall = recall(y_test, pred_rf, classes=[2, 4])\n",
    "rf_recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9503546099290779"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_f1 = f1_score(y_test, pred_rf, classes=[2, 4])\n",
    "rf_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's see how AdaBoost performs on our dataset. We have now, instead of *n_trees*, *n_learners* parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_to_optimize = {\n",
    "    'n_learners': [50, 100, 150, 200],\n",
    "    'tol': [0.1],\n",
    "    'max_depth': [2],\n",
    "    'min_members': [10, 20],\n",
    "    'criterion': ['gini'],\n",
    "    'split_method': ['binary'],\n",
    "    'max_features': [1, 2, 3, 4, 5],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 1\n",
      "trained in 0.058773040771484375 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 2\n",
      "trained in 0.1498410701751709 seconds\n",
      "accuracy:  0.9512195121951219\n",
      "model 3\n",
      "trained in 0.21143436431884766 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 4\n",
      "trained in 0.22395777702331543 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 5\n",
      "trained in 0.25395774841308594 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 6\n",
      "trained in 0.11724472045898438 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 7\n",
      "trained in 0.15119481086730957 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 8\n",
      "trained in 0.18380069732666016 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 9\n",
      "trained in 0.21530532836914062 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 10\n",
      "trained in 0.25317883491516113 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 11\n",
      "trained in 0.1747736930847168 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 12\n",
      "trained in 0.24207186698913574 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 13\n",
      "trained in 0.30335450172424316 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 14\n",
      "trained in 0.3775827884674072 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 15\n",
      "trained in 0.44930076599121094 seconds\n",
      "accuracy:  0.9512195121951219\n",
      "model 16\n",
      "trained in 0.17884016036987305 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 17\n",
      "trained in 0.24578142166137695 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 18\n",
      "trained in 0.3134775161743164 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 19\n",
      "trained in 0.3865208625793457 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 20\n",
      "trained in 0.44711947441101074 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 21\n",
      "trained in 0.23481488227844238 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 22\n",
      "trained in 0.33014440536499023 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 23\n",
      "trained in 0.44212961196899414 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 24\n",
      "trained in 0.5255229473114014 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 25\n",
      "trained in 0.6402876377105713 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 26\n",
      "trained in 0.23775744438171387 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 27\n",
      "trained in 0.3418118953704834 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 28\n",
      "trained in 0.439838171005249 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 29\n",
      "trained in 0.5473592281341553 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 30\n",
      "trained in 0.6343135833740234 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 31\n",
      "trained in 0.2884542942047119 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 32\n",
      "trained in 0.4282689094543457 seconds\n",
      "accuracy:  0.9365853658536586\n",
      "model 33\n",
      "trained in 0.5595977306365967 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 34\n",
      "trained in 0.7026462554931641 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 35\n",
      "trained in 0.8460783958435059 seconds\n",
      "accuracy:  0.9463414634146341\n",
      "model 36\n",
      "trained in 0.30286383628845215 seconds\n",
      "accuracy:  0.9317073170731708\n",
      "model 37\n",
      "trained in 0.4371819496154785 seconds\n",
      "accuracy:  0.9512195121951219\n",
      "model 38\n",
      "trained in 0.5583088397979736 seconds\n",
      "accuracy:  0.9512195121951219\n",
      "model 39\n",
      "trained in 0.6810638904571533 seconds\n",
      "accuracy:  0.9414634146341463\n",
      "model 40\n",
      "trained in 0.8421585559844971 seconds\n",
      "accuracy:  0.9609756097560975\n"
     ]
    }
   ],
   "source": [
    "best_ab_accuracy, best_ab_model = grid_search(AdaBoostClassifier, params_to_optimize, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9609756097560975"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_ab_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50, 10, 4)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_ab_model.n_learners, best_ab_model.min_members, best_ab_model.max_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "variables": {
     "best_ab_model.max_features": "4",
     "best_ab_model.min_members": "10",
     "best_ab_model.n_learners": "50",
     "np.round(best_ab_accuracy*100, 2).astype('int')": "96"
    }
   },
   "source": [
    "From the 64 different models we evaluated, the model with hyperparameters min_members={{best_ab_model.min_members}}, max_features={{best_ab_model.max_features}}, n_learners={{best_ab_model.n_learners}} performs best one with the {{np.round(best_ab_accuracy*100, 2).astype('int')}}% accuracy on validation set. Let's compute some metrics again."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_ab = best_ab_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual Positive</th>\n",
       "      <th>Actual Negative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Predicted Positive</th>\n",
       "      <td>65</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predicted Negative</th>\n",
       "      <td>4</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Actual Positive  Actual Negative\n",
       "Predicted Positive               65                4\n",
       "Predicted Negative                4              132"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, pred_ab, classes=[2, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9420289855072463"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_precision = precision(y_test, pred_ab, classes=[2, 4])\n",
    "ab_precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9420289855072463"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_recall = recall(y_test, pred_ab, classes=[2, 4])\n",
    "ab_recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9420289855072463"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_f1 = f1_score(y_test, pred_ab, classes=[2, 4])\n",
    "ab_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's finally look at the performance of Gradient Boosting Classifier. We have two additional features to AdaBoost, alpha and n_iters_stop. alpha is the learning rate. It control the effect of each regression tree that predicts the residuals. n_iters_stop defines the stopping criterion of how many number of subsequent iterations that loss is not changed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_to_optimize = {\n",
    "    'n_learners': [50, 100, 150, 200],\n",
    "    'n_iters_stop': [5],\n",
    "    'loss_tol': [10e-4],\n",
    "    'alpha': [0.1, 0.3, 0.5, 0.8],\n",
    "    'tol': [0.1],\n",
    "    'max_depth': [2],\n",
    "    'min_members': [10, 20],\n",
    "    'split_method': ['binary'],\n",
    "    'max_features': [1, 2, 3],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.8/site-packages/numpy/core/fromnumeric.py:3334: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/usr/lib/python3.8/site-packages/numpy/core/_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model 1\n",
      "trained in 0.24473261833190918 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 2\n",
      "trained in 0.31832313537597656 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 3\n",
      "trained in 0.845130205154419 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 4\n",
      "trained in 0.22322964668273926 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 5\n",
      "trained in 0.30461597442626953 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 6\n",
      "trained in 0.8693137168884277 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 7\n",
      "trained in 0.23079752922058105 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 8\n",
      "trained in 0.33502650260925293 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 9\n",
      "trained in 0.9166715145111084 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 10\n",
      "trained in 0.23220419883728027 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 11\n",
      "trained in 0.33174872398376465 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 12\n",
      "trained in 0.9008445739746094 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 13\n",
      "trained in 0.23565101623535156 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 14\n",
      "trained in 0.3366818428039551 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 15\n",
      "trained in 0.9103045463562012 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 16\n",
      "trained in 0.23215484619140625 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 17\n",
      "trained in 0.34714341163635254 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 18\n",
      "trained in 0.9315814971923828 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 19\n",
      "trained in 0.2341458797454834 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 20\n",
      "trained in 0.32457756996154785 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 21\n",
      "trained in 0.9269871711730957 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 22\n",
      "trained in 0.24631905555725098 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 23\n",
      "trained in 0.3561258316040039 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 24\n",
      "trained in 0.989443302154541 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 25\n",
      "trained in 0.4753992557525635 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 26\n",
      "trained in 0.6919786930084229 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 27\n",
      "trained in 1.9017407894134521 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 28\n",
      "trained in 0.4802207946777344 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 29\n",
      "trained in 0.6873631477355957 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 30\n",
      "trained in 1.856909990310669 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 31\n",
      "trained in 0.48920178413391113 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 32\n",
      "trained in 0.7006514072418213 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 33\n",
      "trained in 1.9031429290771484 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 34\n",
      "trained in 0.46500420570373535 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 35\n",
      "trained in 0.6867454051971436 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 36\n",
      "trained in 1.940242052078247 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 37\n",
      "trained in 0.4908792972564697 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 38\n",
      "trained in 0.6810667514801025 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 39\n",
      "trained in 1.9006764888763428 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 40\n",
      "trained in 0.48503589630126953 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 41\n",
      "trained in 0.7033615112304688 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 42\n",
      "trained in 1.8991570472717285 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 43\n",
      "trained in 0.486422061920166 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 44\n",
      "trained in 0.7019569873809814 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 45\n",
      "trained in 1.9763846397399902 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 46\n",
      "trained in 0.4859731197357178 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 47\n",
      "trained in 0.7081928253173828 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 48\n",
      "trained in 1.9976963996887207 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 49\n",
      "trained in 0.720381498336792 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 50\n",
      "trained in 1.0446248054504395 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 51\n",
      "trained in 2.923851251602173 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 52\n",
      "trained in 0.7200024127960205 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 53\n",
      "trained in 1.0599210262298584 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 54\n",
      "trained in 2.934415102005005 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 55\n",
      "trained in 0.7485589981079102 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 56\n",
      "trained in 1.0708074569702148 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 57\n",
      "trained in 2.975191354751587 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 58\n",
      "trained in 0.7677426338195801 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 59\n",
      "trained in 1.087808609008789 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 60\n",
      "trained in 2.97451114654541 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 61\n",
      "trained in 0.7635514736175537 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 62\n",
      "trained in 1.080498456954956 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 63\n",
      "trained in 3.030555486679077 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 64\n",
      "trained in 0.735034704208374 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 65\n",
      "trained in 1.0778608322143555 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 66\n",
      "trained in 2.954861640930176 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 67\n",
      "trained in 0.7575960159301758 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 68\n",
      "trained in 1.0926711559295654 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 69\n",
      "trained in 3.0269742012023926 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 70\n",
      "trained in 0.7660672664642334 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 71\n",
      "trained in 1.0955636501312256 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 72\n",
      "trained in 3.016977071762085 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 73\n",
      "trained in 1.003570795059204 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 74\n",
      "trained in 1.4607422351837158 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 75\n",
      "trained in 4.137808084487915 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 76\n",
      "trained in 1.0052332878112793 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 77\n",
      "trained in 1.4336190223693848 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 78\n",
      "trained in 4.153461933135986 seconds\n",
      "accuracy:  0.9560975609756097\n",
      "model 79\n",
      "trained in 0.9934377670288086 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 80\n",
      "trained in 1.474701166152954 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 81\n",
      "trained in 4.039931774139404 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 82\n",
      "trained in 1.0298469066619873 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 83\n",
      "trained in 1.4303207397460938 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 84\n",
      "trained in 4.079066038131714 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 85\n",
      "trained in 1.0461862087249756 seconds\n",
      "accuracy:  0.9658536585365853\n",
      "model 86\n",
      "trained in 1.5073940753936768 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 87\n",
      "trained in 4.070384740829468 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 88\n",
      "trained in 1.0007615089416504 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 89\n",
      "trained in 1.4609637260437012 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 90\n",
      "trained in 4.016440391540527 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 91\n",
      "trained in 1.0331368446350098 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 92\n",
      "trained in 1.4576985836029053 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 93\n",
      "trained in 4.098828077316284 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 94\n",
      "trained in 1.0502409934997559 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 95\n",
      "trained in 1.4811947345733643 seconds\n",
      "accuracy:  0.9609756097560975\n",
      "model 96\n",
      "trained in 4.106555223464966 seconds\n",
      "accuracy:  0.9609756097560975\n"
     ]
    }
   ],
   "source": [
    "best_gb_accuracy, best_gb_model = grid_search(GradientBoostingClassifier, params_to_optimize, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9658536585365853"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_gb_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50, 10, 2, 0.5)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_gb_model.n_learners, best_gb_model.min_members, best_gb_model.max_features, best_gb_model.alpha"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "variables": {
     "best_gb_model.alpha": "0.5",
     "best_gb_model.max_features": "2",
     "best_gb_model.min_members": "10",
     "np.round(best_gb_accuracy*100, 2).astype('int')": "96"
    }
   },
   "source": [
    "Amongst 96 different models we evaluated, the model with hyperparameters n_learners={{best_gb_model.min_members}}, max_features={{best_gb_model.max_features}}, alpha={{best_gb_model.alpha}}, min_members={{best_gb_model.min_members}} performs best one with the {{np.round(best_gb_accuracy*100, 2).astype('int')}}% accuracy on validation set. Again, we calculate metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_gb = best_gb_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual Positive</th>\n",
       "      <th>Actual Negative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Predicted Positive</th>\n",
       "      <td>67</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predicted Negative</th>\n",
       "      <td>2</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Actual Positive  Actual Negative\n",
       "Predicted Positive               67                4\n",
       "Predicted Negative                2              132"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, pred_gb, classes=[2,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9436619718309859"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_precision = precision(y_test, pred_gb, classes=[2, 4])\n",
    "gb_precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9710144927536232"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_recall = recall(y_test, pred_gb, classes=[2, 4])\n",
    "gb_recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9571428571428571"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_f1 = f1_score(y_test, pred_gb, classes=[2, 4])\n",
    "gb_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's take a final look at the models we selected and compare their performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_eval_matrix = np.array([\n",
    "    [best_dt_accuracy, best_rf_accuracy, best_ab_accuracy, best_gb_accuracy],\n",
    "    [dt_precision, rf_precision, ab_precision, gb_precision],\n",
    "    [dt_recall, rf_recall, ab_recall, gb_recall],\n",
    "    [dt_f1, rf_f1, ab_f1, gb_f1]\n",
    "])\n",
    "model_eval_df = pd.DataFrame(model_eval_matrix, columns=['Decision Tree', 'Random Forest', 'AdaBoost', 'Gradient Boosting'], index=['Accuracy', 'Precision', 'Recall', 'F1 Score'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Decision Tree</th>\n",
       "      <th>Random Forest</th>\n",
       "      <th>AdaBoost</th>\n",
       "      <th>Gradient Boosting</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.941463</td>\n",
       "      <td>0.965854</td>\n",
       "      <td>0.960976</td>\n",
       "      <td>0.970732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.938462</td>\n",
       "      <td>0.930556</td>\n",
       "      <td>0.942029</td>\n",
       "      <td>0.943662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.884058</td>\n",
       "      <td>0.971014</td>\n",
       "      <td>0.942029</td>\n",
       "      <td>0.971014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 Score</th>\n",
       "      <td>0.910448</td>\n",
       "      <td>0.950355</td>\n",
       "      <td>0.942029</td>\n",
       "      <td>0.957143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Decision Tree  Random Forest  AdaBoost  Gradient Boosting\n",
       "Accuracy        0.941463       0.965854  0.960976           0.970732\n",
       "Precision       0.938462       0.930556  0.942029           0.943662\n",
       "Recall          0.884058       0.971014  0.942029           0.971014\n",
       "F1 Score        0.910448       0.950355  0.942029           0.957143"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.1 64-bit",
   "language": "python",
   "name": "python38164bitb67002bdd6e54363b3a4785271f5d5b5"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}