03.0 - Hyperparameter Optimization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from scipy.stats import randint as sp_randint\n",
    "from scipy.stats import uniform\n",
    "from scipy.stats import uniform as sp_randFloat\n",
    "from sklearn import svm\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from time import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from tqdm import tqdm\n",
    "from tabulate import tabulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import randint as sp_randInt\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, PredefinedSplit\n",
    "from sklearn.metrics import make_scorer\n",
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "features= ['pck_size', 'Ether_type', 'LLC_ctrl', 'EAPOL_version', 'EAPOL_type', 'IP_ihl', 'IP_tos', 'IP_len', 'IP_flags', 'IP_DF', 'IP_ttl', 'IP_options', 'ICMP_code', 'TCP_dataofs', 'TCP_FIN', 'TCP_ACK', 'TCP_window', 'UDP_len', 'DHCP_options', 'BOOTP_hlen', 'BOOTP_flags', 'BOOTP_sname', 'BOOTP_file', 'BOOTP_options', 'DNS_qr', 'DNS_rd', 'DNS_qdcount', 'dport_class', 'payload_bytes', 'entropy', 'Label']\n",
    "\n",
    "\n",
    "df=pd.read_csv('./HPOTrain.csv',usecols=features) \n",
    "X_train = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_train=df['Label'].cat.codes  \n",
    "\n",
    "df=pd.read_csv( './HPOTest.csv',usecols=features) \n",
    "X_test = df.iloc[:,0:-1]\n",
    "df['Label'] = df['Label'].astype('category')\n",
    "y_test=df['Label'].cat.codes  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1505000, 30) (1493336, 30)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X= np.concatenate([X_train, X_test])\n",
    "test_fold = [-1 for _ in range(X_train.shape[0])] + [0 for _ in range(X_test.shape[0])]\n",
    "y = np.concatenate([y_train, y_test])\n",
    "ps = PredefinedSplit(test_fold)\n",
    "\n",
    "def run_random_search(model, params, x_train, y_train):\n",
    "    #grid = GridSearchCV(model, params, cv = ps, n_jobs = -1, scoring = score, verbose = 0, refit = False)\n",
    "    grid =RandomizedSearchCV(model, param_grid, cv=ps,scoring = 'f1_macro')\n",
    "    grid.fit(x_train, y_train)\n",
    "    return (grid.best_params_, round(grid.best_score_,8),grid.best_estimator_)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HYPERPARAMETERS                     F1 Score             Time     No      \n",
      "default                             0.9202238204785516   54.633   9       \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 10/10 [09:28<00:00, 56.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    criterion      max_depth    max_features    min_samples_split        F1    Std    Time    No\n",
      "--  -----------  -----------  --------------  -------------------  --------  -----  ------  ----\n",
      " 0  entropy               19               4                    2  0.920224      0  50.383     0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "lines=[['criterion', 'max_depth', 'max_features', 'min_samples_split', \"F1\",\"Std\",\"Time\",\"No\"]]\n",
    "print ('%-35s %-20s %-8s %-8s' % (\"HYPERPARAMETERS\",\"F1 Score\", \"Time\", \"No\"))\n",
    "\n",
    "nfolds=10\n",
    "param_grid = { 'criterion':['gini','entropy'],\n",
    "                  \"max_depth\":np.linspace(1, 32, 32, endpoint=True).astype(int),\n",
    "                 \"min_samples_split\": sp_randint(2,10),#uniform(0.1,1 ),\n",
    "                    # \"min_samples_leafs\" : np.linspace(0.1, 0.5, 5, endpoint=True)\n",
    "                    \"max_features\" : sp_randint(1,X_train.shape[1])}\n",
    "\n",
    "second=time()\n",
    "f1=[]\n",
    "clf=DecisionTreeClassifier()\n",
    "for ii in range(10):\n",
    "    clf.fit(X, y)\n",
    "    predict =clf.predict(X_test)\n",
    "    f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "f1=sum(f1)/len(f1)   \n",
    "#if f1>0.76:\n",
    "print('%-35s %-20s %-8s %-8s' % (\"default\",f1,round(time()-second,3),ii))\n",
    "######################################################################################################################\n",
    "for i in tqdm(range(10)):\n",
    "    second=time()\n",
    "    a,b,clf=run_random_search(DecisionTreeClassifier(),param_grid,X,y)\n",
    "    f1=[]\n",
    "    for ii in range(5):\n",
    "        clf.fit(X, y)\n",
    "        predict =clf.predict(X_test)\n",
    "        f1.append(sklearn.metrics.f1_score(y_test, predict,average= \"macro\") )\n",
    "    f1_result=sum(f1)/len(f1)   \n",
    "    f1=np.array(f1)\n",
    "    stndtd=f1.std()\n",
    "    temp=list(a.values())\n",
    "    #print('%-90s %-20s %-8s %-8s' % (a,f1_result,round(time()-second,3),i))\n",
    "    temp=temp+[f1_result,stndtd,round(time()-second,3),i]\n",
    "    lines.append(temp)\n",
    "\n",
    "    #if f1>0.76:\n",
    "results = pd.DataFrame (lines[1:], columns = lines[0])\n",
    "results.to_csv(\"DT_HPO.csv\",index=False)\n",
    "\n",
    "final_parametres=[['criterion', 'max_depth', 'max_features', 'min_samples_split', \"F1\",\"Std\",\"Time\",\"No\"]]\n",
    "\n",
    "df=results\n",
    "m=df[\"F1\"].max()\n",
    "df=df[df[\"F1\"]==m]\n",
    "m=df[\"max_depth\"].min()\n",
    "df=df[df[\"max_depth\"]==m]  \n",
    "final_parametres.append(list(df.values)[0])\n",
    "results = pd.DataFrame (final_parametres[1:], columns=  final_parametres[0])\n",
    "print (tabulate(results, headers=list(results.columns)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}