DataPreprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The original dataset files for the KDDCup99 can be found here: http://kdd.ics.uci.edu/databases/kddcup99/kddcup99.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The original dataset files for the NSL-KDD can be found here: https://www.unb.ca/cic/datasets/nsl.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**KDDCup99**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Reading in the data and adding the relevant columns '''\n",
    "just_for_columns = pd.read_pickle('kddcup99dataframe')\n",
    "\n",
    "KDD_data_10_percent = pd.read_csv('kddcup.data_10_percent.gz') #this is the one we want\n",
    "KDD_data_10_percent.columns = list(just_for_columns.columns)\n",
    "KDD_data_10_percent_x_train, KDD_data_10_percent_y_train = KDD_data_10_percent.iloc[:,:KDD_data_10_percent.shape[1]-1], KDD_data_10_percent.iloc[:,KDD_data_10_percent.shape[1]-1]\n",
    "\n",
    "corrected = pd.read_csv('corrected.gz') #this is the test data\n",
    "corrected.columns = list(just_for_columns.columns)\n",
    "corrected_x_train, corrected_y_train = corrected.iloc[:,:corrected.shape[1]-1], corrected.iloc[:,corrected.shape[1]-1]\n",
    "\n",
    "kddcup_newtestdata_10_percent_unlabeled = pd.read_csv('kddcup.newtestdata_10_percent_unlabeled.gz')\n",
    "kddcup_newtestdata_10_percent_unlabeled.columns = list(just_for_columns.columns)[0:kddcup_newtestdata_10_percent_unlabeled.shape[1]]\n",
    "kddcup_newtestdata_10_percent_unlabeled_x_train, kddcup_newtestdata_10_percent_unlabeled_y_train = kddcup_newtestdata_10_percent_unlabeled.iloc[:,:kddcup_newtestdata_10_percent_unlabeled.shape[1]-1], kddcup_newtestdata_10_percent_unlabeled.iloc[:,kddcup_newtestdata_10_percent_unlabeled.shape[1]-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = [KDD_data_10_percent, corrected]\n",
    "combined_df = pd.concat(frames)\n",
    "combined_df_all_numerical = pd.get_dummies(combined_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "KDD_data_10_percent_dummy,corrected_dummy = combined_df_all_numerical.iloc[0:KDD_data_10_percent.shape[0], :], combined_df_all_numerical.iloc[KDD_data_10_percent.shape[0]:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "KDD_data_10_percent_dummy_x_train = KDD_data_10_percent_dummy.iloc[:,0:119]\n",
    "corrected_dummy_x_test = corrected_dummy.iloc[:,0:119]\n",
    "''' This data is the 23 dimension one hot encoded vector for each intrusion type'''\n",
    "KDD_data_10_percent_dummy_y_train = KDD_data_10_percent_dummy.iloc[:,119:]\n",
    "corrected_dummy_y_test = corrected_dummy.iloc[:,119:] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    normal.\n",
       "1    normal.\n",
       "2    normal.\n",
       "3    normal.\n",
       "4    normal.\n",
       "Name: label, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KDD_data_10_percent_y_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' creation of the binary classification and 4-way classification datasets '''\n",
    "\n",
    "intrusion_name_to_type = {\n",
    "    'apache2' : 'dos',\n",
    "    'normal' : 'normal',\n",
    "    'back' : 'dos',\n",
    "    'buffer_overflow':'u2r',\n",
    "    'ftp_write' :'r2l',\n",
    "    'guess_passwd': 'r2l',\n",
    "    'httptunnel' : 'r2l',\n",
    "    'imap': 'r2l',\n",
    "    'ipsweep' :'probe',\n",
    "    'land': 'dos',\n",
    "    'loadmodule': 'u2r',\n",
    "    'mailbomb' : 'dos',\n",
    "    'mscan' : 'probe',\n",
    "    'multihop': 'r2l',\n",
    "    'named' : 'r2l',\n",
    "    'neptune': 'dos',\n",
    "    'nmap': 'probe',\n",
    "    'ps' : 'u2r',\n",
    "    'perl': 'u2r',\n",
    "    'phf' :'r2l',\n",
    "    'pod':'dos',\n",
    "    'processtable' : 'dos',\n",
    "    'portsweep': 'probe',\n",
    "    'rootkit': 'u2r',\n",
    "    'saint' : 'probe',\n",
    "    'satan' :'probe',\n",
    "    'sendmail' : 'r2l',\n",
    "    'smurf': 'dos',\n",
    "    'snmpguess' : 'r2l',\n",
    "    'snmpgetattack' : 'r2l',\n",
    "    'sqlattack' : 'u2r',\n",
    "    'spy': 'r2l',\n",
    "    'teardrop': 'dos',\n",
    "    'udpstorm' : 'dos',\n",
    "    'worm' : 'dos',\n",
    "    'warezclient': 'r2l',\n",
    "    'warezmaster': 'r2l',\n",
    "    'xlock' : 'r2l',\n",
    "    'xsnoop' : 'r2l',\n",
    "    'xterm' : 'u2r',\n",
    "    'snmpgetattack': 'r2l',\n",
    "    'named' : 'r2l',\n",
    "    \n",
    "}\n",
    "\n",
    "intrusion_type_to_one_hot_index = {\n",
    "    'normal' : 0,\n",
    "    'dos' : 1,\n",
    "    'r2l' : 2,\n",
    "    'u2r' : 3,\n",
    "    'probe' : 4\n",
    "}\n",
    "\n",
    "def create_binary_data(original_y_labels):\n",
    "    \"\"\" input should be the list of the y_labels ie. \"normal, neptune, etc.\n",
    "        returns a dataframe. \\\"\"\"\"\n",
    "    \n",
    "    y_data_binary = []\n",
    "    \n",
    "    for row in range(original_y_labels.shape[0]):\n",
    "        if \"normal\" in original_y_labels.iloc[row]:\n",
    "            y_data_binary.append(0)\n",
    "        else:\n",
    "            y_data_binary.append(1)\n",
    "    \n",
    "    y_data_binary = np.asarray(y_data_binary)\n",
    "    y_data_binary_df = pd.DataFrame(y_data_binary)\n",
    "    y_data_binary_df.columns = [\"label\"]\n",
    "    \n",
    "    return y_data_binary_df\n",
    "\n",
    "def create_attack_type_data(original_y_labels):\n",
    "    \"\"\" input should be the list of the y_labels ie. \"normal, neptune, etc.\"\n",
    "        returns a dataframe. \"\"\"\n",
    "    \n",
    "    y_data_attack_type = []\n",
    "    \n",
    "    for row in range(original_y_labels.shape[0]):\n",
    "        one_hot = [0, 0, 0, 0, 0]\n",
    "        intrusion_type = intrusion_name_to_type[original_y_labels[row][:-1]]\n",
    "        one_hot_index = intrusion_type_to_one_hot_index[intrusion_type]\n",
    "        one_hot[one_hot_index] = 1\n",
    "        y_data_attack_type.append(one_hot)\n",
    "    \n",
    "    y_data_attack_type = np.asarray(y_data_attack_type)\n",
    "    y_data_attack_type_df = pd.DataFrame(y_data_attack_type)\n",
    "    y_data_attack_type_df.columns = [\"normal\", \"dos\", \"r2l\", \"u2r\", \"probe\"]\n",
    "    \n",
    "    return y_data_attack_type_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "KDD_data_10_percent_dummy_y_train_binary = create_binary_data(KDD_data_10_percent_y_train)\n",
    "corrected_dummy_y_test_binary = create_binary_data(corrected_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "KDD_data_10_percent_dummy_y_train_attacktype = create_attack_type_data(KDD_data_10_percent_y_train)\n",
    "corrected_dummy_y_test_attacktype = create_attack_type_data(corrected_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['label_apache2.',\n",
       " 'label_back.',\n",
       " 'label_buffer_overflow.',\n",
       " 'label_ftp_write.',\n",
       " 'label_guess_passwd.',\n",
       " 'label_httptunnel.',\n",
       " 'label_imap.',\n",
       " 'label_ipsweep.',\n",
       " 'label_land.',\n",
       " 'label_loadmodule.',\n",
       " 'label_mailbomb.',\n",
       " 'label_mscan.',\n",
       " 'label_multihop.',\n",
       " 'label_named.',\n",
       " 'label_neptune.',\n",
       " 'label_nmap.',\n",
       " 'label_normal.',\n",
       " 'label_perl.',\n",
       " 'label_phf.',\n",
       " 'label_pod.',\n",
       " 'label_portsweep.',\n",
       " 'label_processtable.',\n",
       " 'label_ps.',\n",
       " 'label_rootkit.',\n",
       " 'label_saint.',\n",
       " 'label_satan.',\n",
       " 'label_sendmail.',\n",
       " 'label_smurf.',\n",
       " 'label_snmpgetattack.',\n",
       " 'label_snmpguess.',\n",
       " 'label_spy.',\n",
       " 'label_sqlattack.',\n",
       " 'label_teardrop.',\n",
       " 'label_udpstorm.',\n",
       " 'label_warezclient.',\n",
       " 'label_warezmaster.',\n",
       " 'label_worm.',\n",
       " 'label_xlock.',\n",
       " 'label_xsnoop.',\n",
       " 'label_xterm.']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(KDD_data_10_percent_dummy_y_train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' time to pickle all the data files '''\n",
    "\n",
    "#by train I mean like the entire dataset, but i think I will use this as train??\n",
    "KDD_data_10_percent_dummy_x_train.to_pickle('KDD_x_train_dummy.pkl')\n",
    "KDD_data_10_percent_dummy_y_train.to_pickle('KDD_y_train_col_40.pkl')\n",
    "KDD_data_10_percent_dummy_y_train_binary.to_pickle('KDD_y_train_col_1.pkl')\n",
    "KDD_data_10_percent_dummy_y_train_attacktype.to_pickle('KDD_y_train_col_5.pkl')\n",
    "\n",
    "corrected_dummy_x_test.to_pickle('KDD_x_test_dummy.pkl')\n",
    "corrected_dummy_y_test.to_pickle('KDD_y_test_col_40.pkl')\n",
    "corrected_dummy_y_test_binary.to_pickle('KDD_y_test_col_1.pkl')\n",
    "corrected_dummy_y_test_attacktype.to_pickle('KDD_y_test_col_5.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NSL-KDD**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>tcp</th>\n",
       "      <th>ftp_data</th>\n",
       "      <th>SF</th>\n",
       "      <th>491</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.3</th>\n",
       "      <th>0.4</th>\n",
       "      <th>0.5</th>\n",
       "      <th>...</th>\n",
       "      <th>0.17.1</th>\n",
       "      <th>0.03</th>\n",
       "      <th>0.17.2</th>\n",
       "      <th>0.00.6</th>\n",
       "      <th>0.00.7</th>\n",
       "      <th>0.00.8</th>\n",
       "      <th>0.05</th>\n",
       "      <th>0.00.9</th>\n",
       "      <th>normal</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>udp</td>\n",
       "      <td>other</td>\n",
       "      <td>SF</td>\n",
       "      <td>146</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>private</td>\n",
       "      <td>S0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>neptune</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>232</td>\n",
       "      <td>8153</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.01</td>\n",
       "      <td>normal</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>199</td>\n",
       "      <td>420</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>private</td>\n",
       "      <td>REJ</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>neptune</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  tcp ftp_data   SF  491   0.1  0.2  0.3  0.4  0.5  ...  0.17.1  0.03  \\\n",
       "0  0  udp    other   SF  146     0    0    0    0    0  ...    0.00  0.60   \n",
       "1  0  tcp  private   S0    0     0    0    0    0    0  ...    0.10  0.05   \n",
       "2  0  tcp     http   SF  232  8153    0    0    0    0  ...    1.00  0.00   \n",
       "3  0  tcp     http   SF  199   420    0    0    0    0  ...    1.00  0.00   \n",
       "4  0  tcp  private  REJ    0     0    0    0    0    0  ...    0.07  0.07   \n",
       "\n",
       "   0.17.2  0.00.6  0.00.7  0.00.8  0.05  0.00.9   normal  20  \n",
       "0    0.88    0.00    0.00    0.00   0.0    0.00   normal  15  \n",
       "1    0.00    0.00    1.00    1.00   0.0    0.00  neptune  19  \n",
       "2    0.03    0.04    0.03    0.01   0.0    0.01   normal  21  \n",
       "3    0.00    0.00    0.00    0.00   0.0    0.00   normal  21  \n",
       "4    0.00    0.00    0.00    0.00   1.0    1.00  neptune  21  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NSL_train = pd.read_csv('KDDTrain+.txt')\n",
    "NSL_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Reading in the data and adding the relevant columns '''\n",
    "just_for_columns = pd.read_pickle('kddcup99dataframe')\n",
    "\n",
    "NSL_train = pd.read_csv('KDDTrain+.txt')\n",
    "difficulty_train = NSL_train.iloc[:, NSL_train.shape[1]-1]\n",
    "NSL_train = NSL_train.drop(list(NSL_train.columns)[NSL_train.shape[1]-1], axis=1)\n",
    "NSL_train.columns = list(just_for_columns.columns)\n",
    "NSL_x_train, NSL_y_train = NSL_train.iloc[:, :NSL_train.shape[1]-1], NSL_train.iloc[:, NSL_train.shape[1]-1]\n",
    "\n",
    "NSL_test = pd.read_csv('KDDTest+.txt')\n",
    "difficulty_test = NSL_test.iloc[:, NSL_test.shape[1]-1]\n",
    "NSL_test = NSL_test.drop(list(NSL_test.columns)[NSL_test.shape[1]-1], axis=1)\n",
    "NSL_test.columns = list(just_for_columns.columns)\n",
    "NSL_x_test, NSL_y_test = NSL_test.iloc[:, :NSL_test.shape[1]-1], NSL_test.iloc[:, NSL_test.shape[1]-1]\n",
    "\n",
    "\n",
    "NSL_test_21 = pd.read_csv('KDDTest-21.txt')\n",
    "difficulty_test_21 = NSL_test_21.iloc[:, NSL_test_21.shape[1]-1]\n",
    "NSL_test_21 = NSL_test_21.drop(list(NSL_test_21.columns)[NSL_test_21.shape[1]-1], axis=1)\n",
    "NSL_test_21.columns = list(just_for_columns.columns)\n",
    "NSL_x_test_21, NSL_y_test_21 = NSL_test_21.iloc[:, :NSL_test_21.shape[1]-1], NSL_test_21.iloc[:, NSL_test_21.shape[1]-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11849,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NSL_y_test_21.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = [NSL_train, NSL_test, NSL_test_21]\n",
    "combined_df = pd.concat(frames)\n",
    "combined_df_all_numerical = pd.get_dummies(combined_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "NSL_train_dummy, NSL_test_dummy, NSL_test_21_dummy = combined_df_all_numerical.iloc[0:NSL_train.shape[0], :], combined_df_all_numerical.iloc[NSL_train.shape[0]:NSL_train.shape[0]+NSL_test.shape[0], :], combined_df_all_numerical.iloc[NSL_train.shape[0]+NSL_test.shape[0]:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "121"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(NSL_train_dummy.columns).index('flag_SH')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "NSL_x_train_dummy = NSL_train_dummy.iloc[:,0:122]\n",
    "NSL_x_test_dummy = NSL_test_dummy.iloc[:,0:122]\n",
    "NSL_x_test_21_dummy = NSL_test_21_dummy.iloc[:,0:122]\n",
    "''' This data is the 23 dimension one hot encoded vector for each intrusion type'''\n",
    "NSL_y_train_dummy = NSL_train_dummy.iloc[:,122:]\n",
    "NSL_y_test_dummy = NSL_test_dummy.iloc[:,122:]\n",
    "NSL_y_test_21_dummy = NSL_test_21_dummy.iloc[:,122:] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11849, 122)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NSL_x_test_21_dummy.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11849, 40)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NSL_y_test_21_dummy.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' creation of the binary classification and 4-way classification datasets '''\n",
    "\n",
    "intrusion_name_to_type = {\n",
    "    'apache2' : 'dos',\n",
    "    'normal' : 'normal',\n",
    "    'back' : 'dos',\n",
    "    'buffer_overflow':'u2r',\n",
    "    'ftp_write' :'r2l',\n",
    "    'guess_passwd': 'r2l',\n",
    "    'httptunnel' : 'r2l',\n",
    "    'imap': 'r2l',\n",
    "    'ipsweep' :'probe',\n",
    "    'land': 'dos',\n",
    "    'loadmodule': 'u2r',\n",
    "    'mailbomb' : 'dos',\n",
    "    'mscan' : 'probe',\n",
    "    'multihop': 'r2l',\n",
    "    'named' : 'r2l',\n",
    "    'neptune': 'dos',\n",
    "    'nmap': 'probe',\n",
    "    'ps' : 'u2r',\n",
    "    'perl': 'u2r',\n",
    "    'phf' :'r2l',\n",
    "    'pod':'dos',\n",
    "    'processtable' : 'dos',\n",
    "    'portsweep': 'probe',\n",
    "    'rootkit': 'u2r',\n",
    "    'saint' : 'probe',\n",
    "    'satan' :'probe',\n",
    "    'sendmail' : 'r2l',\n",
    "    'smurf': 'dos',\n",
    "    'snmpguess' : 'r2l',\n",
    "    'snmpgetattack' : 'r2l',\n",
    "    'sqlattack' : 'u2r',\n",
    "    'spy': 'r2l',\n",
    "    'teardrop': 'dos',\n",
    "    'udpstorm' : 'dos',\n",
    "    'worm' : 'dos',\n",
    "    'warezclient': 'r2l',\n",
    "    'warezmaster': 'r2l',\n",
    "    'xlock' : 'r2l',\n",
    "    'xsnoop' : 'r2l',\n",
    "    'xterm' : 'u2r',\n",
    "    'snmpgetattack': 'r2l',\n",
    "    'named' : 'r2l',\n",
    "    \n",
    "}\n",
    "\n",
    "intrusion_type_to_one_hot_index = {\n",
    "    'normal' : 0,\n",
    "    'dos' : 1,\n",
    "    'r2l' : 2,\n",
    "    'u2r' : 3,\n",
    "    'probe' : 4\n",
    "}\n",
    "\n",
    "def create_binary_data(original_y_labels):\n",
    "    ''' input should be the list of the y_labels ie. \"normal, neptune, etc. \n",
    "        returns a dataframe. \"'''\n",
    "    \n",
    "    y_data_binary = []\n",
    "    \n",
    "    for row in range(original_y_labels.shape[0]):\n",
    "        if \"normal\" in original_y_labels.iloc[row]:\n",
    "            y_data_binary.append(0)\n",
    "        else:\n",
    "            y_data_binary.append(1)\n",
    "    \n",
    "    y_data_binary = np.asarray(y_data_binary)\n",
    "    y_data_binary_df = pd.DataFrame(y_data_binary)\n",
    "    y_data_binary_df.columns = [\"label\"]\n",
    "    \n",
    "    return y_data_binary_df\n",
    "\n",
    "def create_attack_type_data(original_y_labels):\n",
    "    ''' input should be the list of the y_labels ie. \"normal, neptune, etc.\" \n",
    "        returns a dataframe. '''\n",
    "    \n",
    "    y_data_attack_type = []\n",
    "    \n",
    "    for row in range(original_y_labels.shape[0]):\n",
    "        one_hot = [0, 0, 0, 0, 0]\n",
    "        intrusion_type = intrusion_name_to_type[original_y_labels[row]] #format this such that you capture the entire label, so do [:-1] for the KDD data\n",
    "        one_hot_index = intrusion_type_to_one_hot_index[intrusion_type] # and do the entire thing for the NSL data.\n",
    "        one_hot[one_hot_index] = 1\n",
    "        y_data_attack_type.append(one_hot)\n",
    "    \n",
    "    y_data_attack_type = np.asarray(y_data_attack_type)\n",
    "    y_data_attack_type_df = pd.DataFrame(y_data_attack_type)\n",
    "    y_data_attack_type_df.columns = [\"normal\", \"dos\", \"r2l\", \"u2r\", \"probe\"]\n",
    "    \n",
    "    return y_data_attack_type_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "NSL_y_train_binary = create_binary_data(NSL_y_train)\n",
    "NSL_y_test_binary = create_binary_data(NSL_y_test)\n",
    "NSL_y_test_21_binary = create_binary_data(NSL_y_test_21)\n",
    "\n",
    "\n",
    "NSL_y_train_attacktype = create_attack_type_data(NSL_y_train)\n",
    "NSL_y_test_attacktype = create_attack_type_data(NSL_y_test)\n",
    "NSL_y_test_21_attacktype = create_attack_type_data(NSL_y_test_21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11849, 5)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NSL_y_test_21_attacktype.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' time to pickle all the data files '''\n",
    "\n",
    "#by train I mean like the entire dataset, but i think I will use this as train??\n",
    "NSL_x_train_dummy.to_pickle('NSL_x_train_dummy.pkl')\n",
    "NSL_y_train_dummy.to_pickle('NSL_y_train_col_40.pkl')\n",
    "NSL_y_train_binary.to_pickle('NSL_y_train_col_1.pkl')\n",
    "NSL_y_train_attacktype.to_pickle('NSL_y_train_col_5.pkl')\n",
    "\n",
    "NSL_x_test_dummy.to_pickle('NSL_x_test_dummy.pkl')\n",
    "NSL_y_test_dummy.to_pickle('NSL_y_test_col_40.pkl')\n",
    "NSL_y_test_binary.to_pickle('NSL_y_test_col_1.pkl')\n",
    "NSL_y_test_attacktype.to_pickle('NSL_y_test_col_5.pkl')\n",
    "\n",
    "NSL_x_test_21_dummy.to_pickle('NSL_x_test_21_dummy.pkl')\n",
    "NSL_y_test_21_dummy.to_pickle('NSL_y_test_21_col_40.pkl')\n",
    "NSL_y_test_21_binary.to_pickle('NSL_y_test_21_col_1.pkl')\n",
    "NSL_y_test_21_attacktype.to_pickle('NSL_y_test_21_col_5.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}