blog_notebooks/regression/regression_2_blog.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cuml\n",
    "import cudf\n",
    "import nvcategory\n",
    "\n",
    "import xgboost as xgb\n",
    "import numpy as np\n",
    "from sklearn.metrics import mean_squared_error, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import urllib.request\n",
    "\n",
    "data_dir = '../../data/blackfriday/'\n",
    "if not os.path.exists(data_dir):\n",
    "    print('creating black friday data directory')\n",
    "    os.system('mkdir ../../data/blackfriday')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://datahack-prod.s3.amazonaws.com/train_zip/train_oSwQCTC.zip to ../../data/blackfriday/train.zip\n"
     ]
    }
   ],
   "source": [
    "base_url = 'https://datahack-prod.s3.amazonaws.com/train_zip/'\n",
    "ofn = 'train_oSwQCTC.zip'\n",
    "fn = 'train.zip'\n",
    "if not os.path.isfile(data_dir+ofn):\n",
    "        print(f'Downloading {base_url+ofn} to {data_dir+fn}')\n",
    "        urllib.request.urlretrieve(base_url+ofn, data_dir+fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read in the data. Notice how it decompresses as it reads the data into memory. \n",
    "gdf = cudf.read_csv(data_dir+fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "550068"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Product_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>City_Category</th>\n",
       "      <th>Stay_In_Current_City_Years</th>\n",
       "      <th>Marital_Status</th>\n",
       "      <th>Product_Category_1</th>\n",
       "      <th>Product_Category_2</th>\n",
       "      <th>Product_Category_3</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00069042</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>8370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00248942</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00087842</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>1422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00085442</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>null</td>\n",
       "      <td>1057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000002</td>\n",
       "      <td>P00285442</td>\n",
       "      <td>M</td>\n",
       "      <td>55+</td>\n",
       "      <td>16</td>\n",
       "      <td>C</td>\n",
       "      <td>4+</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>7969</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_ID Product_ID Gender   Age  Occupation City_Category  \\\n",
       "0  1000001  P00069042      F  0-17          10             A   \n",
       "1  1000001  P00248942      F  0-17          10             A   \n",
       "2  1000001  P00087842      F  0-17          10             A   \n",
       "3  1000001  P00085442      F  0-17          10             A   \n",
       "4  1000002  P00285442      M   55+          16             C   \n",
       "\n",
       "  Stay_In_Current_City_Years  Marital_Status  Product_Category_1  \\\n",
       "0                          2               0                   3   \n",
       "1                          2               0                   1   \n",
       "2                          2               0                  12   \n",
       "3                          2               0                  12   \n",
       "4                         4+               0                   8   \n",
       "\n",
       "  Product_Category_2 Product_Category_3  Purchase  \n",
       "0               null               null      8370  \n",
       "1                6.0               14.0     15200  \n",
       "2               null               null      1422  \n",
       "3               14.0               null      1057  \n",
       "4               null               null      7969  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Taking a look at the data.\n",
    "gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#grabbing the first character of the years in city string to get rid of plus sign, and converting to int\n",
    "gdf['city_years'] = gdf.Stay_In_Current_City_Years.str.get(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Here we can see how we can control the value of our category variables with the replace method and turn strings to ints\n",
    "gdf['City_Category'] = gdf.City_Category.str.replace('A', '1')\n",
    "gdf['City_Category'] = gdf.City_Category.str.replace('B', '2')\n",
    "gdf['City_Category'] = gdf.City_Category.str.replace('C', '3')\n",
    "gdf['City_Category'] = gdf['City_Category'].str.stoi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Product_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>City_Category</th>\n",
       "      <th>Stay_In_Current_City_Years</th>\n",
       "      <th>Marital_Status</th>\n",
       "      <th>Product_Category_1</th>\n",
       "      <th>Product_Category_2</th>\n",
       "      <th>Product_Category_3</th>\n",
       "      <th>Purchase</th>\n",
       "      <th>city_years</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00069042</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>8370</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00248942</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15200</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00087842</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>1422</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00085442</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>null</td>\n",
       "      <td>1057</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000002</td>\n",
       "      <td>P00285442</td>\n",
       "      <td>M</td>\n",
       "      <td>55+</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>4+</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>7969</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_ID Product_ID Gender   Age  Occupation  City_Category  \\\n",
       "0  1000001  P00069042      F  0-17          10              1   \n",
       "1  1000001  P00248942      F  0-17          10              1   \n",
       "2  1000001  P00087842      F  0-17          10              1   \n",
       "3  1000001  P00085442      F  0-17          10              1   \n",
       "4  1000002  P00285442      M   55+          16              3   \n",
       "\n",
       "  Stay_In_Current_City_Years  Marital_Status  Product_Category_1  \\\n",
       "0                          2               0                   3   \n",
       "1                          2               0                   1   \n",
       "2                          2               0                  12   \n",
       "3                          2               0                  12   \n",
       "4                         4+               0                   8   \n",
       "\n",
       "  Product_Category_2 Product_Category_3  Purchase city_years  \n",
       "0               null               null      8370          2  \n",
       "1                6.0               14.0     15200          2  \n",
       "2               null               null      1422          2  \n",
       "3               14.0               null      1057          2  \n",
       "4               null               null      7969          4  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf['Gender'] = gdf.Gender.str.replace('F', '1')\n",
    "gdf['Gender'] = gdf.Gender.str.replace('M', '0')\n",
    "gdf['Gender'] = gdf.Gender.str.stoi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique Products: 3631\n"
     ]
    }
   ],
   "source": [
    "#Let's take a look at how many products we have\n",
    "prod_count = gdf.Product_ID.unique().count()\n",
    "print(\"Unique Products: {}\".format(prod_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique Product Categories: 20\n"
     ]
    }
   ],
   "source": [
    "#Let's take a look at how many primary product categories we have\n",
    "#We do it differently here because the variable is a number, not a string\n",
    "prod1_count = gdf.Product_Category_1.unique().count()\n",
    "print(\"Unique Product Categories: {}\".format(prod1_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dummy for multi-category products\n",
    "gdf['Product_Category_2'] = gdf['Product_Category_2'].fillna(0)\n",
    "gdf['Product_Category_3'] = gdf['Product_Category_3'].fillna(0)\n",
    "gdf['multi'] = ((gdf['Product_Category_2'] + gdf['Product_Category_3'])>0).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Product_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>City_Category</th>\n",
       "      <th>Stay_In_Current_City_Years</th>\n",
       "      <th>Marital_Status</th>\n",
       "      <th>Product_Category_1</th>\n",
       "      <th>Product_Category_2</th>\n",
       "      <th>Product_Category_3</th>\n",
       "      <th>Purchase</th>\n",
       "      <th>city_years</th>\n",
       "      <th>multi</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00069042</td>\n",
       "      <td>1</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8370</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00248942</td>\n",
       "      <td>1</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15200</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00087842</td>\n",
       "      <td>1</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1422</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00085442</td>\n",
       "      <td>1</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1057</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000002</td>\n",
       "      <td>P00285442</td>\n",
       "      <td>0</td>\n",
       "      <td>55+</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>4+</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7969</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_ID Product_ID  Gender   Age  Occupation  City_Category  \\\n",
       "0  1000001  P00069042       1  0-17          10              1   \n",
       "1  1000001  P00248942       1  0-17          10              1   \n",
       "2  1000001  P00087842       1  0-17          10              1   \n",
       "3  1000001  P00085442       1  0-17          10              1   \n",
       "4  1000002  P00285442       0   55+          16              3   \n",
       "\n",
       "  Stay_In_Current_City_Years  Marital_Status  Product_Category_1  \\\n",
       "0                          2               0                   3   \n",
       "1                          2               0                   1   \n",
       "2                          2               0                  12   \n",
       "3                          2               0                  12   \n",
       "4                         4+               0                   8   \n",
       "\n",
       "   Product_Category_2  Product_Category_3  Purchase city_years  multi  \n",
       "0                 0.0                 0.0      8370          2      0  \n",
       "1                 6.0                14.0     15200          2      1  \n",
       "2                 0.0                 0.0      1422          2      0  \n",
       "3                14.0                 0.0      1057          2      1  \n",
       "4                 0.0                 0.0      7969          4      0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Gender/Marital Status interaction variable\n",
    "gdf['gen_mar_interaction'] = gdf['Gender']*gdf['Marital_Status']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Because Occupation is a code, it should converted into indicator variables\n",
    "gdf = gdf.one_hot_encoding('Occupation', 'occ_dummy', gdf.Occupation.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "User_ID                         int64\n",
       "Product_ID                     object\n",
       "Gender                          int64\n",
       "Age                            object\n",
       "Occupation                      int64\n",
       "City_Category                   int64\n",
       "Stay_In_Current_City_Years     object\n",
       "Marital_Status                  int64\n",
       "Product_Category_1              int64\n",
       "Product_Category_2            float64\n",
       "Product_Category_3            float64\n",
       "Purchase                        int64\n",
       "city_years                     object\n",
       "multi                           int64\n",
       "gen_mar_interaction             int64\n",
       "occ_dummy_0                   float64\n",
       "occ_dummy_1                   float64\n",
       "occ_dummy_2                   float64\n",
       "occ_dummy_3                   float64\n",
       "occ_dummy_4                   float64\n",
       "occ_dummy_5                   float64\n",
       "occ_dummy_6                   float64\n",
       "occ_dummy_7                   float64\n",
       "occ_dummy_8                   float64\n",
       "occ_dummy_9                   float64\n",
       "occ_dummy_10                  float64\n",
       "occ_dummy_11                  float64\n",
       "occ_dummy_12                  float64\n",
       "occ_dummy_13                  float64\n",
       "occ_dummy_14                  float64\n",
       "occ_dummy_15                  float64\n",
       "occ_dummy_16                  float64\n",
       "occ_dummy_17                  float64\n",
       "occ_dummy_18                  float64\n",
       "occ_dummy_19                  float64\n",
       "occ_dummy_20                  float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dummy variable from Int\n",
    "gdf = gdf.one_hot_encoding('City_Category', 'city_cat', gdf.City_Category.unique())\n",
    "\n",
    "#Dummy from string\n",
    "gdf = gdf.one_hot_encoding('Age', 'age', gdf.Age.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Product_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>City_Category</th>\n",
       "      <th>Stay_In_Current_City_Years</th>\n",
       "      <th>Marital_Status</th>\n",
       "      <th>Product_Category_1</th>\n",
       "      <th>Product_Category_2</th>\n",
       "      <th>...</th>\n",
       "      <th>city_cat_1</th>\n",
       "      <th>city_cat_2</th>\n",
       "      <th>city_cat_3</th>\n",
       "      <th>age_0-17</th>\n",
       "      <th>age_18-25</th>\n",
       "      <th>age_26-35</th>\n",
       "      <th>age_36-45</th>\n",
       "      <th>age_46-50</th>\n",
       "      <th>age_51-55</th>\n",
       "      <th>age_55+</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00069042</td>\n",
       "      <td>1</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00248942</td>\n",
       "      <td>1</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00087842</td>\n",
       "      <td>1</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00085442</td>\n",
       "      <td>1</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000002</td>\n",
       "      <td>P00285442</td>\n",
       "      <td>0</td>\n",
       "      <td>55+</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>4+</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_ID Product_ID  Gender   Age  Occupation  City_Category  \\\n",
       "0  1000001  P00069042       1  0-17          10              1   \n",
       "1  1000001  P00248942       1  0-17          10              1   \n",
       "2  1000001  P00087842       1  0-17          10              1   \n",
       "3  1000001  P00085442       1  0-17          10              1   \n",
       "4  1000002  P00285442       0   55+          16              3   \n",
       "\n",
       "  Stay_In_Current_City_Years  Marital_Status  Product_Category_1  \\\n",
       "0                          2               0                   3   \n",
       "1                          2               0                   1   \n",
       "2                          2               0                  12   \n",
       "3                          2               0                  12   \n",
       "4                         4+               0                   8   \n",
       "\n",
       "   Product_Category_2  ...  city_cat_1  city_cat_2  city_cat_3  age_0-17  \\\n",
       "0                 0.0  ...         1.0         0.0         0.0       1.0   \n",
       "1                 6.0  ...         1.0         0.0         0.0       1.0   \n",
       "2                 0.0  ...         1.0         0.0         0.0       1.0   \n",
       "3                14.0  ...         1.0         0.0         0.0       1.0   \n",
       "4                 0.0  ...         0.0         0.0         1.0       0.0   \n",
       "\n",
       "   age_18-25  age_26-35  age_36-45  age_46-50  age_51-55  age_55+  \n",
       "0        0.0        0.0        0.0        0.0        0.0      0.0  \n",
       "1        0.0        0.0        0.0        0.0        0.0      0.0  \n",
       "2        0.0        0.0        0.0        0.0        0.0      0.0  \n",
       "3        0.0        0.0        0.0        0.0        0.0      0.0  \n",
       "4        0.0        0.0        0.0        0.0        0.0      1.0  \n",
       "\n",
       "[5 rows x 46 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "User_ID                         int64\n",
       "Product_ID                     object\n",
       "Gender                          int64\n",
       "Age                            object\n",
       "Occupation                      int64\n",
       "City_Category                   int64\n",
       "Stay_In_Current_City_Years     object\n",
       "Marital_Status                  int64\n",
       "Product_Category_1              int64\n",
       "Product_Category_2            float64\n",
       "Product_Category_3            float64\n",
       "Purchase                        int64\n",
       "city_years                     object\n",
       "multi                           int64\n",
       "gen_mar_interaction             int64\n",
       "occ_dummy_0                   float64\n",
       "occ_dummy_1                   float64\n",
       "occ_dummy_2                   float64\n",
       "occ_dummy_3                   float64\n",
       "occ_dummy_4                   float64\n",
       "occ_dummy_5                   float64\n",
       "occ_dummy_6                   float64\n",
       "occ_dummy_7                   float64\n",
       "occ_dummy_8                   float64\n",
       "occ_dummy_9                   float64\n",
       "occ_dummy_10                  float64\n",
       "occ_dummy_11                  float64\n",
       "occ_dummy_12                  float64\n",
       "occ_dummy_13                  float64\n",
       "occ_dummy_14                  float64\n",
       "occ_dummy_15                  float64\n",
       "occ_dummy_16                  float64\n",
       "occ_dummy_17                  float64\n",
       "occ_dummy_18                  float64\n",
       "occ_dummy_19                  float64\n",
       "occ_dummy_20                  float64\n",
       "city_cat_1                    float64\n",
       "city_cat_2                    float64\n",
       "city_cat_3                    float64\n",
       "age_0-17                      float64\n",
       "age_18-25                     float64\n",
       "age_26-35                     float64\n",
       "age_36-45                     float64\n",
       "age_46-50                     float64\n",
       "age_51-55                     float64\n",
       "age_55+                       float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Solution:\n",
    "gdf = gdf.one_hot_encoding('Product_Category_1', 'product', gdf.Product_Category_1.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We're going to drop the variables we've transformed\n",
    "drop_list = ['User_ID', 'Age', 'Stay_In_Current_City_Years', 'City_Category','Product_ID', 'Product_Category_1', 'Product_Category_2', 'Product_Category_3']\n",
    "gdf = gdf.drop(drop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We're going to make a list of all the first indicator variables in a series now so it will be\n",
    "#easier to exclude them when we're doing regressions later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy_list = ['occ_dummy_0', 'city_cat_1', 'age_0-17', 'product_1', 'Purchase']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#All variables currently have to have the same type for some methods in cuML\n",
    "for col in gdf.columns.tolist():\n",
    "    gdf[col] = gdf[col].astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cuml.preprocessing.model_selection.train_test_split\n",
    "test_size = round(len(gdf)*0.2)\n",
    "train_size = round(len(gdf)-test_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = gdf.iloc[0:test_size]\n",
    "gdf_train = gdf.iloc[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Deleting the main gdf because we're going to be making other subsets and other stuff, so it will be nice to have the memory. \n",
    "del(gdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = gdf_train['Purchase'].log()\n",
    "X_reg = gdf_train.drop(dummy_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAX RMSE: RMSE_RIDGE_0.15\n"
     ]
    }
   ],
   "source": [
    "# # I'm going to perform a hyperparameter search for alpha in a ridge regression\n",
    "output_ridge = {}\n",
    "for alpha in np.around(np.arange(0.01, 1, 0.01), decimals=2):\n",
    "    \n",
    "    Ridge = cuml.Ridge(alpha=alpha, fit_intercept=False, normalize=True)\n",
    "    _fit = Ridge.fit(X_reg, y_train)\n",
    "    _y_hat = _fit.predict(X_reg)\n",
    "    _mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "    output_ridge['RMSE_RIDGE_{}'.format(alpha)] = _mse\n",
    "\n",
    "print('MAX RMSE: {}'.format(min(output_ridge, key=output_ridge.get)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "323.17560048679417\n"
     ]
    }
   ],
   "source": [
    "Ridge = cuml.Ridge(alpha=.1, fit_intercept=False, normalize=True)\n",
    "_fit = Ridge.fit(X_reg, y_train)\n",
    "_y_hat = _fit.predict(X_reg)\n",
    "_mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "print('{:,}'.format(_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAX RMSE: RMSE_Lasso_0.1\n"
     ]
    }
   ],
   "source": [
    "##Lasso\n",
    "output_lasso = {}\n",
    "for alpha in np.around(np.arange(0.1, 10, 0.1), decimals=2):\n",
    "    \n",
    "    Lasso = cuml.Lasso(alpha=alpha, fit_intercept=False, normalize=True)\n",
    "    _fit = Lasso.fit(X_reg, y_train)\n",
    "    _y_hat = _fit.predict(X_reg)\n",
    "    _mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "    output_lasso['RMSE_Lasso_{}'.format(alpha)] = _mse\n",
    "\n",
    "print('MAX RMSE: {}'.format(min(output_lasso, key=output_lasso.get)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2,944.920712005673\n"
     ]
    }
   ],
   "source": [
    "Lasso = cuml.Lasso(alpha=.1, fit_intercept=False, normalize=True)\n",
    "_fit = Lasso.fit(X_reg, y_train)\n",
    "_y_hat = _fit.predict(X_reg)\n",
    "_mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "print('{:,}'.format(_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAX RMSE: RMSE_ElasticNet_0.1_0.1\n"
     ]
    }
   ],
   "source": [
    "##Elastic Net\n",
    "output_en = {}\n",
    "for alpha in np.around(np.arange(0.1, 10, 0.1), decimals=2):\n",
    "    for ratio in np.around(np.arange(0.1, 1, 0.1), decimals=2):\n",
    "    \n",
    "        ElasticNet = cuml.ElasticNet(alpha=alpha, l1_ratio=ratio, fit_intercept=False, normalize=True)\n",
    "        _fit = ElasticNet.fit(X_reg, y_train)\n",
    "        _y_hat = _fit.predict(X_reg)\n",
    "        _mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "        output_en['RMSE_ElasticNet_{}_{}'.format(alpha, ratio)] = _mse\n",
    "\n",
    "print('MAX RMSE: {}'.format(min(output_en, key=output_en.get)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2,944.920712005673\n"
     ]
    }
   ],
   "source": [
    "ElasticNet = cuml.ElasticNet(alpha=.1, l1_ratio=.1, fit_intercept=False, normalize=True)\n",
    "_fit = ElasticNet.fit(X_reg, y_train)\n",
    "_y_hat = _fit.predict(X_reg)\n",
    "_mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "print('{:,}'.format(_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_xgb = gdf_train[['Purchase']].log()\n",
    "X_xgb = gdf_train.drop('Purchase')\n",
    "xgb_train_set = xgb.DMatrix(data=X_xgb, label=y_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    'nround':100,\n",
    "    'max_depth':4,\n",
    "    'max_leaves':2**4,\n",
    "    'tree_method':'gpu_hist',\n",
    "    'n_gpus':1,\n",
    "    'loss':'ls',\n",
    "    'objective':'reg:squarederror',\n",
    "    'max_features':'auto',\n",
    "    'criterion':'friedman_mse',\n",
    "    'grow_policy':'lossguide',\n",
    "    'verbose':True\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[22:02:13] WARNING: /conda/conda-bld/xgboost_1579803340791/work/include/xgboost/generic_parameters.h:36: \n",
      "n_gpus: \n",
      "\tDeprecated. Single process multi-GPU training is no longer supported.\n",
      "\tPlease switch to distributed training with one process per GPU.\n",
      "\tThis can be done using Dask or Spark.  See documentation for details.\n",
      "[22:02:13] WARNING: /conda/conda-bld/xgboost_1579803340791/work/src/learner.cc:306: \n",
      "Parameters: { criterion, loss, max_features, nround, verbose } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "xgb_model = xgb.train(xgb_params, dtrain=xgb_train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat_xgb = xgb_model.predict(xgb_train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "RMSE = np.sqrt(mean_squared_error(y_xgb['Purchase'].to_pandas(), y_hat_xgb)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4940293\n"
     ]
    }
   ],
   "source": [
    "print(RMSE)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}