Kaggle_Airbnb_part2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "from subprocess import check_output \n",
    "import sys\n",
    "from datetime import datetime\n",
    "%matplotlib inline\n",
    "os.chdir('C:/Kaggle/Airbnb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "countries=pd.read_csv('countries.csv')\n",
    "sessions=pd.read_csv('sessions.csv')\n",
    "test_users=pd.read_csv('test_users.csv')\n",
    "train_users_2=pd.read_csv('train_users_2.csv')\n",
    "train_lr=pd.read_csv('train_p.csv')\n",
    "test_lr=pd.read_csv('test_p.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train=train_users_2[train_users_2['date_account_created'].str.slice(0,4,1).astype(int)>=2014]\n",
    "X_test=test_users\n",
    "X_test['country_destination']=\"NDF\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:3443: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._update_inplace(new_data)\n"
     ]
    }
   ],
   "source": [
    "X_train['first_browser']=X_train['first_browser'].replace(X_train['first_browser'][X_train['first_browser'].str.contains(r'Chrom',case=False,na=False)].unique().tolist(),\"Chrome\")\n",
    "X_train['first_browser'].replace([\"Firefox\",\"Mobile Firefox\",\"Mozilla\"],'Firefox',inplace=True)\n",
    "X_train['first_browser'].replace([\"IE\",\"IE Mobile\"],'IE',inplace=True)\n",
    "X_train['first_browser'].replace([\"Mobile Safari\",\"Safari\"],'Safari',inplace=True)\n",
    "X_train['first_browser'].replace([\"Opera\",\"Opera Mini\",\"Opera Mobile\"],'Opera',inplace=True)\n",
    "X_train['first_browser'].replace([\"Arora\",\"Avant Browser\",\"Camino\",\"CometBird\",\"Comodo Dragon\",\"Conkeror\",\"CoolNovo\",\"Crazy Browser\",\"Epic\",\"Flock\",\"Google Earth\",\"Googlebot\",\"IBrowse\",\"IceDragon\",\"IceWeasel\",\"Iron\",\"Kindle Browser\",\"Maxthon\",\"Nintendo Browser\",\"NetNewsWire\",\"OmniWeb\",\"Outlook 2007\",\"Pale Moon\",\"Palm Pre web browser\",\"PS Vita browser\",\"RockMelt\",\"SeaMonkey\",\"SiteKiosk\",\"SlimBrowser\",\"Sogou Explorer\",\"Stainless\",\"TenFourFox\",\"TheWorld Browser\",\"UC Browser\",\"wOSBrowser\",\"Yandex.Browser\"],'Other',inplace=True)\n",
    "\n",
    "X_test['first_browser']=X_test['first_browser'].replace(X_test['first_browser'][X_test['first_browser'].str.contains(r'Chrom',case=False,na=False)].unique().tolist(),\"Chrome\")\n",
    "X_test['first_browser'].replace([\"Firefox\",\"Mobile Firefox\",\"Mozilla\"],'Firefox',inplace=True)\n",
    "X_test['first_browser'].replace([\"IE\",\"IE Mobile\"],'IE',inplace=True)\n",
    "X_test['first_browser'].replace([\"Mobile Safari\",\"Safari\"],'Safari',inplace=True)\n",
    "X_test['first_browser'].replace([\"Opera\",\"Opera Mini\",\"Opera Mobile\"],'Opera',inplace=True)\n",
    "X_test['first_browser'].replace([\"Arora\",\"Avant Browser\",\"Camino\",\"CometBird\",\"Comodo Dragon\",\"Conkeror\",\"CoolNovo\",\"Crazy Browser\",\"Epic\",\"Flock\",\"Google Earth\",\"Googlebot\",\"IBrowse\",\"IceDragon\",\"IceWeasel\",\"Iron\",\"Kindle Browser\",\"Maxthon\",\"Nintendo Browser\",\"NetNewsWire\",\"OmniWeb\",\"Outlook 2007\",\"Pale Moon\",\"Palm Pre web browser\",\"PS Vita browser\",\"RockMelt\",\"SeaMonkey\",\"SiteKiosk\",\"SlimBrowser\",\"Sogou Explorer\",\"Stainless\",\"TenFourFox\",\"TheWorld Browser\",\"UC Browser\",\"wOSBrowser\",\"Yandex.Browser\"],'Other',inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:4485: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._update_inplace(new_data)\n",
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2881: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  from ipykernel import kernelapp as app\n",
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  app.launch_new_instance()\n",
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "X_train['age'][X_train['age'].isnull()]=-1\n",
    "X_train['age'][X_train['age']>1800]=2014-X_train['age'][X_train['age']>1800]\n",
    "X_test['age'][X_test['age'].isnull()]=-1\n",
    "X_test['age'][X_test['age']>1800]=2014-X_test['age'][X_test['age']>1800]\n",
    "#X_train['age'].quantile(q=[0,0.01,0.05,0.1,0.25,0.5,0.75,0.9,0.95,0.99,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "#Merging LR predictions\n",
    "train_lr.drop(['Unnamed: 0'],inplace=True,axis=1)\n",
    "train_lr.columns=['id', 'pred_lr']\n",
    "test_lr.drop(['Unnamed: 0'],inplace=True,axis=1)\n",
    "test_lr.columns=['id', 'pred_lr']\n",
    "X_train=pd.merge(X_train,train_lr,on='id',how='left')\n",
    "X_test=pd.merge(X_test,test_lr,on='id',how='left')\n",
    "X_train['pred_lr'][X_train['pred_lr'].isnull()]=-1\n",
    "X_test['pred_lr'][X_test['pred_lr'].isnull()]=-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2651"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(X_train.pred_lr==-1).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Sessions data features\n",
    "sessions=sessions.rename(columns={'user_id':'id'})\n",
    "sessions=sessions[sessions['id'].isin(X_train['id'].unique().tolist()+X_test['id'].unique().tolist())]\n",
    "sessions['action'].replace(['agree_terms_check','agree_terms_uncheck'],'agree_terms',inplace=True)\n",
    "sessions['action'].replace(['ajax_google_translate_description','ajax_google_translate_reviews'],'google_translate',inplace=True)\n",
    "sessions['action'].replace(['ajax_payout_edit','ajax_payout_options_by_country'],'payout',inplace=True)\n",
    "sessions['action'].replace(['ajax_photo_widget','ajax_photo_widget_form_iframe'],'photo_widget',inplace=True)\n",
    "sessions['action'].replace(['ajax_referral_banner_experiment_type','ajax_referral_banner_type'],'banner',inplace=True)\n",
    "sessions['action'].replace(['create_ach','create_multiple','create_paypal'],'create',inplace=True)\n",
    "sessions['action'].replace(['complete_redirect','complete_status'],'complete',inplace=True)\n",
    "sessions['action'].replace(['departments','department'],'department',inplace=True)\n",
    "sessions['action'].replace(['edit_verification','edit'],'edit',inplace=True)\n",
    "sessions['action'].replace(['email_itinerary_colorbox','email_share','email_wishlist'],'email',inplace=True)\n",
    "sessions['action'].replace(['friends_new','friends'],'friends',inplace=True)\n",
    "sessions['action'].replace(['home_safety_landing','home_safety_terms'],'home_safety',inplace=True)\n",
    "sessions['action'].replace(['jumio_redirect','jumio_token'],'jumio',inplace=True)\n",
    "sessions['action'].replace(['kba_update','kba'],'kba',inplace=True)\n",
    "sessions['action'].replace(['listings','listing'],'listing',inplace=True)\n",
    "sessions['action'].replace(['message_to_host_change','message_to_host_focus'],'message_to_host',inplace=True)\n",
    "sessions['action'].replace(['multi_message','multi'],'multi',inplace=True)\n",
    "sessions['action'].replace(['qt_reply_v2','qt_with','qt2'],'qt',inplace=True)\n",
    "sessions['action'].replace(['reviews_new','reviews'],'reviews',inplace=True)\n",
    "sessions['action'].replace(['similar_listings_v2'],'similar_listings',inplace=True)\n",
    "sessions['action'].replace(['show_code'],'show',inplace=True)\n",
    "sessions['action'].replace(['social_connections'],'social',inplace=True)\n",
    "sessions['action'].replace(['terms_and_conditions'],'terms',inplace=True)\n",
    "sessions['action'].replace(['transaction_history_paginated'],'transaction_history',inplace=True)\n",
    "sessions['action'].replace(['travel_plans_current','travel_plans_previous'],'travel_plans',inplace=True)\n",
    "sessions['action'].replace(['update_cached','update_friends_display','update_hide_from_search_engines','update_notifications'],'update',inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#sessions.head()\n",
    "df_action = pd.crosstab(index=sessions['id'], columns=[sessions['action']]).fillna(0)\n",
    "df_action.reset_index(inplace=True)\n",
    "df_action.iloc[:,1:]=df_action.iloc[:,1:].clip(0,1)\n",
    "df_action.columns = ['id']+['action_' + str(col) for col in df_action.columns[1:]]\n",
    "\n",
    "df_action_type = pd.crosstab(index=sessions['id'], columns=[sessions['action_type']]).fillna(0)\n",
    "df_action_type.reset_index(inplace=True)\n",
    "df_action_type.iloc[:,1:]=df_action_type.iloc[:,1:].clip(0,1)\n",
    "df_action_type.columns = ['id']+['action_type_' + str(col) for col in df_action_type.columns[1:]]\n",
    "\n",
    "df_action_detail = pd.crosstab(index=sessions['id'], columns=[sessions['action_detail']]).fillna(0)\n",
    "df_action_detail.reset_index(inplace=True)\n",
    "df_action_detail.iloc[:,1:]=df_action_detail.iloc[:,1:].clip(0,1)\n",
    "df_action_detail.columns = ['id']+['action_detail_' + str(col) for col in df_action_detail.columns[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_action_detail=df_action_detail.assign(\n",
    "          action_detail_coupon =df_action_detail.action_detail_apply_coupon+df_action_detail.action_detail_apply_coupon_click+df_action_detail.action_detail_apply_coupon_click_success+df_action_detail.action_detail_apply_coupon_error+df_action_detail.action_detail_coupon_code_click+df_action_detail.action_detail_coupon_field_focus,\n",
    "          action_detail_cancel =df_action_detail.action_detail_cancellation_policies+df_action_detail.action_detail_cancellation_policy+df_action_detail.action_detail_cancellation_policy_click,\n",
    "          action_detail_guest  =df_action_detail.action_detail_guest_cancellation+df_action_detail.action_detail_guest_itinerary+df_action_detail.action_detail_guest_receipt,\n",
    "          action_detail_host   =df_action_detail.action_detail_host_guarantee+df_action_detail.action_detail_host_home+df_action_detail.action_detail_host_refund_guest+df_action_detail.action_detail_host_respond+df_action_detail.action_detail_host_respond_page+df_action_detail.action_detail_host_standard_suspension,\n",
    "          action_detail_listing=df_action_detail.action_detail_listing_descriptions+df_action_detail.action_detail_listing_recommendations+df_action_detail.action_detail_listing_reviews+df_action_detail.action_detail_listing_reviews_page+df_action_detail.action_detail_manage_listing+df_action_detail.action_detail_view_listing,\n",
    "          action_detail_message=df_action_detail.action_detail_lookup_message_thread+df_action_detail.action_detail_message_inbox+df_action_detail.action_detail_message_post+df_action_detail.action_detail_message_thread+df_action_detail.action_detail_message_to_host_change+df_action_detail.action_detail_message_to_host_focus,\n",
    "                                        )\n",
    "df_action_detail.iloc[:,1:]=df_action_detail.iloc[:,1:].clip(0,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train=pd.merge(X_train,df_action,on='id',how='left')\n",
    "X_train=pd.merge(X_train,df_action_type,on='id',how='left')\n",
    "X_train=pd.merge(X_train,df_action_detail,on='id',how='left')\n",
    "X_train.loc[:,X_train.loc[:,X_train.columns.to_series().str.startswith('action').tolist()].columns.tolist()]=X_train.loc[:,X_train.loc[:,X_train.columns.to_series().str.startswith('action').tolist()].columns.tolist()].replace(np.nan,0)\n",
    "\n",
    "X_test=pd.merge(X_test,df_action,on='id',how='left')\n",
    "X_test=pd.merge(X_test,df_action_type,on='id',how='left')\n",
    "X_test=pd.merge(X_test,df_action_detail,on='id',how='left')\n",
    "X_test.loc[:,X_test.loc[:,X_test.columns.to_series().str.startswith('action').tolist()].columns.tolist()]=X_test.loc[:,X_test.loc[:,X_test.columns.to_series().str.startswith('action').tolist()].columns.tolist()].replace(np.nan,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_ids=X_train['id']\n",
    "test_ids=X_test['id']\n",
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(X_train['country_destination'].unique().tolist())\n",
    "target=le.transform(X_train['country_destination'])\n",
    "\n",
    "train_id_target=pd.DataFrame({'target':target.tolist(),'target_map':le.inverse_transform(target).tolist()})\n",
    "train_id_target.to_pickle('train_id_target')\n",
    "pd.DataFrame(test_ids).to_pickle('test_ids')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#cp=X_train.copy()\n",
    "X_train=X_train.assign(year_account=X_train['date_account_created'].str.slice(0,4,1).astype(int),\n",
    "                       month_account=X_train['date_account_created'].str.slice(5,7,1).astype(int),\n",
    "                       day_account=X_train['date_account_created'].str.slice(8,10,1).astype(int),\n",
    "                       weekday_account=pd.to_datetime(X_train['date_account_created'],format='%Y-%m-%d').dt.dayofweek,\n",
    "                       year_active=X_train['timestamp_first_active'].astype(str).str.slice(0,4,1).astype(int),\n",
    "                       month_active=X_train['timestamp_first_active'].astype(str).str.slice(4,6,1).astype(int),\n",
    "                       day_active=X_train['timestamp_first_active'].astype(str).str.slice(6,8,1).astype(int),\n",
    "                       weekday_active=pd.to_datetime(((X_train.timestamp_first_active).astype(str).str.slice(0,4,1)+ '-' + (X_train.timestamp_first_active).astype(str).str.slice(4,6,1) + '-' + (X_train.timestamp_first_active).astype(str).str.slice(6,8,1)),format='%Y-%m-%d').dt.dayofweek,\n",
    "                      )\n",
    "X_test=X_test.assign(year_account=X_test['date_account_created'].str.slice(0,4,1).astype(int),\n",
    "                       month_account=X_test['date_account_created'].str.slice(5,7,1).astype(int),\n",
    "                       day_account=X_test['date_account_created'].str.slice(8,10,1).astype(int),\n",
    "                       weekday_account=pd.to_datetime(X_test['date_account_created'],format='%Y-%m-%d').dt.dayofweek,\n",
    "                       year_active=X_test['timestamp_first_active'].astype(str).str.slice(0,4,1).astype(int),\n",
    "                       month_active=X_test['timestamp_first_active'].astype(str).str.slice(4,6,1).astype(int),\n",
    "                       day_active=X_test['timestamp_first_active'].astype(str).str.slice(6,8,1).astype(int),\n",
    "                       weekday_active=pd.to_datetime(((X_test.timestamp_first_active).astype(str).str.slice(0,4,1)+ '-' + (X_test.timestamp_first_active).astype(str).str.slice(4,6,1) + '-' + (X_test.timestamp_first_active).astype(str).str.slice(6,8,1)),format='%Y-%m-%d').dt.dayofweek,\n",
    "                      )\n",
    "X_train.drop(['id', 'date_first_booking','country_destination','date_account_created','timestamp_first_active'], axis=1, inplace=True)\n",
    "X_test.drop(['id', 'date_first_booking','country_destination','date_account_created','timestamp_first_active'], axis=1, inplace=True)\n",
    "X_train['age']=X_train['age'].map(lambda x: -1 if x<15 or x>100 else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#X_train.loc[:,list(~np.array(X_train.columns.str.contains('action').tolist()))].columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nitin.jain\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:2762: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  downcast=downcast, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "X_train['signup_flow']='signup_flow'+ X_train['signup_flow'].astype(str)\n",
    "X_train['weekday_account']='weekday_account'+ X_train['weekday_account'].astype(str)\n",
    "X_train['weekday_active']='weekday_active'+ X_train['weekday_active'].astype(str)\n",
    "X_test['signup_flow']='signup_flow'+ X_test['signup_flow'].astype(str)\n",
    "X_test['weekday_account']='weekday_account'+ X_test['weekday_account'].astype(str)\n",
    "X_test['weekday_active']='weekday_active'+ X_test['weekday_active'].astype(str)\n",
    "tot=X_train.append(X_test)\n",
    "enc_var= ['gender','signup_method','signup_flow','language','affiliate_channel','affiliate_provider','first_affiliate_tracked','signup_app','first_device_type','first_browser','weekday_active','weekday_account']\n",
    "onehot=pd.get_dummies(tot[enc_var])\n",
    "tot=pd.concat([tot.drop(enc_var,axis=1),onehot],axis=1)\n",
    "X_train=tot.iloc[:len(X_train),:]\n",
    "X_test=tot.iloc[len(X_train):,:]\n",
    "\n",
    "X_train.fillna(0,inplace=True)\n",
    "X_test.fillna(0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 511,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#cptrain=X_train.copy()\n",
    "#cptest=X_test.copy()\n",
    "\n",
    "#pd.DataFrame(X_train1.describe()).to_csv('xgbdataset1.csv')\n",
    "#removing binary vars whose sum is less than 5% of length  \n",
    "\n",
    "#X_train1=X_train.drop(list(set(X_train.loc[:,X_train.sum()<0.01*len(X_train)].columns) & set(X_train.loc[:,X_train.apply(pd.Series.nunique)<=2].columns)),axis=1)\n",
    "#X_test1=X_test.drop(list(set(X_train.loc[:,X_train.sum()<0.01*len(X_train)].columns) & set(X_train.loc[:,X_train.apply(pd.Series.nunique)<=2].columns)),axis=1)\n",
    "X_train1.to_pickle('X_train1')\n",
    "X_test1.to_pickle('X_test1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#X_train1.head()\n",
    "#X_train['weekday_account_weekday_account6'].value_counts().apply(lambda x: x*100/len(X_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#import xgboost as xgb"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}