stack_01_1,4-5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import Lasso, LassoCV, LinearRegression\n",
    "from sklearn.model_selection import KFold\n",
    "from utilities import cal_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack_idx = '01'\n",
    "models = '1,4-5'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read CV predictions and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_models(exp):\n",
    "    exp_split = exp.split(',')\n",
    "    idx_models = []\n",
    "    for e in exp_split:\n",
    "        if '-' in e:\n",
    "            n0, n1 = e.split('-')\n",
    "            idx_models.extend(list(range(int(n0), int(n1)+1, 1)))\n",
    "        else:\n",
    "            idx_models.append(int(e))\n",
    "    return idx_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_models = parse_models(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_in_output = [f for f in os.listdir('output/') if os.path.isfile('output/'+f)]\n",
    "files_cv = [[f for f in files_in_output if 'model-%02d-' % idx in f and 'cv' in f][0] for idx in idx_models]\n",
    "files_test_one = [[f for f in files_in_output if 'model-%02d-' % idx in f and 'test-one' in f][0] \\\n",
    "                  for idx in idx_models]\n",
    "files_test_kf = [[f for f in files_in_output if 'model-%02d-' % idx in f and 'test-kfold' in f][0] \\\n",
    "                 for idx in idx_models]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['model-01-lgb-test-kfold.csv',\n",
       " 'model-04-lgb-PCA-test-kfold.csv',\n",
       " 'model-05-lgb-wo-per-area-test-kfold.csv']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_test_kf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = None\n",
    "test_one = None\n",
    "test_kf = None\n",
    "\n",
    "for i, f in enumerate(files_cv):\n",
    "    df = pd.read_csv('output/'+f)\n",
    "    if cv is None:\n",
    "        cv = df[['building_id','total_price_predict']].copy()\n",
    "    else:\n",
    "        cv = pd.merge(cv, df[['building_id','total_price_predict']], on='building_id')\n",
    "    cv = cv.rename(columns = {'total_price_predict':'pred_{}'.format(idx_models[i])})\n",
    "    cv[f'log_pred_{idx_models[i]}'] = np.log1p(cv[f'pred_{idx_models[i]}'])\n",
    "cv = pd.merge(cv, df[['building_id','total_price']], on='building_id')\n",
    "cv['log_total_price'] = np.log1p(cv['total_price'])\n",
    "\n",
    "for i, f in enumerate(files_test_one):\n",
    "    df = pd.read_csv('output/'+f)\n",
    "    if test_one is None:\n",
    "        test_one = df[['building_id','total_price']].copy()\n",
    "    else:\n",
    "        test_one = pd.merge(test_one, df[['building_id','total_price']], on='building_id')\n",
    "    test_one = test_one.rename(columns = {'total_price':'pred_{}'.format(idx_models[i])})\n",
    "    test_one[f'log_pred_{idx_models[i]}'] = np.log1p(test_one[f'pred_{idx_models[i]}'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_one[['building_id','pred_5']].rename(columns={'building_id':'id','pred_5':'target'})\\\n",
    "    .to_csv('test_5.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>pred_4</th>\n",
       "      <th>log_pred_4</th>\n",
       "      <th>pred_5</th>\n",
       "      <th>log_pred_5</th>\n",
       "      <th>total_price</th>\n",
       "      <th>log_total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jre1pJhcQj91Kdky</td>\n",
       "      <td>1.255350e+07</td>\n",
       "      <td>16.345510</td>\n",
       "      <td>1.288381e+07</td>\n",
       "      <td>16.371482</td>\n",
       "      <td>1.282213e+07</td>\n",
       "      <td>16.366683</td>\n",
       "      <td>14215011.0</td>\n",
       "      <td>16.469809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jcbuA8q3KPH9SzpS</td>\n",
       "      <td>9.117771e+06</td>\n",
       "      <td>16.025736</td>\n",
       "      <td>7.095057e+06</td>\n",
       "      <td>15.774909</td>\n",
       "      <td>9.570172e+06</td>\n",
       "      <td>16.074162</td>\n",
       "      <td>7642884.5</td>\n",
       "      <td>15.849286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EqWJpHmkXPyfHkB8</td>\n",
       "      <td>4.491672e+06</td>\n",
       "      <td>15.317736</td>\n",
       "      <td>4.586149e+06</td>\n",
       "      <td>15.338551</td>\n",
       "      <td>4.687745e+06</td>\n",
       "      <td>15.360462</td>\n",
       "      <td>4807053.0</td>\n",
       "      <td>15.385595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ihrMWGM8Nq99Uvu6</td>\n",
       "      <td>6.918995e+06</td>\n",
       "      <td>15.749781</td>\n",
       "      <td>6.906435e+06</td>\n",
       "      <td>15.747964</td>\n",
       "      <td>6.981481e+06</td>\n",
       "      <td>15.758772</td>\n",
       "      <td>7493026.5</td>\n",
       "      <td>15.829483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jRhwzoVMFLFzxAX1</td>\n",
       "      <td>2.750253e+06</td>\n",
       "      <td>14.827204</td>\n",
       "      <td>2.725200e+06</td>\n",
       "      <td>14.818053</td>\n",
       "      <td>2.838090e+06</td>\n",
       "      <td>14.858642</td>\n",
       "      <td>2846855.8</td>\n",
       "      <td>14.861726</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id        pred_1  log_pred_1        pred_4  log_pred_4  \\\n",
       "0  jre1pJhcQj91Kdky  1.255350e+07   16.345510  1.288381e+07   16.371482   \n",
       "1  jcbuA8q3KPH9SzpS  9.117771e+06   16.025736  7.095057e+06   15.774909   \n",
       "2  EqWJpHmkXPyfHkB8  4.491672e+06   15.317736  4.586149e+06   15.338551   \n",
       "3  ihrMWGM8Nq99Uvu6  6.918995e+06   15.749781  6.906435e+06   15.747964   \n",
       "4  jRhwzoVMFLFzxAX1  2.750253e+06   14.827204  2.725200e+06   14.818053   \n",
       "\n",
       "         pred_5  log_pred_5  total_price  log_total_price  \n",
       "0  1.282213e+07   16.366683   14215011.0        16.469809  \n",
       "1  9.570172e+06   16.074162    7642884.5        15.849286  \n",
       "2  4.687745e+06   15.360462    4807053.0        15.385595  \n",
       "3  6.981481e+06   15.758772    7493026.5        15.829483  \n",
       "4  2.838090e+06   14.858642    2846855.8        14.861726  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/model_selection/_split.py:2053: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:474: UserWarning: Coordinate descent with alpha=0 may lead to unexpected results and is discouraged.\n",
      "  tol, rng, random, positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:474: UserWarning: Coordinate descent with alpha=0 may lead to unexpected results and is discouraged.\n",
      "  tol, rng, random, positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:1239: UserWarning: With alpha=0, this algorithm does not converge well. You are advised to use the LinearRegression estimator\n",
      "  model.fit(X, y)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:478: UserWarning: Coordinate descent with no regularization may lead to unexpected results and is discouraged.\n",
      "  positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LassoCV(alphas=[0, 0.0001, 0.0014677992676220704, 0.021544346900318846, 0.31622776601683794, 4.641588833612782, 68.12920690579622, 1000.0],\n",
       "    copy_X=True, cv='warn', eps=0.001, fit_intercept=True, max_iter=100000,\n",
       "    n_alphas=100, n_jobs=-1, normalize=False, positive=False,\n",
       "    precompute='auto', random_state=None, selection='cyclic', tol=1e-06,\n",
       "    verbose=False)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = cv[['log_pred_{}'.format(idx) for idx in idx_models]]\n",
    "y = cv['log_total_price']\n",
    "\n",
    "reg = LassoCV(alphas=[0]+list(np.logspace(-4, 3, 7)), max_iter=100000, tol=1e-6, n_jobs=-1)\n",
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(reg.alpha_)\n",
    "print(reg.mse_path_)\n",
    "print(reg.coef_, reg.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for a in [0]+list(np.logspace(-4, 3, 7)):\n",
    "    reg_single = Lasso(alpha=a, max_iter=100000, tol=1e-6)\n",
    "    reg_single.fit(X, y)\n",
    "    print(reg_single.coef_, reg.intercept_)\n",
    "    print(reg_single.score(X,y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alphas = [0, 0.0001, 0.0002, 0.0005, 0.0008, 0.001, 0.002, 0.005, 0.008, 0.01, 0.02]\n",
    "gsearch = {}\n",
    "\n",
    "folds = KFold(n_splits=3, shuffle=True, random_state=1208)\n",
    "for i_fold, (itrain, ival) in enumerate(folds.split(X)): # kfold\n",
    "    print('==== Fold', i_fold+1, '====')\n",
    "    \n",
    "    # split train, val\n",
    "    X_train = X.iloc[itrain]\n",
    "    X_val = X.iloc[ival]\n",
    "    y_train = y.iloc[itrain]\n",
    "    y_val = y.iloc[ival]\n",
    "    \n",
    "    # random sample - grid search\n",
    "    for a in alphas:\n",
    "        if a == 0:\n",
    "            reg_single = LinearRegression()\n",
    "        else:\n",
    "            reg_single = Lasso(alpha=a, max_iter=100000, tol=1e-6)\n",
    "        reg_single.fit(X_train, y_train)\n",
    "        \n",
    "        y_pred = reg_single.predict(X_val)\n",
    "        y_pred_final = np.expm1(y_pred)\n",
    "        y_true_final = np.expm1(y_val)\n",
    "        score = cal_score(y_true_final, y_pred_final)\n",
    "\n",
    "        print('alpha, score:', a, score)\n",
    "        gsearch[a] = gsearch.get(a,[]) + [score]\n",
    "\n",
    "results = [[key, np.mean(value), value] for key, value, in gsearch.items()]\n",
    "results.sort(key= lambda x: x[1], reverse=True)\n",
    "for item in results:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha_set = results[0][0]\n",
    "print(alpha_set)\n",
    "if alpha_set == 0:\n",
    "    reg = LinearRegression()\n",
    "else:\n",
    "    reg = Lasso(alpha=alpha_set, max_iter=1000000, tol=1e-6)\n",
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(reg.coef_, reg.intercept_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate cv score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_pred_final = np.zeros(X.shape[0])\n",
    "for i, col in enumerate(X):\n",
    "    cv_pred_final = cv_pred_final + X[col] * reg.coef_[i]\n",
    "cv_pred_final = cv_pred_final + reg.intercept_\n",
    "cv_pred_final = np.expm1(cv_pred_final)\n",
    "cv_true_final = np.expm1(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame({'a':cv_true_final,'b':cv_pred_final}).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cal_score(cv_true_final, cv_pred_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pred_final = pd.DataFrame({'building_id': test_one['building_id'], 'total_price': np.zeros(test_one.shape[0])})\n",
    "for i, idx in enumerate(idx_models):\n",
    "    test_pred_final['total_price'] = test_pred_final['total_price'] + test_one[f'log_pred_{idx}'] * reg.coef_[i]\n",
    "test_pred_final['total_price'] = test_pred_final['total_price'] + reg.intercept_\n",
    "test_pred_final['total_price'] = np.expm1(test_pred_final['total_price'])\n",
    "test_pred_final['total_price'] = np.clip(test_pred_final['total_price'], 0, None)\n",
    "\n",
    "test_pred_final.to_csv('output/stack_{}_{}.csv'.format(stack_idx, models), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}