stack_04_1,3-7_model7-kfold.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import Lasso, LassoCV, LinearRegression\n",
    "from sklearn.model_selection import KFold\n",
    "from utilities import cal_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack_idx = '05'\n",
    "models = '1,3-7,'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read CV predictions and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_models(exp):\n",
    "    exp_split = exp.split(',')\n",
    "    idx_models = []\n",
    "    for e in exp_split:\n",
    "        if '-' in e:\n",
    "            n0, n1 = e.split('-')\n",
    "            idx_models.extend(list(range(int(n0), int(n1)+1, 1)))\n",
    "        else:\n",
    "            idx_models.append(int(e))\n",
    "    return idx_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_models = parse_models(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_in_output = [f for f in os.listdir('output/') if os.path.isfile('output/'+f)]\n",
    "files_cv = [[f for f in files_in_output if 'model-%02d-' % idx in f and 'cv' in f][0] for idx in idx_models]\n",
    "files_test_one = [[f for f in files_in_output if 'model-%02d-' % idx in f and 'test-one' in f][0] \\\n",
    "                  for idx in idx_models]\n",
    "files_test_kf = [[f for f in files_in_output if 'model-%02d-' % idx in f and 'test-kfold' in f][0] \\\n",
    "                 for idx in idx_models]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['model-01-lgb-test-kfold.csv', 'model-03-lgb-feats-selection-test-kfold.csv', 'model-04-lgb-PCA-test-kfold.csv', 'model-05-lgb-wo-per-area-test-kfold.csv', 'model-06-lgb-lr0.001-test-kfold.csv', 'model-07-keras-embedding-test-kfold.csv']\n",
      "['model-01-lgb-test-one.csv', 'model-03-lgb-feats-selection-test-one.csv', 'model-04-lgb-PCA-test-one.csv', 'model-05-lgb-wo-per-area-test-one.csv', 'model-06-lgb-lr0.001-test-one.csv', 'model-07-keras-embedding-test-one.csv']\n"
     ]
    }
   ],
   "source": [
    "print(files_test_kf)\n",
    "print(files_test_one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['model-01-lgb-test-one.csv', 'model-03-lgb-feats-selection-test-one.csv', 'model-04-lgb-PCA-test-one.csv', 'model-05-lgb-wo-per-area-test-one.csv', 'model-06-lgb-lr0.001-test-one.csv', 'model-07-keras-embedding-test-kfold.csv']\n"
     ]
    }
   ],
   "source": [
    "### !!!!!!!! HACKING THE CODE HERE !!!!!!!!!! ###\n",
    "files_test_one.remove('model-07-keras-embedding-test-one.csv')\n",
    "files_test_one.append('model-07-keras-embedding-test-kfold.csv')\n",
    "print(files_test_one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = None\n",
    "test_one = None\n",
    "test_kf = None\n",
    "\n",
    "for i, f in enumerate(files_cv):\n",
    "    df = pd.read_csv('output/'+f)\n",
    "    if cv is None:\n",
    "        cv = df[['building_id','total_price_predict']].copy()\n",
    "    else:\n",
    "        cv = pd.merge(cv, df[['building_id','total_price_predict']], on='building_id')\n",
    "    cv = cv.rename(columns = {'total_price_predict':'pred_{}'.format(idx_models[i])})\n",
    "    cv[f'log_pred_{idx_models[i]}'] = np.log1p(cv[f'pred_{idx_models[i]}'])\n",
    "cv = pd.merge(cv, df[['building_id','total_price']], on='building_id')\n",
    "cv['log_total_price'] = np.log1p(cv['total_price'])\n",
    "\n",
    "for i, f in enumerate(files_test_one):\n",
    "    df = pd.read_csv('output/'+f)\n",
    "    if test_one is None:\n",
    "        test_one = df[['building_id','total_price']].copy()\n",
    "    else:\n",
    "        test_one = pd.merge(test_one, df[['building_id','total_price']], on='building_id')\n",
    "    test_one = test_one.rename(columns = {'total_price':'pred_{}'.format(idx_models[i])})\n",
    "    test_one[f'log_pred_{idx_models[i]}'] = np.log1p(test_one[f'pred_{idx_models[i]}'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_one[['building_id','pred_5']].rename(columns={'building_id':'id','pred_5':'target'})\\\n",
    "    .to_csv('test_5.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>log_pred_3</th>\n",
       "      <th>pred_4</th>\n",
       "      <th>log_pred_4</th>\n",
       "      <th>pred_5</th>\n",
       "      <th>log_pred_5</th>\n",
       "      <th>pred_6</th>\n",
       "      <th>log_pred_6</th>\n",
       "      <th>pred_7</th>\n",
       "      <th>log_pred_7</th>\n",
       "      <th>total_price</th>\n",
       "      <th>log_total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jre1pJhcQj91Kdky</td>\n",
       "      <td>1.255350e+07</td>\n",
       "      <td>16.345510</td>\n",
       "      <td>1.255981e+07</td>\n",
       "      <td>16.346013</td>\n",
       "      <td>1.288381e+07</td>\n",
       "      <td>16.371482</td>\n",
       "      <td>1.282213e+07</td>\n",
       "      <td>16.366683</td>\n",
       "      <td>1.247484e+07</td>\n",
       "      <td>16.339224</td>\n",
       "      <td>12309220.0</td>\n",
       "      <td>16.325859</td>\n",
       "      <td>14215011.0</td>\n",
       "      <td>16.469809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jcbuA8q3KPH9SzpS</td>\n",
       "      <td>9.117771e+06</td>\n",
       "      <td>16.025736</td>\n",
       "      <td>9.675385e+06</td>\n",
       "      <td>16.085096</td>\n",
       "      <td>7.095057e+06</td>\n",
       "      <td>15.774909</td>\n",
       "      <td>9.570172e+06</td>\n",
       "      <td>16.074162</td>\n",
       "      <td>9.175013e+06</td>\n",
       "      <td>16.031994</td>\n",
       "      <td>7909433.5</td>\n",
       "      <td>15.883567</td>\n",
       "      <td>7642884.5</td>\n",
       "      <td>15.849286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EqWJpHmkXPyfHkB8</td>\n",
       "      <td>4.491672e+06</td>\n",
       "      <td>15.317736</td>\n",
       "      <td>4.511709e+06</td>\n",
       "      <td>15.322187</td>\n",
       "      <td>4.586149e+06</td>\n",
       "      <td>15.338551</td>\n",
       "      <td>4.687745e+06</td>\n",
       "      <td>15.360462</td>\n",
       "      <td>4.562418e+06</td>\n",
       "      <td>15.333364</td>\n",
       "      <td>4827711.0</td>\n",
       "      <td>15.389883</td>\n",
       "      <td>4807053.0</td>\n",
       "      <td>15.385595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ihrMWGM8Nq99Uvu6</td>\n",
       "      <td>6.918995e+06</td>\n",
       "      <td>15.749781</td>\n",
       "      <td>6.878882e+06</td>\n",
       "      <td>15.743967</td>\n",
       "      <td>6.906435e+06</td>\n",
       "      <td>15.747964</td>\n",
       "      <td>6.981481e+06</td>\n",
       "      <td>15.758772</td>\n",
       "      <td>6.937283e+06</td>\n",
       "      <td>15.752421</td>\n",
       "      <td>7259765.5</td>\n",
       "      <td>15.797858</td>\n",
       "      <td>7493026.5</td>\n",
       "      <td>15.829483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jRhwzoVMFLFzxAX1</td>\n",
       "      <td>2.750253e+06</td>\n",
       "      <td>14.827204</td>\n",
       "      <td>2.754684e+06</td>\n",
       "      <td>14.828814</td>\n",
       "      <td>2.725200e+06</td>\n",
       "      <td>14.818053</td>\n",
       "      <td>2.838090e+06</td>\n",
       "      <td>14.858642</td>\n",
       "      <td>2.766369e+06</td>\n",
       "      <td>14.833047</td>\n",
       "      <td>2949650.8</td>\n",
       "      <td>14.897198</td>\n",
       "      <td>2846855.8</td>\n",
       "      <td>14.861726</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id        pred_1  log_pred_1        pred_3  log_pred_3  \\\n",
       "0  jre1pJhcQj91Kdky  1.255350e+07   16.345510  1.255981e+07   16.346013   \n",
       "1  jcbuA8q3KPH9SzpS  9.117771e+06   16.025736  9.675385e+06   16.085096   \n",
       "2  EqWJpHmkXPyfHkB8  4.491672e+06   15.317736  4.511709e+06   15.322187   \n",
       "3  ihrMWGM8Nq99Uvu6  6.918995e+06   15.749781  6.878882e+06   15.743967   \n",
       "4  jRhwzoVMFLFzxAX1  2.750253e+06   14.827204  2.754684e+06   14.828814   \n",
       "\n",
       "         pred_4  log_pred_4        pred_5  log_pred_5        pred_6  \\\n",
       "0  1.288381e+07   16.371482  1.282213e+07   16.366683  1.247484e+07   \n",
       "1  7.095057e+06   15.774909  9.570172e+06   16.074162  9.175013e+06   \n",
       "2  4.586149e+06   15.338551  4.687745e+06   15.360462  4.562418e+06   \n",
       "3  6.906435e+06   15.747964  6.981481e+06   15.758772  6.937283e+06   \n",
       "4  2.725200e+06   14.818053  2.838090e+06   14.858642  2.766369e+06   \n",
       "\n",
       "   log_pred_6      pred_7  log_pred_7  total_price  log_total_price  \n",
       "0   16.339224  12309220.0   16.325859   14215011.0        16.469809  \n",
       "1   16.031994   7909433.5   15.883567    7642884.5        15.849286  \n",
       "2   15.333364   4827711.0   15.389883    4807053.0        15.385595  \n",
       "3   15.752421   7259765.5   15.797858    7493026.5        15.829483  \n",
       "4   14.833047   2949650.8   14.897198    2846855.8        14.861726  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/model_selection/_split.py:2053: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:474: UserWarning: Coordinate descent with alpha=0 may lead to unexpected results and is discouraged.\n",
      "  tol, rng, random, positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:474: UserWarning: Coordinate descent with alpha=0 may lead to unexpected results and is discouraged.\n",
      "  tol, rng, random, positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:474: UserWarning: Coordinate descent with alpha=0 may lead to unexpected results and is discouraged.\n",
      "  tol, rng, random, positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LassoCV(alphas=[0, 0.0001, 0.0014677992676220704, 0.021544346900318846, 0.31622776601683794, 4.641588833612782, 68.12920690579622, 1000.0],\n",
       "    copy_X=True, cv='warn', eps=0.001, fit_intercept=True, max_iter=100000,\n",
       "    n_alphas=100, n_jobs=-1, normalize=False, positive=False,\n",
       "    precompute='auto', random_state=None, selection='cyclic', tol=1e-06,\n",
       "    verbose=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = cv[['log_pred_{}'.format(idx) for idx in idx_models]]\n",
    "y = cv['log_total_price']\n",
    "\n",
    "reg = LassoCV(alphas=[0]+list(np.logspace(-4, 3, 7)), max_iter=100000, tol=1e-6, n_jobs=-1)\n",
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0001\n",
      "[[1.3539862  1.37718519 1.34124855]\n",
      " [1.3539862  1.37718519 1.34124855]\n",
      " [1.3539862  1.37718519 1.34124855]\n",
      " [0.11259927 0.11703478 0.10974891]\n",
      " [0.03595454 0.03809815 0.03677873]\n",
      " [0.03556187 0.03765765 0.03653478]\n",
      " [0.03555864 0.03765107 0.03654161]\n",
      " [0.03554911 0.03764234 0.03659441]]\n",
      "[0.         0.27814086 0.01312845 0.27129653 0.29085225 0.15274723] -0.09449317064488127\n"
     ]
    }
   ],
   "source": [
    "print(reg.alpha_)\n",
    "print(reg.mse_path_)\n",
    "print(reg.coef_, reg.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:3: UserWarning: With alpha=0, this algorithm does not converge well. You are advised to use the LinearRegression estimator\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:478: UserWarning: Coordinate descent with no regularization may lead to unexpected results and is discouraged.\n",
      "  positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.13904431  0.29221487  0.01357291  0.27301462  0.41369209  0.15281691] -0.09449317064488127\n",
      "0.9731035108458638\n",
      "[0.         0.27814086 0.01312845 0.27129653 0.29085225 0.15274723] -0.09449317064488127\n",
      "0.9730993470042643\n",
      "[0.         0.27917245 0.010655   0.26765744 0.29449091 0.15308509] -0.09449317064488127\n",
      "0.9730980742812304\n",
      "[0.         0.29269405 0.         0.21225627 0.32743589 0.15665231] -0.09449317064488127\n",
      "0.9728273078530136\n",
      "[0.         0.36743025 0.         0.         0.22506023 0.16951731] -0.09449317064488127\n",
      "0.9167269231798255\n",
      "[0. 0. 0. 0. 0. 0.] -0.09449317064488127\n",
      "0.0\n",
      "[0. 0. 0. 0. 0. 0.] -0.09449317064488127\n",
      "0.0\n",
      "[0. 0. 0. 0. 0. 0.] -0.09449317064488127\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "for a in [0]+list(np.logspace(-4, 3, 7)):\n",
    "    reg_single = Lasso(alpha=a, max_iter=100000, tol=1e-6)\n",
    "    reg_single.fit(X, y)\n",
    "    print(reg_single.coef_, reg.intercept_)\n",
    "    print(reg_single.score(X,y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==== Fold 1 ====\n",
      "alpha, score: 0 5904.872604114404\n",
      "alpha, score: 0.0001 5924.872584279537\n",
      "alpha, score: 0.0002 5924.872586001879\n",
      "alpha, score: 0.0005 5926.87259060121\n",
      "alpha, score: 0.0008 5923.872594457766\n",
      "alpha, score: 0.001 5923.872596653028\n",
      "alpha, score: 0.002 5924.872601935233\n",
      "alpha, score: 0.005 5923.872568728334\n",
      "alpha, score: 0.008 5917.872449180822\n",
      "alpha, score: 0.01 5914.872312032804\n",
      "alpha, score: 0.02 5876.871151802474\n",
      "==== Fold 2 ====\n",
      "alpha, score: 0 5888.873197780169\n",
      "alpha, score: 0.0001 5886.873192543575\n",
      "alpha, score: 0.0002 5886.873196578011\n",
      "alpha, score: 0.0005 5885.873208200674\n",
      "alpha, score: 0.0008 5883.873218957404\n",
      "alpha, score: 0.001 5884.873225724938\n",
      "alpha, score: 0.002 5884.8732554480175\n",
      "alpha, score: 0.005 5895.8732947904045\n",
      "alpha, score: 0.008 5893.873248324038\n",
      "alpha, score: 0.01 5878.873163871372\n",
      "alpha, score: 0.02 5860.872264937752\n",
      "==== Fold 3 ====\n",
      "alpha, score: 0 5910.877397052235\n",
      "alpha, score: 0.0001 5918.877416918593\n",
      "alpha, score: 0.0002 5918.877418596997\n",
      "alpha, score: 0.0005 5920.8774232428095\n",
      "alpha, score: 0.0008 5922.877427202089\n",
      "alpha, score: 0.001 5921.877429503652\n",
      "alpha, score: 0.002 5914.8774364371975\n",
      "alpha, score: 0.005 5905.8774091946625\n",
      "alpha, score: 0.008 5911.877297507934\n",
      "alpha, score: 0.01 5916.877172653478\n",
      "alpha, score: 0.02 5866.876095004818\n",
      "[0.0005, 5911.207740681563, [5926.87259060121, 5885.873208200674, 5920.8774232428095]]\n",
      "[0.001, 5910.207750627206, [5923.872596653028, 5884.873225724938, 5921.877429503652]]\n",
      "[0.0008, 5910.20774687242, [5923.872594457766, 5883.873218957404, 5922.877427202089]]\n",
      "[0.0002, 5910.20773372563, [5924.872586001879, 5886.873196578011, 5918.877418596997]]\n",
      "[0.0001, 5910.207731247235, [5924.872584279537, 5886.873192543575, 5918.877416918593]]\n",
      "[0.005, 5908.541090904467, [5923.872568728334, 5895.8732947904045, 5905.8774091946625]]\n",
      "[0.002, 5908.207764606816, [5924.872601935233, 5884.8732554480175, 5914.8774364371975]]\n",
      "[0.008, 5907.874331670931, [5917.872449180822, 5893.873248324038, 5911.877297507934]]\n",
      "[0.01, 5903.5408828525515, [5914.872312032804, 5878.873163871372, 5916.877172653478]]\n",
      "[0, 5901.541066315603, [5904.872604114404, 5888.873197780169, 5910.877397052235]]\n",
      "[0.02, 5868.206503915015, [5876.871151802474, 5860.872264937752, 5866.876095004818]]\n"
     ]
    }
   ],
   "source": [
    "alphas = [0, 0.0001, 0.0002, 0.0005, 0.0008, 0.001, 0.002, 0.005, 0.008, 0.01, 0.02]\n",
    "gsearch = {}\n",
    "\n",
    "folds = KFold(n_splits=3, shuffle=True, random_state=1208)\n",
    "for i_fold, (itrain, ival) in enumerate(folds.split(X)): # kfold\n",
    "    print('==== Fold', i_fold+1, '====')\n",
    "    \n",
    "    # split train, val\n",
    "    X_train = X.iloc[itrain]\n",
    "    X_val = X.iloc[ival]\n",
    "    y_train = y.iloc[itrain]\n",
    "    y_val = y.iloc[ival]\n",
    "    \n",
    "    # random sample - grid search\n",
    "    for a in alphas:\n",
    "        if a == 0:\n",
    "            reg_single = LinearRegression()\n",
    "        else:\n",
    "            reg_single = Lasso(alpha=a, max_iter=100000, tol=1e-6)\n",
    "        reg_single.fit(X_train, y_train)\n",
    "        \n",
    "        y_pred = reg_single.predict(X_val)\n",
    "        y_pred_final = np.expm1(y_pred)\n",
    "        y_true_final = np.expm1(y_val)\n",
    "        score = cal_score(y_true_final, y_pred_final)\n",
    "\n",
    "        print('alpha, score:', a, score)\n",
    "        gsearch[a] = gsearch.get(a,[]) + [score]\n",
    "\n",
    "results = [[key, np.mean(value), value] for key, value, in gsearch.items()]\n",
    "results.sort(key= lambda x: x[1], reverse=True)\n",
    "for item in results:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0005\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Lasso(alpha=0.0005, copy_X=True, fit_intercept=True, max_iter=1000000,\n",
       "   normalize=False, positive=False, precompute=False, random_state=None,\n",
       "   selection='cyclic', tol=1e-06, warm_start=False)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha_set = results[0][0]\n",
    "print(alpha_set)\n",
    "if alpha_set == 0:\n",
    "    reg = LinearRegression()\n",
    "else:\n",
    "    reg = Lasso(alpha=alpha_set, max_iter=1000000, tol=1e-6)\n",
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.27844247 0.01240511 0.27023231 0.29191642 0.15284603] -0.08948078176987906\n"
     ]
    }
   ],
   "source": [
    "print(reg.coef_, reg.intercept_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate cv score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_pred_final = np.zeros(X.shape[0])\n",
    "for i, col in enumerate(X):\n",
    "    cv_pred_final = cv_pred_final + X[col] * reg.coef_[i]\n",
    "cv_pred_final = cv_pred_final + reg.intercept_\n",
    "cv_pred_final = np.expm1(cv_pred_final)\n",
    "cv_true_final = np.expm1(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14215011.0</td>\n",
       "      <td>1.264624e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7642884.5</td>\n",
       "      <td>9.216287e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4807053.0</td>\n",
       "      <td>4.622191e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7493026.5</td>\n",
       "      <td>6.998615e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2846855.8</td>\n",
       "      <td>2.801267e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            a             b\n",
       "0  14215011.0  1.264624e+07\n",
       "1   7642884.5  9.216287e+06\n",
       "2   4807053.0  4.622191e+06\n",
       "3   7493026.5  6.998615e+06\n",
       "4   2846855.8  2.801267e+06"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'a':cv_true_final,'b':cv_pred_final}).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5907.874460662518"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cal_score(cv_true_final, cv_pred_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pred_final = pd.DataFrame({'building_id': test_one['building_id'], 'total_price': np.zeros(test_one.shape[0])})\n",
    "for i, idx in enumerate(idx_models):\n",
    "    test_pred_final['total_price'] = test_pred_final['total_price'] + test_one[f'log_pred_{idx}'] * reg.coef_[i]\n",
    "test_pred_final['total_price'] = test_pred_final['total_price'] + reg.intercept_\n",
    "test_pred_final['total_price'] = np.expm1(test_pred_final['total_price'])\n",
    "test_pred_final['total_price'] = np.clip(test_pred_final['total_price'], 0, None)\n",
    "\n",
    "test_pred_final.to_csv('output/stack_{}_{}.csv'.format(stack_idx, models), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a= pd.read_csv('output/model-03-lgb-feats-selection-cv.csv')\n",
    "#b= pd.read_csv('output/model-03-lgb-feats-selection-test-one.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a.rename(columns={'building_id':'id','total_price_predict':'target'}).to_csv('opt-pred3.csv',index=False)\n",
    "#b.rename(columns={'building_id':'id','total_price':'target'}).to_csv('opt-test3.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}