stack_05_1,3-7,9.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import Lasso, LassoCV, LinearRegression\n",
    "from sklearn.model_selection import KFold\n",
    "from utilities import cal_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack_idx = '05'\n",
    "models = '1,3-7,9'\n",
    "use_test_kfold = set([7])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read CV predictions and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_models(exp):\n",
    "    exp_split = exp.split(',')\n",
    "    idx_models = []\n",
    "    for e in exp_split:\n",
    "        if '-' in e:\n",
    "            n0, n1 = e.split('-')\n",
    "            idx_models.extend(list(range(int(n0), int(n1)+1, 1)))\n",
    "        else:\n",
    "            idx_models.append(int(e))\n",
    "    return idx_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_models = parse_models(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 3, 4, 5, 6, 7, 9]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_in_output = [f for f in os.listdir('output/') if os.path.isfile('output/'+f)]\n",
    "\n",
    "files_cv = {idx: [f for f in files_in_output if 'model-%02d-' % idx in f and 'cv' in f][0] for idx in idx_models}\n",
    "\n",
    "files_test_one = {idx: [f for f in files_in_output if 'model-%02d-' % idx in f and 'test-one' in f][0] \\\n",
    "                  for idx in idx_models}\n",
    "files_test_kf = {idx: [f for f in files_in_output if 'model-%02d-' % idx in f and 'test-kfold' in f][0] \\\n",
    "                 for idx in idx_models}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 'model-01-lgb-cv.csv', 3: 'model-03-lgb-feats-selection-cv.csv', 4: 'model-04-lgb-PCA-cv.csv', 5: 'model-05-lgb-wo-per-area-cv.csv', 6: 'model-06-lgb-lr0.001-cv.csv', 7: 'model-07-keras-embedding-cv.csv', 9: 'model-09-lgb-feats-selection-75-cv.csv'}\n",
      "{1: 'model-01-lgb-test-kfold.csv', 3: 'model-03-lgb-feats-selection-test-kfold.csv', 4: 'model-04-lgb-PCA-test-kfold.csv', 5: 'model-05-lgb-wo-per-area-test-kfold.csv', 6: 'model-06-lgb-lr0.001-test-kfold.csv', 7: 'model-07-keras-embedding-test-kfold.csv', 9: 'model-09-lgb-feats-selection-75-test-kfold.csv'}\n",
      "{1: 'model-01-lgb-test-one.csv', 3: 'model-03-lgb-feats-selection-test-one.csv', 4: 'model-04-lgb-PCA-test-one.csv', 5: 'model-05-lgb-wo-per-area-test-one.csv', 6: 'model-06-lgb-lr0.001-test-one.csv', 7: 'model-07-keras-embedding-test-one.csv', 9: 'model-09-lgb-feats-selection-75-test-one.csv'}\n"
     ]
    }
   ],
   "source": [
    "print(files_cv)\n",
    "print(files_test_kf)\n",
    "print(files_test_one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = None\n",
    "test = None\n",
    "\n",
    "for i, idx in enumerate(idx_models):\n",
    "    f = files_cv[idx]\n",
    "    df = pd.read_csv('output/'+f)\n",
    "    \n",
    "    if cv is None:\n",
    "        cv = df[['building_id','total_price_predict']].copy()\n",
    "    else:\n",
    "        cv = pd.merge(cv, df[['building_id','total_price_predict']], on='building_id')\n",
    "    \n",
    "    cv = cv.rename(columns = {'total_price_predict':'pred_{}'.format(idx_models[i])})\n",
    "    cv[f'log_pred_{idx_models[i]}'] = np.log1p(cv[f'pred_{idx_models[i]}'])\n",
    "\n",
    "cv = pd.merge(cv, df[['building_id','total_price']], on='building_id')\n",
    "cv['log_total_price'] = np.log1p(cv['total_price'])\n",
    "\n",
    "for i, idx in enumerate(idx_models):\n",
    "    f = files_test_kf[idx] if idx in use_test_kfold else files_test_one[idx]\n",
    "    df = pd.read_csv('output/'+f)\n",
    "\n",
    "    if test is None:\n",
    "        test = df[['building_id','total_price']].copy()\n",
    "    else:\n",
    "        test = pd.merge(test, df[['building_id','total_price']], on='building_id')\n",
    "        \n",
    "    test = test.rename(columns = {'total_price':'pred_{}'.format(idx)})\n",
    "    test[f'log_pred_{idx}'] = np.log1p(test[f'pred_{idx}'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>log_pred_3</th>\n",
       "      <th>pred_4</th>\n",
       "      <th>log_pred_4</th>\n",
       "      <th>pred_5</th>\n",
       "      <th>log_pred_5</th>\n",
       "      <th>pred_6</th>\n",
       "      <th>log_pred_6</th>\n",
       "      <th>pred_7</th>\n",
       "      <th>log_pred_7</th>\n",
       "      <th>pred_9</th>\n",
       "      <th>log_pred_9</th>\n",
       "      <th>total_price</th>\n",
       "      <th>log_total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jre1pJhcQj91Kdky</td>\n",
       "      <td>1.255350e+07</td>\n",
       "      <td>16.345510</td>\n",
       "      <td>1.255981e+07</td>\n",
       "      <td>16.346013</td>\n",
       "      <td>1.288381e+07</td>\n",
       "      <td>16.371482</td>\n",
       "      <td>1.282213e+07</td>\n",
       "      <td>16.366683</td>\n",
       "      <td>1.247484e+07</td>\n",
       "      <td>16.339224</td>\n",
       "      <td>12309220.0</td>\n",
       "      <td>16.325859</td>\n",
       "      <td>1.261616e+07</td>\n",
       "      <td>16.350489</td>\n",
       "      <td>14215011.0</td>\n",
       "      <td>16.469809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jcbuA8q3KPH9SzpS</td>\n",
       "      <td>9.117771e+06</td>\n",
       "      <td>16.025736</td>\n",
       "      <td>9.675385e+06</td>\n",
       "      <td>16.085096</td>\n",
       "      <td>7.095057e+06</td>\n",
       "      <td>15.774909</td>\n",
       "      <td>9.570172e+06</td>\n",
       "      <td>16.074162</td>\n",
       "      <td>9.175013e+06</td>\n",
       "      <td>16.031994</td>\n",
       "      <td>7909433.5</td>\n",
       "      <td>15.883567</td>\n",
       "      <td>9.376357e+06</td>\n",
       "      <td>16.053702</td>\n",
       "      <td>7642884.5</td>\n",
       "      <td>15.849286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EqWJpHmkXPyfHkB8</td>\n",
       "      <td>4.491672e+06</td>\n",
       "      <td>15.317736</td>\n",
       "      <td>4.511709e+06</td>\n",
       "      <td>15.322187</td>\n",
       "      <td>4.586149e+06</td>\n",
       "      <td>15.338551</td>\n",
       "      <td>4.687745e+06</td>\n",
       "      <td>15.360462</td>\n",
       "      <td>4.562418e+06</td>\n",
       "      <td>15.333364</td>\n",
       "      <td>4827711.0</td>\n",
       "      <td>15.389883</td>\n",
       "      <td>4.600447e+06</td>\n",
       "      <td>15.341664</td>\n",
       "      <td>4807053.0</td>\n",
       "      <td>15.385595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ihrMWGM8Nq99Uvu6</td>\n",
       "      <td>6.918995e+06</td>\n",
       "      <td>15.749781</td>\n",
       "      <td>6.878882e+06</td>\n",
       "      <td>15.743967</td>\n",
       "      <td>6.906435e+06</td>\n",
       "      <td>15.747964</td>\n",
       "      <td>6.981481e+06</td>\n",
       "      <td>15.758772</td>\n",
       "      <td>6.937283e+06</td>\n",
       "      <td>15.752421</td>\n",
       "      <td>7259765.5</td>\n",
       "      <td>15.797858</td>\n",
       "      <td>6.959559e+06</td>\n",
       "      <td>15.755627</td>\n",
       "      <td>7493026.5</td>\n",
       "      <td>15.829483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jRhwzoVMFLFzxAX1</td>\n",
       "      <td>2.750253e+06</td>\n",
       "      <td>14.827204</td>\n",
       "      <td>2.754684e+06</td>\n",
       "      <td>14.828814</td>\n",
       "      <td>2.725200e+06</td>\n",
       "      <td>14.818053</td>\n",
       "      <td>2.838090e+06</td>\n",
       "      <td>14.858642</td>\n",
       "      <td>2.766369e+06</td>\n",
       "      <td>14.833047</td>\n",
       "      <td>2949650.8</td>\n",
       "      <td>14.897198</td>\n",
       "      <td>2.727247e+06</td>\n",
       "      <td>14.818804</td>\n",
       "      <td>2846855.8</td>\n",
       "      <td>14.861726</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id        pred_1  log_pred_1        pred_3  log_pred_3  \\\n",
       "0  jre1pJhcQj91Kdky  1.255350e+07   16.345510  1.255981e+07   16.346013   \n",
       "1  jcbuA8q3KPH9SzpS  9.117771e+06   16.025736  9.675385e+06   16.085096   \n",
       "2  EqWJpHmkXPyfHkB8  4.491672e+06   15.317736  4.511709e+06   15.322187   \n",
       "3  ihrMWGM8Nq99Uvu6  6.918995e+06   15.749781  6.878882e+06   15.743967   \n",
       "4  jRhwzoVMFLFzxAX1  2.750253e+06   14.827204  2.754684e+06   14.828814   \n",
       "\n",
       "         pred_4  log_pred_4        pred_5  log_pred_5        pred_6  \\\n",
       "0  1.288381e+07   16.371482  1.282213e+07   16.366683  1.247484e+07   \n",
       "1  7.095057e+06   15.774909  9.570172e+06   16.074162  9.175013e+06   \n",
       "2  4.586149e+06   15.338551  4.687745e+06   15.360462  4.562418e+06   \n",
       "3  6.906435e+06   15.747964  6.981481e+06   15.758772  6.937283e+06   \n",
       "4  2.725200e+06   14.818053  2.838090e+06   14.858642  2.766369e+06   \n",
       "\n",
       "   log_pred_6      pred_7  log_pred_7        pred_9  log_pred_9  total_price  \\\n",
       "0   16.339224  12309220.0   16.325859  1.261616e+07   16.350489   14215011.0   \n",
       "1   16.031994   7909433.5   15.883567  9.376357e+06   16.053702    7642884.5   \n",
       "2   15.333364   4827711.0   15.389883  4.600447e+06   15.341664    4807053.0   \n",
       "3   15.752421   7259765.5   15.797858  6.959559e+06   15.755627    7493026.5   \n",
       "4   14.833047   2949650.8   14.897198  2.727247e+06   14.818804    2846855.8   \n",
       "\n",
       "   log_total_price  \n",
       "0        16.469809  \n",
       "1        15.849286  \n",
       "2        15.385595  \n",
       "3        15.829483  \n",
       "4        14.861726  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/model_selection/_split.py:2053: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:474: UserWarning: Coordinate descent with alpha=0 may lead to unexpected results and is discouraged.\n",
      "  tol, rng, random, positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:474: UserWarning: Coordinate descent with alpha=0 may lead to unexpected results and is discouraged.\n",
      "  tol, rng, random, positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:474: UserWarning: Coordinate descent with alpha=0 may lead to unexpected results and is discouraged.\n",
      "  tol, rng, random, positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LassoCV(alphas=[0, 0.0001, 0.0014677992676220704, 0.021544346900318846, 0.31622776601683794, 4.641588833612782, 68.12920690579622, 1000.0],\n",
       "    copy_X=True, cv='warn', eps=0.001, fit_intercept=True, max_iter=100000,\n",
       "    n_alphas=100, n_jobs=-1, normalize=False, positive=False,\n",
       "    precompute='auto', random_state=None, selection='cyclic', tol=1e-06,\n",
       "    verbose=False)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = cv[['log_pred_{}'.format(idx) for idx in idx_models]]\n",
    "y = cv['log_total_price']\n",
    "\n",
    "reg = LassoCV(alphas=[0]+list(np.logspace(-4, 3, 7)), max_iter=100000, tol=1e-6, n_jobs=-1)\n",
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0001\n",
      "[[1.3539862  1.37718519 1.34124855]\n",
      " [1.3539862  1.37718519 1.34124855]\n",
      " [1.3539862  1.37718519 1.34124855]\n",
      " [0.11255819 0.11704416 0.10972506]\n",
      " [0.03595422 0.03810006 0.03677494]\n",
      " [0.0355604  0.0376592  0.0365321 ]\n",
      " [0.03555714 0.03765261 0.03653896]\n",
      " [0.03554657 0.03764356 0.03659121]]\n",
      "[0.         0.26631984 0.01232962 0.27067851 0.2425683  0.15253216\n",
      " 0.06170947] -0.0940669161948744\n"
     ]
    }
   ],
   "source": [
    "print(reg.alpha_)\n",
    "print(reg.mse_path_)\n",
    "print(reg.coef_, reg.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:3: UserWarning: With alpha=0, this algorithm does not converge well. You are advised to use the LinearRegression estimator\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:478: UserWarning: Coordinate descent with no regularization may lead to unexpected results and is discouraged.\n",
      "  positive)\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/linear_model/coordinate_descent.py:492: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Fitting data with very small alpha may cause precision problems.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.14541984  0.28117963  0.01282322  0.27252229  0.36025286  0.15254549\n",
      "  0.07233539] -0.0940669161948744\n",
      "0.973104868100518\n",
      "[0.         0.26631984 0.01232962 0.27067851 0.2425683  0.15253216\n",
      " 0.06170947] -0.0940669161948744\n",
      "0.9731003429484877\n",
      "[0.         0.2671007  0.00984    0.26702789 0.2449908  0.15286434\n",
      " 0.06320916] -0.0940669161948744\n",
      "0.9730990644674576\n",
      "[0.         0.27713949 0.         0.21138691 0.26148732 0.15630291\n",
      " 0.08269349] -0.0940669161948744\n",
      "0.9728282062130117\n",
      "[0.         0.30945399 0.         0.         0.         0.16798424\n",
      " 0.28450805] -0.0940669161948744\n",
      "0.9167204673966535\n",
      "[0. 0. 0. 0. 0. 0. 0.] -0.0940669161948744\n",
      "0.0\n",
      "[0. 0. 0. 0. 0. 0. 0.] -0.0940669161948744\n",
      "0.0\n",
      "[0. 0. 0. 0. 0. 0. 0.] -0.0940669161948744\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "for a in [0]+list(np.logspace(-4, 3, 7)):\n",
    "    reg_single = Lasso(alpha=a, max_iter=100000, tol=1e-6)\n",
    "    reg_single.fit(X, y)\n",
    "    print(reg_single.coef_, reg.intercept_)\n",
    "    print(reg_single.score(X,y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==== Fold 1 ====\n",
      "alpha, score: 0 5916.872608190963\n",
      "alpha, score: 0.0001 5921.872585181543\n",
      "alpha, score: 0.0002 5923.87258670467\n",
      "alpha, score: 0.0005 5924.872590833311\n",
      "alpha, score: 0.0008 5923.872594222182\n",
      "alpha, score: 0.001 5925.872595987911\n",
      "alpha, score: 0.002 5924.872600715948\n",
      "alpha, score: 0.005 5923.8725656290935\n",
      "alpha, score: 0.008 5920.872442529715\n",
      "alpha, score: 0.01 5920.872307270934\n",
      "alpha, score: 0.02 5871.871156451196\n",
      "==== Fold 2 ====\n",
      "alpha, score: 0 5886.873201436377\n",
      "alpha, score: 0.0001 5880.873196686577\n",
      "alpha, score: 0.0002 5880.873200649211\n",
      "alpha, score: 0.0005 5881.873212132588\n",
      "alpha, score: 0.0008 5880.873222855416\n",
      "alpha, score: 0.001 5879.873229544782\n",
      "alpha, score: 0.002 5876.873258812387\n",
      "alpha, score: 0.005 5890.873295140916\n",
      "alpha, score: 0.008 5896.873247334499\n",
      "alpha, score: 0.01 5892.8731635482645\n",
      "alpha, score: 0.02 5861.872261113058\n",
      "==== Fold 3 ====\n",
      "alpha, score: 0 5911.87740133396\n",
      "alpha, score: 0.0001 5916.877419292948\n",
      "alpha, score: 0.0002 5918.877421004957\n",
      "alpha, score: 0.0005 5921.877425713785\n",
      "alpha, score: 0.0008 5921.87742979301\n",
      "alpha, score: 0.001 5920.877432147435\n",
      "alpha, score: 0.002 5914.877439210267\n",
      "alpha, score: 0.005 5906.877412554609\n",
      "alpha, score: 0.008 5915.8773018384645\n",
      "alpha, score: 0.01 5919.877176617054\n",
      "alpha, score: 0.02 5874.8760977511065\n",
      "[0.008, 5911.207663900893, [5920.872442529715, 5896.873247334499, 5915.8773018384645]]\n",
      "[0.01, 5911.207549145417, [5920.872307270934, 5892.8731635482645, 5919.877176617054]]\n",
      "[0.0005, 5909.541076226561, [5924.872590833311, 5881.873212132588, 5921.877425713785]]\n",
      "[0.001, 5908.874419226709, [5925.872595987911, 5879.873229544782, 5920.877432147435]]\n",
      "[0.0008, 5908.874415623536, [5923.872594222182, 5880.873222855416, 5921.87742979301]]\n",
      "[0.0002, 5907.874402786279, [5923.87258670467, 5880.873200649211, 5918.877421004957]]\n",
      "[0.005, 5907.207757774872, [5923.8725656290935, 5890.873295140916, 5906.877412554609]]\n",
      "[0.0001, 5906.54106705369, [5921.872585181543, 5880.873196686577, 5916.877419292948]]\n",
      "[0.002, 5905.541099579535, [5924.872600715948, 5876.873258812387, 5914.877439210267]]\n",
      "[0, 5905.2077369871, [5916.872608190963, 5886.873201436377, 5911.87740133396]]\n",
      "[0.02, 5869.539838438453, [5871.871156451196, 5861.872261113058, 5874.8760977511065]]\n"
     ]
    }
   ],
   "source": [
    "alphas = [0, 0.0001, 0.0002, 0.0005, 0.0008, 0.001, 0.002, 0.005, 0.008, 0.01, 0.02]\n",
    "gsearch = {}\n",
    "\n",
    "folds = KFold(n_splits=3, shuffle=True, random_state=1208)\n",
    "for i_fold, (itrain, ival) in enumerate(folds.split(X)): # kfold\n",
    "    print('==== Fold', i_fold+1, '====')\n",
    "    \n",
    "    # split train, val\n",
    "    X_train = X.iloc[itrain]\n",
    "    X_val = X.iloc[ival]\n",
    "    y_train = y.iloc[itrain]\n",
    "    y_val = y.iloc[ival]\n",
    "    \n",
    "    # random sample - grid search\n",
    "    for a in alphas:\n",
    "        if a == 0:\n",
    "            reg_single = LinearRegression()\n",
    "        else:\n",
    "            reg_single = Lasso(alpha=a, max_iter=100000, tol=1e-6)\n",
    "        reg_single.fit(X_train, y_train)\n",
    "        \n",
    "        y_pred = reg_single.predict(X_val)\n",
    "        y_pred_final = np.expm1(y_pred)\n",
    "        y_true_final = np.expm1(y_val)\n",
    "        score = cal_score(y_true_final, y_pred_final)\n",
    "\n",
    "        print('alpha, score:', a, score)\n",
    "        gsearch[a] = gsearch.get(a,[]) + [score]\n",
    "\n",
    "results = [[key, np.mean(value), value] for key, value, in gsearch.items()]\n",
    "results.sort(key= lambda x: x[1], reverse=True)\n",
    "for item in results:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.008\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Lasso(alpha=0.008, copy_X=True, fit_intercept=True, max_iter=1000000,\n",
       "   normalize=False, positive=False, precompute=False, random_state=None,\n",
       "   selection='cyclic', tol=1e-06, warm_start=False)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha_set = results[0][0]\n",
    "print(alpha_set)\n",
    "if alpha_set == 0:\n",
    "    reg = LinearRegression()\n",
    "else:\n",
    "    reg = Lasso(alpha=alpha_set, max_iter=1000000, tol=1e-6)\n",
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.27063769 0.         0.24943032 0.25503673 0.15434008\n",
      " 0.07032579] 0.0047482405682934825\n"
     ]
    }
   ],
   "source": [
    "print(reg.coef_, reg.intercept_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate cv score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_pred_final = np.zeros(X.shape[0])\n",
    "for i, col in enumerate(X):\n",
    "    cv_pred_final = cv_pred_final + X[col] * reg.coef_[i]\n",
    "cv_pred_final = cv_pred_final + reg.intercept_\n",
    "cv_pred_final = np.expm1(cv_pred_final)\n",
    "cv_true_final = np.expm1(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14215011.0</td>\n",
       "      <td>1.258027e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7642884.5</td>\n",
       "      <td>9.217113e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4807053.0</td>\n",
       "      <td>4.628002e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7493026.5</td>\n",
       "      <td>6.990691e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2846855.8</td>\n",
       "      <td>2.809537e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            a             b\n",
       "0  14215011.0  1.258027e+07\n",
       "1   7642884.5  9.217113e+06\n",
       "2   4807053.0  4.628002e+06\n",
       "3   7493026.5  6.990691e+06\n",
       "4   2846855.8  2.809537e+06"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'a':cv_true_final,'b':cv_pred_final}).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5905.874376386471"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cal_score(cv_true_final, cv_pred_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pred_final = pd.DataFrame({'building_id': test['building_id'], 'total_price': np.zeros(test.shape[0])})\n",
    "for i, idx in enumerate(idx_models):\n",
    "    test_pred_final['total_price'] = test_pred_final['total_price'] + test[f'log_pred_{idx}'] * reg.coef_[i]\n",
    "test_pred_final['total_price'] = test_pred_final['total_price'] + reg.intercept_\n",
    "test_pred_final['total_price'] = np.expm1(test_pred_final['total_price'])\n",
    "test_pred_final['total_price'] = np.clip(test_pred_final['total_price'], 0, None)\n",
    "\n",
    "test_pred_final.to_csv('output/stack_{}_{}.csv'.format(stack_idx, models), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a= pd.read_csv('output/model-03-lgb-feats-selection-cv.csv')\n",
    "#b= pd.read_csv('output/model-03-lgb-feats-selection-test-one.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a.rename(columns={'building_id':'id','total_price_predict':'target'}).to_csv('opt-pred3.csv',index=False)\n",
    "#b.rename(columns={'building_id':'id','total_price':'target'}).to_csv('opt-test3.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}