stack_09_1-11.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import Lasso, LassoCV, LinearRegression\n",
    "from sklearn.model_selection import KFold\n",
    "from utilities import cal_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack_idx = '09'\n",
    "models = '1-11'\n",
    "use_test_kfold = set([2,7,8])\n",
    "\n",
    "is_per_area = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read CV predictions and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_models(exp):\n",
    "    exp_split = exp.split(',')\n",
    "    idx_models = []\n",
    "    for e in exp_split:\n",
    "        if '-' in e:\n",
    "            n0, n1 = e.split('-')\n",
    "            idx_models.extend(list(range(int(n0), int(n1)+1, 1)))\n",
    "        else:\n",
    "            idx_models.append(int(e))\n",
    "    return idx_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_models = parse_models(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_in_output = [f for f in os.listdir('output/') if os.path.isfile('output/'+f)]\n",
    "\n",
    "files_cv = {idx: [f for f in files_in_output if 'model-%02d-' % idx in f and 'cv' in f][0] for idx in idx_models}\n",
    "\n",
    "files_test_one = {idx: [f for f in files_in_output if 'model-%02d-' % idx in f and 'test-one' in f][0] \\\n",
    "                  for idx in idx_models}\n",
    "files_test_kf = {idx: [f for f in files_in_output if 'model-%02d-' % idx in f and 'test-kfold' in f][0] \\\n",
    "                 for idx in idx_models}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 'model-01-lgb-cv.csv', 2: 'model-02-keras-search-cv.csv', 3: 'model-03-lgb-feats-selection-cv.csv', 4: 'model-04-lgb-PCA-cv.csv', 5: 'model-05-lgb-wo-per-area-cv.csv', 6: 'model-06-lgb-lr0.001-cv.csv', 7: 'model-07-keras-embedding-cv.csv', 8: 'model-08-keras-search-long-cv.csv', 9: 'model-09-lgb-feats-selection-75-cv.csv', 10: 'model-10-lgb-feats-selection-75-lr-0.001-cv.csv', 11: 'model-11-rf-cv.csv'}\n",
      "{1: 'model-01-lgb-test-kfold.csv', 2: 'model-02-keras-search-test-kfold.csv', 3: 'model-03-lgb-feats-selection-test-kfold.csv', 4: 'model-04-lgb-PCA-test-kfold.csv', 5: 'model-05-lgb-wo-per-area-test-kfold.csv', 6: 'model-06-lgb-lr0.001-test-kfold.csv', 7: 'model-07-keras-embedding-test-kfold.csv', 8: 'model-08-keras-search-long-test-kfold.csv', 9: 'model-09-lgb-feats-selection-75-test-kfold.csv', 10: 'model-10-lgb-feats-selection-75-lr-0.001-test-kfold.csv', 11: 'model-11-rf-test-kfold.csv'}\n",
      "{1: 'model-01-lgb-test-one.csv', 2: 'model-02-keras-search-test-one.csv', 3: 'model-03-lgb-feats-selection-test-one.csv', 4: 'model-04-lgb-PCA-test-one.csv', 5: 'model-05-lgb-wo-per-area-test-one.csv', 6: 'model-06-lgb-lr0.001-test-one.csv', 7: 'model-07-keras-embedding-test-one.csv', 8: 'model-08-keras-search-long-test-one.csv', 9: 'model-09-lgb-feats-selection-75-test-one.csv', 10: 'model-10-lgb-feats-selection-75-lr-0.001-test-one.csv', 11: 'model-11-rf-test-one.csv'}\n"
     ]
    }
   ],
   "source": [
    "print(files_cv)\n",
    "print(files_test_kf)\n",
    "print(files_test_one)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('dataset/train.csv')\n",
    "df_test = pd.read_csv('dataset/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV predictions:\n",
      "No. 0 file: model-01-lgb-cv.csv\n",
      "No. 1 file: model-02-keras-search-cv.csv\n",
      "No. 2 file: model-03-lgb-feats-selection-cv.csv\n",
      "No. 3 file: model-04-lgb-PCA-cv.csv\n",
      "No. 4 file: model-05-lgb-wo-per-area-cv.csv\n",
      "No. 5 file: model-06-lgb-lr0.001-cv.csv\n",
      "No. 6 file: model-07-keras-embedding-cv.csv\n",
      "No. 7 file: model-08-keras-search-long-cv.csv\n",
      "No. 8 file: model-09-lgb-feats-selection-75-cv.csv\n",
      "No. 9 file: model-10-lgb-feats-selection-75-lr-0.001-cv.csv\n",
      "No. 10 file: model-11-rf-cv.csv\n",
      "Test predictions:\n",
      "No. 0 file: model-01-lgb-test-one.csv\n",
      "No. 1 file: model-02-keras-search-test-kfold.csv\n",
      "No. 2 file: model-03-lgb-feats-selection-test-one.csv\n",
      "No. 3 file: model-04-lgb-PCA-test-one.csv\n",
      "No. 4 file: model-05-lgb-wo-per-area-test-one.csv\n",
      "No. 5 file: model-06-lgb-lr0.001-test-one.csv\n",
      "No. 6 file: model-07-keras-embedding-test-kfold.csv\n",
      "No. 7 file: model-08-keras-search-long-test-kfold.csv\n",
      "No. 8 file: model-09-lgb-feats-selection-75-test-one.csv\n",
      "No. 9 file: model-10-lgb-feats-selection-75-lr-0.001-test-one.csv\n",
      "No. 10 file: model-11-rf-test-one.csv\n"
     ]
    }
   ],
   "source": [
    "cv = df_train[['building_id', 'building_area']]\n",
    "test = df_test[['building_id', 'building_area']]\n",
    "\n",
    "print('CV predictions:')\n",
    "for i, idx in enumerate(idx_models):\n",
    "    f = files_cv[idx]\n",
    "    print('No. {} file: {}'.format(i, f))\n",
    "    df = pd.read_csv('output/'+f)\n",
    "    \n",
    "    cv = pd.merge(cv, df[['building_id','total_price_predict']], on='building_id')\n",
    "    \n",
    "    cv = cv.rename(columns = {'total_price_predict':'pred_{}'.format(idx_models[i])})\n",
    "    cv[f'log_pred_{idx_models[i]}'] = np.log1p(cv[f'pred_{idx_models[i]}'])\n",
    "    cv[f'log_parea_pred_{idx_models[i]}'] = np.log1p( cv[f'pred_{idx_models[i]}'] / cv['building_area'] )\n",
    "\n",
    "cv = pd.merge(cv, df[['building_id','total_price']], on='building_id')\n",
    "cv['log_total_price'] = np.log1p(cv['total_price'])\n",
    "cv['log_parea_total_price'] = np.log1p( cv['total_price'] / cv['building_area'] )\n",
    "\n",
    "print('Test predictions:')\n",
    "for i, idx in enumerate(idx_models):\n",
    "    f = files_test_kf[idx] if idx in use_test_kfold else files_test_one[idx]\n",
    "    print('No. {} file: {}'.format(i, f))\n",
    "    df = pd.read_csv('output/'+f)\n",
    "\n",
    "    test = pd.merge(test, df[['building_id','total_price']], on='building_id')\n",
    "        \n",
    "    test = test.rename(columns = {'total_price':'pred_{}'.format(idx)})\n",
    "    test[f'log_pred_{idx}'] = np.log1p(test[f'pred_{idx}'])\n",
    "    test[f'log_parea_pred_{idx_models[i]}'] = np.log1p( test[f'pred_{idx_models[i]}'] / test['building_area'] )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>building_area</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>log_parea_pred_1</th>\n",
       "      <th>pred_2</th>\n",
       "      <th>log_pred_2</th>\n",
       "      <th>log_parea_pred_2</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>log_pred_3</th>\n",
       "      <th>...</th>\n",
       "      <th>log_parea_pred_9</th>\n",
       "      <th>pred_10</th>\n",
       "      <th>log_pred_10</th>\n",
       "      <th>log_parea_pred_10</th>\n",
       "      <th>pred_11</th>\n",
       "      <th>log_pred_11</th>\n",
       "      <th>log_parea_pred_11</th>\n",
       "      <th>total_price</th>\n",
       "      <th>log_total_price</th>\n",
       "      <th>log_parea_total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e3mMIMR3JJqCaXz1</td>\n",
       "      <td>3.418175</td>\n",
       "      <td>6.331552e+05</td>\n",
       "      <td>13.358472</td>\n",
       "      <td>12.129369</td>\n",
       "      <td>717209.94</td>\n",
       "      <td>13.483125</td>\n",
       "      <td>12.254022</td>\n",
       "      <td>6.656891e+05</td>\n",
       "      <td>13.408579</td>\n",
       "      <td>...</td>\n",
       "      <td>12.161329</td>\n",
       "      <td>6.666997e+05</td>\n",
       "      <td>13.410097</td>\n",
       "      <td>12.180993</td>\n",
       "      <td>1.032484e+06</td>\n",
       "      <td>13.847480</td>\n",
       "      <td>12.618375</td>\n",
       "      <td>647603.75</td>\n",
       "      <td>13.381036</td>\n",
       "      <td>12.151933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LgwzgklNvy4QCtq5</td>\n",
       "      <td>4.041309</td>\n",
       "      <td>3.064324e+06</td>\n",
       "      <td>14.935338</td>\n",
       "      <td>13.538770</td>\n",
       "      <td>2899842.20</td>\n",
       "      <td>14.880167</td>\n",
       "      <td>13.483600</td>\n",
       "      <td>3.079196e+06</td>\n",
       "      <td>14.940179</td>\n",
       "      <td>...</td>\n",
       "      <td>13.553184</td>\n",
       "      <td>3.161088e+06</td>\n",
       "      <td>14.966427</td>\n",
       "      <td>13.569860</td>\n",
       "      <td>3.156863e+06</td>\n",
       "      <td>14.965090</td>\n",
       "      <td>13.568522</td>\n",
       "      <td>3321452.00</td>\n",
       "      <td>15.015913</td>\n",
       "      <td>13.619345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ucIR2NLLsC3T650L</td>\n",
       "      <td>5.584279</td>\n",
       "      <td>9.827776e+06</td>\n",
       "      <td>16.100723</td>\n",
       "      <td>14.380769</td>\n",
       "      <td>9766813.00</td>\n",
       "      <td>16.094501</td>\n",
       "      <td>14.374546</td>\n",
       "      <td>9.814852e+06</td>\n",
       "      <td>16.099407</td>\n",
       "      <td>...</td>\n",
       "      <td>14.381977</td>\n",
       "      <td>9.737051e+06</td>\n",
       "      <td>16.091449</td>\n",
       "      <td>14.371494</td>\n",
       "      <td>9.333706e+06</td>\n",
       "      <td>16.049143</td>\n",
       "      <td>14.329188</td>\n",
       "      <td>9570885.00</td>\n",
       "      <td>16.074236</td>\n",
       "      <td>14.354282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jre1pJhcQj91Kdky</td>\n",
       "      <td>13.563031</td>\n",
       "      <td>1.255350e+07</td>\n",
       "      <td>16.345510</td>\n",
       "      <td>13.738164</td>\n",
       "      <td>12699800.00</td>\n",
       "      <td>16.357097</td>\n",
       "      <td>13.749750</td>\n",
       "      <td>1.255981e+07</td>\n",
       "      <td>16.346013</td>\n",
       "      <td>...</td>\n",
       "      <td>13.743142</td>\n",
       "      <td>1.269176e+07</td>\n",
       "      <td>16.356463</td>\n",
       "      <td>13.749117</td>\n",
       "      <td>1.200650e+07</td>\n",
       "      <td>16.300959</td>\n",
       "      <td>13.693612</td>\n",
       "      <td>14215011.00</td>\n",
       "      <td>16.469809</td>\n",
       "      <td>13.862462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rQpYpY9nRG7X5mmr</td>\n",
       "      <td>4.688108</td>\n",
       "      <td>1.215194e+06</td>\n",
       "      <td>14.010415</td>\n",
       "      <td>12.465389</td>\n",
       "      <td>2012610.50</td>\n",
       "      <td>14.514944</td>\n",
       "      <td>12.969916</td>\n",
       "      <td>1.128419e+06</td>\n",
       "      <td>13.936329</td>\n",
       "      <td>...</td>\n",
       "      <td>12.397690</td>\n",
       "      <td>1.122204e+06</td>\n",
       "      <td>13.930806</td>\n",
       "      <td>12.385780</td>\n",
       "      <td>9.735087e+05</td>\n",
       "      <td>13.788663</td>\n",
       "      <td>12.243638</td>\n",
       "      <td>762712.00</td>\n",
       "      <td>13.544637</td>\n",
       "      <td>11.999613</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id  building_area        pred_1  log_pred_1  \\\n",
       "0  e3mMIMR3JJqCaXz1       3.418175  6.331552e+05   13.358472   \n",
       "1  LgwzgklNvy4QCtq5       4.041309  3.064324e+06   14.935338   \n",
       "2  ucIR2NLLsC3T650L       5.584279  9.827776e+06   16.100723   \n",
       "3  jre1pJhcQj91Kdky      13.563031  1.255350e+07   16.345510   \n",
       "4  rQpYpY9nRG7X5mmr       4.688108  1.215194e+06   14.010415   \n",
       "\n",
       "   log_parea_pred_1       pred_2  log_pred_2  log_parea_pred_2        pred_3  \\\n",
       "0         12.129369    717209.94   13.483125         12.254022  6.656891e+05   \n",
       "1         13.538770   2899842.20   14.880167         13.483600  3.079196e+06   \n",
       "2         14.380769   9766813.00   16.094501         14.374546  9.814852e+06   \n",
       "3         13.738164  12699800.00   16.357097         13.749750  1.255981e+07   \n",
       "4         12.465389   2012610.50   14.514944         12.969916  1.128419e+06   \n",
       "\n",
       "   log_pred_3  ...  log_parea_pred_9       pred_10  log_pred_10  \\\n",
       "0   13.408579  ...         12.161329  6.666997e+05    13.410097   \n",
       "1   14.940179  ...         13.553184  3.161088e+06    14.966427   \n",
       "2   16.099407  ...         14.381977  9.737051e+06    16.091449   \n",
       "3   16.346013  ...         13.743142  1.269176e+07    16.356463   \n",
       "4   13.936329  ...         12.397690  1.122204e+06    13.930806   \n",
       "\n",
       "   log_parea_pred_10       pred_11  log_pred_11  log_parea_pred_11  \\\n",
       "0          12.180993  1.032484e+06    13.847480          12.618375   \n",
       "1          13.569860  3.156863e+06    14.965090          13.568522   \n",
       "2          14.371494  9.333706e+06    16.049143          14.329188   \n",
       "3          13.749117  1.200650e+07    16.300959          13.693612   \n",
       "4          12.385780  9.735087e+05    13.788663          12.243638   \n",
       "\n",
       "   total_price  log_total_price  log_parea_total_price  \n",
       "0    647603.75        13.381036              12.151933  \n",
       "1   3321452.00        15.015913              13.619345  \n",
       "2   9570885.00        16.074236              14.354282  \n",
       "3  14215011.00        16.469809              13.862462  \n",
       "4    762712.00        13.544637              11.999613  \n",
       "\n",
       "[5 rows x 38 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>building_area</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>log_parea_pred_1</th>\n",
       "      <th>pred_2</th>\n",
       "      <th>log_pred_2</th>\n",
       "      <th>log_parea_pred_2</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>log_pred_3</th>\n",
       "      <th>...</th>\n",
       "      <th>log_parea_pred_8</th>\n",
       "      <th>pred_9</th>\n",
       "      <th>log_pred_9</th>\n",
       "      <th>log_parea_pred_9</th>\n",
       "      <th>pred_10</th>\n",
       "      <th>log_pred_10</th>\n",
       "      <th>log_parea_pred_10</th>\n",
       "      <th>pred_11</th>\n",
       "      <th>log_pred_11</th>\n",
       "      <th>log_parea_pred_11</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>3.418175</td>\n",
       "      <td>1.526912e+07</td>\n",
       "      <td>16.541343</td>\n",
       "      <td>15.312236</td>\n",
       "      <td>12470072.0</td>\n",
       "      <td>16.338842</td>\n",
       "      <td>15.109735</td>\n",
       "      <td>1.531685e+07</td>\n",
       "      <td>16.544464</td>\n",
       "      <td>...</td>\n",
       "      <td>15.143558</td>\n",
       "      <td>1.468227e+07</td>\n",
       "      <td>16.502151</td>\n",
       "      <td>15.273045</td>\n",
       "      <td>1.427889e+07</td>\n",
       "      <td>16.474293</td>\n",
       "      <td>15.245186</td>\n",
       "      <td>1.084948e+07</td>\n",
       "      <td>16.199628</td>\n",
       "      <td>14.970521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>7.726227</td>\n",
       "      <td>3.924241e+06</td>\n",
       "      <td>15.182684</td>\n",
       "      <td>13.138065</td>\n",
       "      <td>3916552.2</td>\n",
       "      <td>15.180723</td>\n",
       "      <td>13.136104</td>\n",
       "      <td>3.977095e+06</td>\n",
       "      <td>15.196062</td>\n",
       "      <td>...</td>\n",
       "      <td>13.140531</td>\n",
       "      <td>3.899813e+06</td>\n",
       "      <td>15.176439</td>\n",
       "      <td>13.131820</td>\n",
       "      <td>3.920610e+06</td>\n",
       "      <td>15.181758</td>\n",
       "      <td>13.137139</td>\n",
       "      <td>3.840545e+06</td>\n",
       "      <td>15.161125</td>\n",
       "      <td>13.116506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>12.170581</td>\n",
       "      <td>1.096127e+07</td>\n",
       "      <td>16.209879</td>\n",
       "      <td>13.710858</td>\n",
       "      <td>11912735.0</td>\n",
       "      <td>16.293119</td>\n",
       "      <td>13.794098</td>\n",
       "      <td>1.084967e+07</td>\n",
       "      <td>16.199646</td>\n",
       "      <td>...</td>\n",
       "      <td>13.777643</td>\n",
       "      <td>1.118557e+07</td>\n",
       "      <td>16.230135</td>\n",
       "      <td>13.731115</td>\n",
       "      <td>1.067153e+07</td>\n",
       "      <td>16.183090</td>\n",
       "      <td>13.684070</td>\n",
       "      <td>8.393805e+06</td>\n",
       "      <td>15.943005</td>\n",
       "      <td>13.443984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>2.252256</td>\n",
       "      <td>6.155550e+06</td>\n",
       "      <td>15.632865</td>\n",
       "      <td>14.820933</td>\n",
       "      <td>5940670.0</td>\n",
       "      <td>15.597333</td>\n",
       "      <td>14.785400</td>\n",
       "      <td>6.015238e+06</td>\n",
       "      <td>15.609807</td>\n",
       "      <td>...</td>\n",
       "      <td>14.794851</td>\n",
       "      <td>6.070453e+06</td>\n",
       "      <td>15.618944</td>\n",
       "      <td>14.807012</td>\n",
       "      <td>6.082243e+06</td>\n",
       "      <td>15.620884</td>\n",
       "      <td>14.808952</td>\n",
       "      <td>5.798727e+06</td>\n",
       "      <td>15.573149</td>\n",
       "      <td>14.761217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>5.813985</td>\n",
       "      <td>1.062995e+06</td>\n",
       "      <td>13.876602</td>\n",
       "      <td>12.116340</td>\n",
       "      <td>1088488.1</td>\n",
       "      <td>13.900301</td>\n",
       "      <td>12.140039</td>\n",
       "      <td>1.027248e+06</td>\n",
       "      <td>13.842395</td>\n",
       "      <td>...</td>\n",
       "      <td>12.127129</td>\n",
       "      <td>1.050322e+06</td>\n",
       "      <td>13.864608</td>\n",
       "      <td>12.104347</td>\n",
       "      <td>1.059463e+06</td>\n",
       "      <td>13.873273</td>\n",
       "      <td>12.113012</td>\n",
       "      <td>1.017048e+06</td>\n",
       "      <td>13.832416</td>\n",
       "      <td>12.072154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id  building_area        pred_1  log_pred_1  \\\n",
       "0  X5gsdTWGS3W7JJQB       3.418175  1.526912e+07   16.541343   \n",
       "1  BTshNOJyKHnT2YIT       7.726227  3.924241e+06   15.182684   \n",
       "2  dhdymr0lV8N5kZOT      12.170581  1.096127e+07   16.209879   \n",
       "3  VEwyGGMcD56w5BOc       2.252256  6.155550e+06   15.632865   \n",
       "4  wmUeMoJZfsqaSX9b       5.813985  1.062995e+06   13.876602   \n",
       "\n",
       "   log_parea_pred_1      pred_2  log_pred_2  log_parea_pred_2        pred_3  \\\n",
       "0         15.312236  12470072.0   16.338842         15.109735  1.531685e+07   \n",
       "1         13.138065   3916552.2   15.180723         13.136104  3.977095e+06   \n",
       "2         13.710858  11912735.0   16.293119         13.794098  1.084967e+07   \n",
       "3         14.820933   5940670.0   15.597333         14.785400  6.015238e+06   \n",
       "4         12.116340   1088488.1   13.900301         12.140039  1.027248e+06   \n",
       "\n",
       "   log_pred_3  ...  log_parea_pred_8        pred_9  log_pred_9  \\\n",
       "0   16.544464  ...         15.143558  1.468227e+07   16.502151   \n",
       "1   15.196062  ...         13.140531  3.899813e+06   15.176439   \n",
       "2   16.199646  ...         13.777643  1.118557e+07   16.230135   \n",
       "3   15.609807  ...         14.794851  6.070453e+06   15.618944   \n",
       "4   13.842395  ...         12.127129  1.050322e+06   13.864608   \n",
       "\n",
       "   log_parea_pred_9       pred_10  log_pred_10  log_parea_pred_10  \\\n",
       "0         15.273045  1.427889e+07    16.474293          15.245186   \n",
       "1         13.131820  3.920610e+06    15.181758          13.137139   \n",
       "2         13.731115  1.067153e+07    16.183090          13.684070   \n",
       "3         14.807012  6.082243e+06    15.620884          14.808952   \n",
       "4         12.104347  1.059463e+06    13.873273          12.113012   \n",
       "\n",
       "        pred_11  log_pred_11  log_parea_pred_11  \n",
       "0  1.084948e+07    16.199628          14.970521  \n",
       "1  3.840545e+06    15.161125          13.116506  \n",
       "2  8.393805e+06    15.943005          13.443984  \n",
       "3  5.798727e+06    15.573149          14.761217  \n",
       "4  1.017048e+06    13.832416          12.072154  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make Xy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "if is_per_area:\n",
    "    X = cv[['log_parea_pred_{}'.format(idx) for idx in idx_models]]\n",
    "else:\n",
    "    X = cv[['log_pred_{}'.format(idx) for idx in idx_models]]\n",
    "\n",
    "if is_per_area:\n",
    "    y = cv['log_parea_total_price']\n",
    "else:\n",
    "    y = cv['log_total_price']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### start regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reg = LassoCV(alphas=[0]+list(np.logspace(-4, 3, 7)), max_iter=100000, tol=1e-6, n_jobs=-1)\n",
    "#reg.fit(X, y)\n",
    "\n",
    "#print(reg.alpha_)\n",
    "#print(reg.mse_path_)\n",
    "#print(reg.coef_, reg.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for a in [0]+list(np.logspace(-4, 3, 7)):\n",
    "#    reg_single = Lasso(alpha=a, max_iter=100000, tol=1e-6)\n",
    "#    reg_single.fit(X, y)\n",
    "#    print(reg_single.coef_, reg.intercept_)\n",
    "#    print(reg_single.score(X,y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==== Fold 1 ====\n",
      "alpha, score: 0 5974.87492255022\n",
      "alpha, score: 0.0001 5976.874969829337\n",
      "alpha, score: 0.0002 5970.8749786336175\n",
      "alpha, score: 0.0005 5974.874905619782\n",
      "alpha, score: 0.0008 5965.874707742936\n",
      "alpha, score: 0.001 5968.874691042386\n",
      "alpha, score: 0.002 5976.874700053724\n",
      "alpha, score: 0.005 5968.874679048809\n",
      "alpha, score: 0.008 5978.874574262472\n",
      "alpha, score: 0.01 5984.874455401696\n",
      "alpha, score: 0.02 5955.873392552923\n",
      "==== Fold 2 ====\n",
      "alpha, score: 0 5930.875510010007\n",
      "alpha, score: 0.0001 5936.875607795234\n",
      "alpha, score: 0.0002 5936.87564695572\n",
      "alpha, score: 0.0005 5942.875683280254\n",
      "alpha, score: 0.0008 5922.875602552039\n",
      "alpha, score: 0.001 5923.8755405045595\n",
      "alpha, score: 0.002 5924.875565174514\n",
      "alpha, score: 0.005 5938.875579421313\n",
      "alpha, score: 0.008 5935.875508394832\n",
      "alpha, score: 0.01 5937.87541544477\n",
      "alpha, score: 0.02 5899.8744747752335\n",
      "==== Fold 3 ====\n",
      "alpha, score: 0 5892.87448312699\n",
      "alpha, score: 0.0001 5896.874563931169\n",
      "alpha, score: 0.0002 5893.874587235169\n",
      "alpha, score: 0.0005 5891.874586467914\n",
      "alpha, score: 0.0008 5879.874441698092\n",
      "alpha, score: 0.001 5874.874378354397\n",
      "alpha, score: 0.002 5876.874382972452\n",
      "alpha, score: 0.005 5873.874346974211\n",
      "alpha, score: 0.008 5874.874224629921\n",
      "alpha, score: 0.01 5867.874088318114\n",
      "alpha, score: 0.02 5832.8729448071745\n",
      "[0.0001, 5936.875047185247, [5976.874969829337, 5936.875607795234, 5896.874563931169]]\n",
      "[0.0005, 5936.54172512265, [5974.874905619782, 5942.875683280254, 5891.874586467914]]\n",
      "[0.0002, 5933.875070941503, [5970.8749786336175, 5936.87564695572, 5893.874587235169]]\n",
      "[0, 5932.87497189574, [5974.87492255022, 5930.875510010007, 5892.87448312699]]\n",
      "[0.01, 5930.207986388193, [5984.874455401696, 5937.87541544477, 5867.874088318114]]\n",
      "[0.008, 5929.874769095742, [5978.874574262472, 5935.875508394832, 5874.874224629921]]\n",
      "[0.005, 5927.208201814778, [5968.874679048809, 5938.875579421313, 5873.874346974211]]\n",
      "[0.002, 5926.208216066897, [5976.874700053724, 5924.875565174514, 5876.874382972452]]\n",
      "[0.0008, 5922.874917331022, [5965.874707742936, 5922.875602552039, 5879.874441698092]]\n",
      "[0.001, 5922.541536633781, [5968.874691042386, 5923.8755405045595, 5874.874378354397]]\n",
      "[0.02, 5896.206937378443, [5955.873392552923, 5899.8744747752335, 5832.8729448071745]]\n"
     ]
    }
   ],
   "source": [
    "alphas = [0, 0.0001, 0.0002, 0.0005, 0.0008, 0.001, 0.002, 0.005, 0.008, 0.01, 0.02]\n",
    "gsearch = {}\n",
    "\n",
    "folds = KFold(n_splits=3, shuffle=True, random_state=1208)\n",
    "for i_fold, (itrain, ival) in enumerate(folds.split(X)): # kfold\n",
    "    print('==== Fold', i_fold+1, '====')\n",
    "    \n",
    "    # split train, val\n",
    "    X_train = X.iloc[itrain]\n",
    "    X_val = X.iloc[ival]\n",
    "    y_train = y.iloc[itrain]\n",
    "    y_val = y.iloc[ival]\n",
    "    \n",
    "    # random sample - grid search\n",
    "    for a in alphas:\n",
    "        if a == 0:\n",
    "            reg_single = LinearRegression()\n",
    "        else:\n",
    "            reg_single = Lasso(alpha=a, max_iter=100000, tol=1e-6)\n",
    "        reg_single.fit(X_train, y_train)\n",
    "        \n",
    "        y_pred = reg_single.predict(X_val)\n",
    "        if is_per_area:\n",
    "            y_pred_final = np.expm1(y_pred) * cv.iloc[ival]['building_area']\n",
    "            y_true_final = np.expm1(y_val) * cv.iloc[ival]['building_area']\n",
    "        else:\n",
    "            y_pred_final = np.expm1(y_pred)\n",
    "            y_true_final = np.expm1(y_val)\n",
    "        score = cal_score(y_true_final, y_pred_final)\n",
    "\n",
    "        print('alpha, score:', a, score)\n",
    "        gsearch[a] = gsearch.get(a,[]) + [score]\n",
    "\n",
    "results = [[key, np.mean(value), value] for key, value, in gsearch.items()]\n",
    "results.sort(key= lambda x: x[1], reverse=True)\n",
    "for item in results:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0001\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Lasso(alpha=0.0001, copy_X=True, fit_intercept=True, max_iter=1000000,\n",
       "   normalize=False, positive=False, precompute=False, random_state=None,\n",
       "   selection='cyclic', tol=1e-06, warm_start=False)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha_set = results[0][0]\n",
    "print(alpha_set)\n",
    "if alpha_set == 0:\n",
    "    reg = LinearRegression()\n",
    "else:\n",
    "    reg = Lasso(alpha=alpha_set, max_iter=1000000, tol=1e-6)\n",
    "reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.          0.05607782  0.09983258  0.02023411  0.23176893  0.\n",
      "  0.10554324  0.04674615  0.          0.54793765 -0.10495695] -0.047580354534654035\n"
     ]
    }
   ],
   "source": [
    "print(reg.coef_, reg.intercept_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate cv score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_pred_final = np.zeros(X.shape[0])\n",
    "for i, col in enumerate(X):\n",
    "    cv_pred_final = cv_pred_final + X[col] * reg.coef_[i]\n",
    "cv_pred_final = cv_pred_final + reg.intercept_\n",
    "\n",
    "if is_per_area:\n",
    "    cv_pred_final = np.expm1(cv_pred_final) * cv['building_area']\n",
    "    cv_true_final = np.expm1(y) * cv['building_area']\n",
    "else:\n",
    "    cv_pred_final = np.expm1(cv_pred_final)\n",
    "    cv_true_final = np.expm1(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>647603.75</td>\n",
       "      <td>6.341272e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3321452.00</td>\n",
       "      <td>3.130435e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9570885.00</td>\n",
       "      <td>9.784449e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14215011.00</td>\n",
       "      <td>1.281375e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>762712.00</td>\n",
       "      <td>1.268092e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             a             b\n",
       "0    647603.75  6.341272e+05\n",
       "1   3321452.00  3.130435e+06\n",
       "2   9570885.00  9.784449e+06\n",
       "3  14215011.00  1.281375e+07\n",
       "4    762712.00  1.268092e+06"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'a':cv_true_final,'b':cv_pred_final}).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5940.875212617113"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cal_score(cv_true_final, cv_pred_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "if is_per_area:\n",
    "    col_prefix = 'log_parea_pred'\n",
    "else:\n",
    "    col_prefix = 'log_pred'\n",
    "\n",
    "test_pred_final = pd.DataFrame({'building_id': test['building_id'], 'total_price': np.zeros(test.shape[0])})\n",
    "\n",
    "for i, idx in enumerate(idx_models):\n",
    "    test_pred_final['total_price'] = test_pred_final['total_price'] + test[f'{col_prefix}_{idx}'] * reg.coef_[i]\n",
    "test_pred_final['total_price'] = test_pred_final['total_price'] + reg.intercept_\n",
    "\n",
    "if is_per_area:\n",
    "    test_pred_final['total_price'] = np.expm1(test_pred_final['total_price']) * test['building_area'] \n",
    "else:\n",
    "    test_pred_final['total_price'] = np.expm1(test_pred_final['total_price'])\n",
    "    \n",
    "test_pred_final['total_price'] = np.clip(test_pred_final['total_price'], 0, None)\n",
    "\n",
    "if is_per_area:\n",
    "    test_pred_final.to_csv('output/stack_parea_{}_{}.csv'.format(stack_idx, models), index=False)\n",
    "else:\n",
    "    test_pred_final.to_csv('output/stack_{}_{}.csv'.format(stack_idx, models), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a= pd.read_csv('output/model-03-lgb-feats-selection-cv.csv')\n",
    "#b= pd.read_csv('output/model-03-lgb-feats-selection-test-one.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a.rename(columns={'building_id':'id','total_price_predict':'target'}).to_csv('opt-pred3.csv',index=False)\n",
    "#b.rename(columns={'building_id':'id','total_price':'target'}).to_csv('opt-test3.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}