stack_mean_18_1-10,12-22,24-29.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from utilities import cal_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack_idx = '18'\n",
    "models = '1-10,12-22,24-29'\n",
    "use_test_kfold = set([2, 7, 8, 12, 13])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read CV predictions and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_models(exp):\n",
    "    exp_split = exp.split(',')\n",
    "    idx_models = []\n",
    "    for e in exp_split:\n",
    "        if '-' in e:\n",
    "            n0, n1 = e.split('-')\n",
    "            idx_models.extend(list(range(int(n0), int(n1)+1, 1)))\n",
    "        else:\n",
    "            idx_models.append(int(e))\n",
    "    return idx_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n"
     ]
    }
   ],
   "source": [
    "idx_models = parse_models(models)\n",
    "print(idx_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_in_output = [f for f in os.listdir('output/') if os.path.isfile('output/'+f)]\n",
    "files_cv = {idx: [f for f in files_in_output \n",
    "                  if f.startswith('model-%02d-' % idx) and f.endswith('cv.csv')][0] \n",
    "            for idx in idx_models}\n",
    "files_test_one = {idx: [f for f in files_in_output \n",
    "                        if f.startswith('model-%02d-' % idx) and f.endswith('test-one.csv')][0]\n",
    "                  for idx in idx_models}\n",
    "files_test_kf = {idx: [f for f in files_in_output \n",
    "                       if f.startswith('model-%02d-' % idx) and f.endswith('test-kfold.csv')][0]\n",
    "                 for idx in idx_models}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 1 model-01-lgb-cv.csv\n",
      " 1 model-01-lgb-test-kfold.csv\n",
      " 1 model-01-lgb-test-one.csv\n",
      " 2 model-02-keras-search-cv.csv\n",
      " 2 model-02-keras-search-test-kfold.csv\n",
      " 2 model-02-keras-search-test-one.csv\n",
      " 3 model-03-lgb-feats-selection-cv.csv\n",
      " 3 model-03-lgb-feats-selection-test-kfold.csv\n",
      " 3 model-03-lgb-feats-selection-test-one.csv\n",
      " 4 model-04-lgb-PCA-cv.csv\n",
      " 4 model-04-lgb-PCA-test-kfold.csv\n",
      " 4 model-04-lgb-PCA-test-one.csv\n",
      " 5 model-05-lgb-wo-per-area-cv.csv\n",
      " 5 model-05-lgb-wo-per-area-test-kfold.csv\n",
      " 5 model-05-lgb-wo-per-area-test-one.csv\n",
      " 6 model-06-lgb-lr0.001-cv.csv\n",
      " 6 model-06-lgb-lr0.001-test-kfold.csv\n",
      " 6 model-06-lgb-lr0.001-test-one.csv\n",
      " 7 model-07-keras-embedding-cv.csv\n",
      " 7 model-07-keras-embedding-test-kfold.csv\n",
      " 7 model-07-keras-embedding-test-one.csv\n",
      " 8 model-08-keras-search-long-cv.csv\n",
      " 8 model-08-keras-search-long-test-kfold.csv\n",
      " 8 model-08-keras-search-long-test-one.csv\n",
      " 9 model-09-lgb-feats-selection-75-cv.csv\n",
      " 9 model-09-lgb-feats-selection-75-test-kfold.csv\n",
      " 9 model-09-lgb-feats-selection-75-test-one.csv\n",
      "10 model-10-lgb-feats-selection-75-lr-0.001-cv.csv\n",
      "10 model-10-lgb-feats-selection-75-lr-0.001-test-kfold.csv\n",
      "10 model-10-lgb-feats-selection-75-lr-0.001-test-one.csv\n",
      "12 model-12-predict-keras-search-prelu-cv.csv\n",
      "12 model-12-predict-keras-search-prelu-test-kfold.csv\n",
      "12 model-12-predict-keras-search-prelu-test-one.csv\n",
      "13 model-13-predict-keras-he_uni-cv.csv\n",
      "13 model-13-predict-keras-he_uni-test-kfold.csv\n",
      "13 model-13-predict-keras-he_uni-test-one.csv\n",
      "14 model-14-lgb-feats-selection-75-lr-0.001-rand-cv.csv\n",
      "14 model-14-lgb-feats-selection-75-lr-0.001-rand-test-kfold.csv\n",
      "14 model-14-lgb-feats-selection-75-lr-0.001-rand-test-one.csv\n",
      "15 model-15-lgb-feats-selection-75-lr-0.001-rand323-cv.csv\n",
      "15 model-15-lgb-feats-selection-75-lr-0.001-rand323-test-kfold.csv\n",
      "15 model-15-lgb-feats-selection-75-lr-0.001-rand323-test-one.csv\n",
      "16 model-16-lgb-feats-selection-68-lr-0.001-mix5-cv.csv\n",
      "16 model-16-lgb-feats-selection-68-lr-0.001-mix5-test-kfold.csv\n",
      "16 model-16-lgb-feats-selection-68-lr-0.001-mix5-test-one.csv\n",
      "17 model-17-lgb-feats-selection-70-lr-0.001-mix5-cv.csv\n",
      "17 model-17-lgb-feats-selection-70-lr-0.001-mix5-test-kfold.csv\n",
      "17 model-17-lgb-feats-selection-70-lr-0.001-mix5-test-one.csv\n",
      "18 model-18-lgb-feats-selection-70-lr-0.001-p5-cv.csv\n",
      "18 model-18-lgb-feats-selection-70-lr-0.001-p5-test-kfold.csv\n",
      "18 model-18-lgb-feats-selection-70-lr-0.001-p5-test-one.csv\n",
      "19 model-19-lgb-search-bins-lr-0.0005-cv.csv\n",
      "19 model-19-lgb-search-bins-lr-0.0005-test-kfold.csv\n",
      "19 model-19-lgb-search-bins-lr-0.0005-test-one.csv\n",
      "20 model-20-lgb-lr-0.0008-mix5-cv.csv\n",
      "20 model-20-lgb-lr-0.0008-mix5-test-kfold.csv\n",
      "20 model-20-lgb-lr-0.0008-mix5-test-one.csv\n",
      "21 model-21-lgb-wo-per-area-long-cv.csv\n",
      "21 model-21-lgb-wo-per-area-long-test-kfold.csv\n",
      "21 model-21-lgb-wo-per-area-long-test-one.csv\n",
      "22 model-22-lgb-wo-per-area-long-2-cv.csv\n",
      "22 model-22-lgb-wo-per-area-long-2-test-kfold.csv\n",
      "22 model-22-lgb-wo-per-area-long-2-test-one.csv\n",
      "24 model-24-lgb-binary-augment-cv.csv\n",
      "24 model-24-lgb-binary-augment-test-kfold.csv\n",
      "24 model-24-lgb-binary-augment-test-one.csv\n",
      "25 model-25-lgb-search-bins-lr-0.0005-250-cv.csv\n",
      "25 model-25-lgb-search-bins-lr-0.0005-250-test-kfold.csv\n",
      "25 model-25-lgb-search-bins-lr-0.0005-250-test-one.csv\n",
      "26 model-26-lgb-search-bins-lr-0.0005-350-cv.csv\n",
      "26 model-26-lgb-search-bins-lr-0.0005-350-test-kfold.csv\n",
      "26 model-26-lgb-search-bins-lr-0.0005-350-test-one.csv\n",
      "27 model-27-lgb-feat_rm_new-cv.csv\n",
      "27 model-27-lgb-feat_rm_new-test-kfold.csv\n",
      "27 model-27-lgb-feat_rm_new-test-one.csv\n",
      "28 model-28-lgb-search-bins-lr-0.0005-255-cv.csv\n",
      "28 model-28-lgb-search-bins-lr-0.0005-255-test-kfold.csv\n",
      "28 model-28-lgb-search-bins-lr-0.0005-255-test-one.csv\n",
      "29 model-29-lgb-building_age-fillna-cv.csv\n",
      "29 model-29-lgb-building_age-fillna-test-kfold.csv\n",
      "29 model-29-lgb-building_age-fillna-test-one.csv\n"
     ]
    }
   ],
   "source": [
    "for k in files_cv: \n",
    "    print('%2d'%k, files_cv[k])\n",
    "    print('%2d'%k, files_test_kf[k])\n",
    "    print('%2d'%k, files_test_one[k])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('dataset/train.csv')\n",
    "df_test = pd.read_csv('dataset/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV predictions:\n",
      "No. 0 file: model-01-lgb-cv.csv\n",
      "No. 1 file: model-02-keras-search-cv.csv\n",
      "No. 2 file: model-03-lgb-feats-selection-cv.csv\n",
      "No. 3 file: model-04-lgb-PCA-cv.csv\n",
      "No. 4 file: model-05-lgb-wo-per-area-cv.csv\n",
      "No. 5 file: model-06-lgb-lr0.001-cv.csv\n",
      "No. 6 file: model-07-keras-embedding-cv.csv\n",
      "No. 7 file: model-08-keras-search-long-cv.csv\n",
      "No. 8 file: model-09-lgb-feats-selection-75-cv.csv\n",
      "No. 9 file: model-10-lgb-feats-selection-75-lr-0.001-cv.csv\n",
      "No. 10 file: model-12-predict-keras-search-prelu-cv.csv\n",
      "No. 11 file: model-13-predict-keras-he_uni-cv.csv\n",
      "No. 12 file: model-14-lgb-feats-selection-75-lr-0.001-rand-cv.csv\n",
      "No. 13 file: model-15-lgb-feats-selection-75-lr-0.001-rand323-cv.csv\n",
      "No. 14 file: model-16-lgb-feats-selection-68-lr-0.001-mix5-cv.csv\n",
      "No. 15 file: model-17-lgb-feats-selection-70-lr-0.001-mix5-cv.csv\n",
      "No. 16 file: model-18-lgb-feats-selection-70-lr-0.001-p5-cv.csv\n",
      "No. 17 file: model-19-lgb-search-bins-lr-0.0005-cv.csv\n",
      "No. 18 file: model-20-lgb-lr-0.0008-mix5-cv.csv\n",
      "No. 19 file: model-21-lgb-wo-per-area-long-cv.csv\n",
      "No. 20 file: model-22-lgb-wo-per-area-long-2-cv.csv\n",
      "No. 21 file: model-24-lgb-binary-augment-cv.csv\n",
      "No. 22 file: model-25-lgb-search-bins-lr-0.0005-250-cv.csv\n",
      "No. 23 file: model-26-lgb-search-bins-lr-0.0005-350-cv.csv\n",
      "No. 24 file: model-27-lgb-feat_rm_new-cv.csv\n",
      "No. 25 file: model-28-lgb-search-bins-lr-0.0005-255-cv.csv\n",
      "No. 26 file: model-29-lgb-building_age-fillna-cv.csv\n",
      "Test predictions:\n",
      "No. 0 file: model-01-lgb-test-one.csv\n",
      "No. 1 file: model-02-keras-search-test-kfold.csv\n",
      "No. 2 file: model-03-lgb-feats-selection-test-one.csv\n",
      "No. 3 file: model-04-lgb-PCA-test-one.csv\n",
      "No. 4 file: model-05-lgb-wo-per-area-test-one.csv\n",
      "No. 5 file: model-06-lgb-lr0.001-test-one.csv\n",
      "No. 6 file: model-07-keras-embedding-test-kfold.csv\n",
      "No. 7 file: model-08-keras-search-long-test-kfold.csv\n",
      "No. 8 file: model-09-lgb-feats-selection-75-test-one.csv\n",
      "No. 9 file: model-10-lgb-feats-selection-75-lr-0.001-test-one.csv\n",
      "No. 10 file: model-12-predict-keras-search-prelu-test-kfold.csv\n",
      "No. 11 file: model-13-predict-keras-he_uni-test-kfold.csv\n",
      "No. 12 file: model-14-lgb-feats-selection-75-lr-0.001-rand-test-one.csv\n",
      "No. 13 file: model-15-lgb-feats-selection-75-lr-0.001-rand323-test-one.csv\n",
      "No. 14 file: model-16-lgb-feats-selection-68-lr-0.001-mix5-test-one.csv\n",
      "No. 15 file: model-17-lgb-feats-selection-70-lr-0.001-mix5-test-one.csv\n",
      "No. 16 file: model-18-lgb-feats-selection-70-lr-0.001-p5-test-one.csv\n",
      "No. 17 file: model-19-lgb-search-bins-lr-0.0005-test-one.csv\n",
      "No. 18 file: model-20-lgb-lr-0.0008-mix5-test-one.csv\n",
      "No. 19 file: model-21-lgb-wo-per-area-long-test-one.csv\n",
      "No. 20 file: model-22-lgb-wo-per-area-long-2-test-one.csv\n",
      "No. 21 file: model-24-lgb-binary-augment-test-one.csv\n",
      "No. 22 file: model-25-lgb-search-bins-lr-0.0005-250-test-one.csv\n",
      "No. 23 file: model-26-lgb-search-bins-lr-0.0005-350-test-one.csv\n",
      "No. 24 file: model-27-lgb-feat_rm_new-test-one.csv\n",
      "No. 25 file: model-28-lgb-search-bins-lr-0.0005-255-test-one.csv\n",
      "No. 26 file: model-29-lgb-building_age-fillna-test-one.csv\n"
     ]
    }
   ],
   "source": [
    "cv = df_train[['building_id', 'building_area', 'total_price']]\n",
    "test = df_test[['building_id', 'building_area']]\n",
    "\n",
    "print('CV predictions:')\n",
    "for i, idx_model in enumerate(idx_models):\n",
    "    f = files_cv[idx_model]\n",
    "    print('No. {} file: {}'.format(i, f))\n",
    "    df = pd.read_csv('output/'+f)\n",
    "    \n",
    "    cv = pd.merge(cv, df[['building_id', 'total_price_predict']], on='building_id')\n",
    "    \n",
    "    cv = cv.rename(columns = {'total_price_predict': 'pred_{}'.format(idx_model)})\n",
    "    cv[f'log_pred_{idx_model}'] = np.log1p(cv[f'pred_{idx_model}'])\n",
    "    cv[f'log_parea_pred_{idx_model}'] = np.log1p( cv[f'pred_{idx_model}'] / cv['building_area'] )\n",
    "\n",
    "cv['log_total_price'] = np.log1p(cv['total_price'])\n",
    "cv['log_parea_total_price'] = np.log1p( cv['total_price'] / cv['building_area'] )\n",
    "\n",
    "print('Test predictions:')\n",
    "for i, idx in enumerate(idx_models):\n",
    "    f = files_test_kf[idx] if idx in use_test_kfold else files_test_one[idx]\n",
    "    print('No. {} file: {}'.format(i, f))\n",
    "    df = pd.read_csv('output/'+f)\n",
    "\n",
    "    test = pd.merge(test, df[['building_id','total_price']], on='building_id')\n",
    "        \n",
    "    test = test.rename(columns = {'total_price': 'pred_{}'.format(idx)})\n",
    "    test[f'log_pred_{idx}'] = np.log1p(test[f'pred_{idx}'])\n",
    "    test[f'log_parea_pred_{idx}'] = np.log1p( test[f'pred_{idx}'] / test['building_area'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>building_area</th>\n",
       "      <th>total_price</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>log_parea_pred_1</th>\n",
       "      <th>pred_2</th>\n",
       "      <th>log_pred_2</th>\n",
       "      <th>log_parea_pred_2</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>...</th>\n",
       "      <th>log_pred_27</th>\n",
       "      <th>log_parea_pred_27</th>\n",
       "      <th>pred_28</th>\n",
       "      <th>log_pred_28</th>\n",
       "      <th>log_parea_pred_28</th>\n",
       "      <th>pred_29</th>\n",
       "      <th>log_pred_29</th>\n",
       "      <th>log_parea_pred_29</th>\n",
       "      <th>log_total_price</th>\n",
       "      <th>log_parea_total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e3mMIMR3JJqCaXz1</td>\n",
       "      <td>3.418175</td>\n",
       "      <td>6.476038e+05</td>\n",
       "      <td>6.331552e+05</td>\n",
       "      <td>13.358472</td>\n",
       "      <td>12.129369</td>\n",
       "      <td>717209.94</td>\n",
       "      <td>13.483125</td>\n",
       "      <td>12.254022</td>\n",
       "      <td>6.656891e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>13.381324</td>\n",
       "      <td>12.152221</td>\n",
       "      <td>6.374413e+05</td>\n",
       "      <td>13.365219</td>\n",
       "      <td>12.136116</td>\n",
       "      <td>6.274171e+05</td>\n",
       "      <td>13.349368</td>\n",
       "      <td>12.120265</td>\n",
       "      <td>13.381036</td>\n",
       "      <td>12.151933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LgwzgklNvy4QCtq5</td>\n",
       "      <td>4.041309</td>\n",
       "      <td>3.321452e+06</td>\n",
       "      <td>3.064324e+06</td>\n",
       "      <td>14.935338</td>\n",
       "      <td>13.538770</td>\n",
       "      <td>2899842.20</td>\n",
       "      <td>14.880167</td>\n",
       "      <td>13.483600</td>\n",
       "      <td>3.079196e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>14.955716</td>\n",
       "      <td>13.559149</td>\n",
       "      <td>3.160098e+06</td>\n",
       "      <td>14.966114</td>\n",
       "      <td>13.569546</td>\n",
       "      <td>3.145481e+06</td>\n",
       "      <td>14.961478</td>\n",
       "      <td>13.564910</td>\n",
       "      <td>15.015913</td>\n",
       "      <td>13.619345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ucIR2NLLsC3T650L</td>\n",
       "      <td>5.584279</td>\n",
       "      <td>9.570885e+06</td>\n",
       "      <td>9.827776e+06</td>\n",
       "      <td>16.100723</td>\n",
       "      <td>14.380769</td>\n",
       "      <td>9766813.00</td>\n",
       "      <td>16.094501</td>\n",
       "      <td>14.374546</td>\n",
       "      <td>9.814852e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>16.092359</td>\n",
       "      <td>14.372404</td>\n",
       "      <td>9.804749e+06</td>\n",
       "      <td>16.098377</td>\n",
       "      <td>14.378423</td>\n",
       "      <td>9.762486e+06</td>\n",
       "      <td>16.094058</td>\n",
       "      <td>14.374103</td>\n",
       "      <td>16.074236</td>\n",
       "      <td>14.354282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jre1pJhcQj91Kdky</td>\n",
       "      <td>13.563031</td>\n",
       "      <td>1.421501e+07</td>\n",
       "      <td>1.255350e+07</td>\n",
       "      <td>16.345510</td>\n",
       "      <td>13.738164</td>\n",
       "      <td>12699800.00</td>\n",
       "      <td>16.357097</td>\n",
       "      <td>13.749750</td>\n",
       "      <td>1.255981e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>16.348848</td>\n",
       "      <td>13.741502</td>\n",
       "      <td>1.263065e+07</td>\n",
       "      <td>16.351637</td>\n",
       "      <td>13.744290</td>\n",
       "      <td>1.262574e+07</td>\n",
       "      <td>16.351248</td>\n",
       "      <td>13.743901</td>\n",
       "      <td>16.469809</td>\n",
       "      <td>13.862462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rQpYpY9nRG7X5mmr</td>\n",
       "      <td>4.688108</td>\n",
       "      <td>7.627120e+05</td>\n",
       "      <td>1.215194e+06</td>\n",
       "      <td>14.010415</td>\n",
       "      <td>12.465389</td>\n",
       "      <td>2012610.50</td>\n",
       "      <td>14.514944</td>\n",
       "      <td>12.969916</td>\n",
       "      <td>1.128419e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>13.938992</td>\n",
       "      <td>12.393966</td>\n",
       "      <td>1.132637e+06</td>\n",
       "      <td>13.940060</td>\n",
       "      <td>12.395034</td>\n",
       "      <td>1.128514e+06</td>\n",
       "      <td>13.936413</td>\n",
       "      <td>12.391388</td>\n",
       "      <td>13.544637</td>\n",
       "      <td>11.999613</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 86 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id  building_area   total_price        pred_1  log_pred_1  \\\n",
       "0  e3mMIMR3JJqCaXz1       3.418175  6.476038e+05  6.331552e+05   13.358472   \n",
       "1  LgwzgklNvy4QCtq5       4.041309  3.321452e+06  3.064324e+06   14.935338   \n",
       "2  ucIR2NLLsC3T650L       5.584279  9.570885e+06  9.827776e+06   16.100723   \n",
       "3  jre1pJhcQj91Kdky      13.563031  1.421501e+07  1.255350e+07   16.345510   \n",
       "4  rQpYpY9nRG7X5mmr       4.688108  7.627120e+05  1.215194e+06   14.010415   \n",
       "\n",
       "   log_parea_pred_1       pred_2  log_pred_2  log_parea_pred_2        pred_3  \\\n",
       "0         12.129369    717209.94   13.483125         12.254022  6.656891e+05   \n",
       "1         13.538770   2899842.20   14.880167         13.483600  3.079196e+06   \n",
       "2         14.380769   9766813.00   16.094501         14.374546  9.814852e+06   \n",
       "3         13.738164  12699800.00   16.357097         13.749750  1.255981e+07   \n",
       "4         12.465389   2012610.50   14.514944         12.969916  1.128419e+06   \n",
       "\n",
       "           ...            log_pred_27  log_parea_pred_27       pred_28  \\\n",
       "0          ...              13.381324          12.152221  6.374413e+05   \n",
       "1          ...              14.955716          13.559149  3.160098e+06   \n",
       "2          ...              16.092359          14.372404  9.804749e+06   \n",
       "3          ...              16.348848          13.741502  1.263065e+07   \n",
       "4          ...              13.938992          12.393966  1.132637e+06   \n",
       "\n",
       "   log_pred_28  log_parea_pred_28       pred_29  log_pred_29  \\\n",
       "0    13.365219          12.136116  6.274171e+05    13.349368   \n",
       "1    14.966114          13.569546  3.145481e+06    14.961478   \n",
       "2    16.098377          14.378423  9.762486e+06    16.094058   \n",
       "3    16.351637          13.744290  1.262574e+07    16.351248   \n",
       "4    13.940060          12.395034  1.128514e+06    13.936413   \n",
       "\n",
       "   log_parea_pred_29  log_total_price  log_parea_total_price  \n",
       "0          12.120265        13.381036              12.151933  \n",
       "1          13.564910        15.015913              13.619345  \n",
       "2          14.374103        16.074236              14.354282  \n",
       "3          13.743901        16.469809              13.862462  \n",
       "4          12.391388        13.544637              11.999613  \n",
       "\n",
       "[5 rows x 86 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>building_area</th>\n",
       "      <th>pred_1</th>\n",
       "      <th>log_pred_1</th>\n",
       "      <th>log_parea_pred_1</th>\n",
       "      <th>pred_2</th>\n",
       "      <th>log_pred_2</th>\n",
       "      <th>log_parea_pred_2</th>\n",
       "      <th>pred_3</th>\n",
       "      <th>log_pred_3</th>\n",
       "      <th>...</th>\n",
       "      <th>log_parea_pred_26</th>\n",
       "      <th>pred_27</th>\n",
       "      <th>log_pred_27</th>\n",
       "      <th>log_parea_pred_27</th>\n",
       "      <th>pred_28</th>\n",
       "      <th>log_pred_28</th>\n",
       "      <th>log_parea_pred_28</th>\n",
       "      <th>pred_29</th>\n",
       "      <th>log_pred_29</th>\n",
       "      <th>log_parea_pred_29</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>3.418175</td>\n",
       "      <td>1.526912e+07</td>\n",
       "      <td>16.541343</td>\n",
       "      <td>15.312236</td>\n",
       "      <td>12470072.0</td>\n",
       "      <td>16.338842</td>\n",
       "      <td>15.109735</td>\n",
       "      <td>1.531685e+07</td>\n",
       "      <td>16.544464</td>\n",
       "      <td>...</td>\n",
       "      <td>15.195091</td>\n",
       "      <td>1.405424e+07</td>\n",
       "      <td>16.458435</td>\n",
       "      <td>15.229328</td>\n",
       "      <td>1.485198e+07</td>\n",
       "      <td>16.513644</td>\n",
       "      <td>15.284537</td>\n",
       "      <td>1.458851e+07</td>\n",
       "      <td>16.495745</td>\n",
       "      <td>15.266638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>7.726227</td>\n",
       "      <td>3.924241e+06</td>\n",
       "      <td>15.182684</td>\n",
       "      <td>13.138065</td>\n",
       "      <td>3916552.2</td>\n",
       "      <td>15.180723</td>\n",
       "      <td>13.136104</td>\n",
       "      <td>3.977095e+06</td>\n",
       "      <td>15.196062</td>\n",
       "      <td>...</td>\n",
       "      <td>13.135851</td>\n",
       "      <td>3.932854e+06</td>\n",
       "      <td>15.184876</td>\n",
       "      <td>13.140257</td>\n",
       "      <td>3.925095e+06</td>\n",
       "      <td>15.182901</td>\n",
       "      <td>13.138282</td>\n",
       "      <td>3.930381e+06</td>\n",
       "      <td>15.184247</td>\n",
       "      <td>13.139628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>12.170581</td>\n",
       "      <td>1.096127e+07</td>\n",
       "      <td>16.209879</td>\n",
       "      <td>13.710858</td>\n",
       "      <td>11912735.0</td>\n",
       "      <td>16.293119</td>\n",
       "      <td>13.794098</td>\n",
       "      <td>1.084967e+07</td>\n",
       "      <td>16.199646</td>\n",
       "      <td>...</td>\n",
       "      <td>13.692710</td>\n",
       "      <td>1.040041e+07</td>\n",
       "      <td>16.157355</td>\n",
       "      <td>13.658335</td>\n",
       "      <td>1.066340e+07</td>\n",
       "      <td>16.182328</td>\n",
       "      <td>13.683307</td>\n",
       "      <td>1.053625e+07</td>\n",
       "      <td>16.170332</td>\n",
       "      <td>13.671312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>2.252256</td>\n",
       "      <td>6.155550e+06</td>\n",
       "      <td>15.632865</td>\n",
       "      <td>14.820933</td>\n",
       "      <td>5940670.0</td>\n",
       "      <td>15.597333</td>\n",
       "      <td>14.785400</td>\n",
       "      <td>6.015238e+06</td>\n",
       "      <td>15.609807</td>\n",
       "      <td>...</td>\n",
       "      <td>14.801146</td>\n",
       "      <td>5.905111e+06</td>\n",
       "      <td>15.591329</td>\n",
       "      <td>14.779397</td>\n",
       "      <td>6.025775e+06</td>\n",
       "      <td>15.611557</td>\n",
       "      <td>14.799625</td>\n",
       "      <td>5.922934e+06</td>\n",
       "      <td>15.594343</td>\n",
       "      <td>14.782410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>5.813985</td>\n",
       "      <td>1.062995e+06</td>\n",
       "      <td>13.876602</td>\n",
       "      <td>12.116340</td>\n",
       "      <td>1088488.1</td>\n",
       "      <td>13.900301</td>\n",
       "      <td>12.140039</td>\n",
       "      <td>1.027248e+06</td>\n",
       "      <td>13.842395</td>\n",
       "      <td>...</td>\n",
       "      <td>12.147091</td>\n",
       "      <td>1.048505e+06</td>\n",
       "      <td>13.862877</td>\n",
       "      <td>12.102615</td>\n",
       "      <td>1.078927e+06</td>\n",
       "      <td>13.891479</td>\n",
       "      <td>12.131217</td>\n",
       "      <td>1.092658e+06</td>\n",
       "      <td>13.904125</td>\n",
       "      <td>12.143863</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 83 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        building_id  building_area        pred_1  log_pred_1  \\\n",
       "0  X5gsdTWGS3W7JJQB       3.418175  1.526912e+07   16.541343   \n",
       "1  BTshNOJyKHnT2YIT       7.726227  3.924241e+06   15.182684   \n",
       "2  dhdymr0lV8N5kZOT      12.170581  1.096127e+07   16.209879   \n",
       "3  VEwyGGMcD56w5BOc       2.252256  6.155550e+06   15.632865   \n",
       "4  wmUeMoJZfsqaSX9b       5.813985  1.062995e+06   13.876602   \n",
       "\n",
       "   log_parea_pred_1      pred_2  log_pred_2  log_parea_pred_2        pred_3  \\\n",
       "0         15.312236  12470072.0   16.338842         15.109735  1.531685e+07   \n",
       "1         13.138065   3916552.2   15.180723         13.136104  3.977095e+06   \n",
       "2         13.710858  11912735.0   16.293119         13.794098  1.084967e+07   \n",
       "3         14.820933   5940670.0   15.597333         14.785400  6.015238e+06   \n",
       "4         12.116340   1088488.1   13.900301         12.140039  1.027248e+06   \n",
       "\n",
       "   log_pred_3        ...          log_parea_pred_26       pred_27  \\\n",
       "0   16.544464        ...                  15.195091  1.405424e+07   \n",
       "1   15.196062        ...                  13.135851  3.932854e+06   \n",
       "2   16.199646        ...                  13.692710  1.040041e+07   \n",
       "3   15.609807        ...                  14.801146  5.905111e+06   \n",
       "4   13.842395        ...                  12.147091  1.048505e+06   \n",
       "\n",
       "   log_pred_27  log_parea_pred_27       pred_28  log_pred_28  \\\n",
       "0    16.458435          15.229328  1.485198e+07    16.513644   \n",
       "1    15.184876          13.140257  3.925095e+06    15.182901   \n",
       "2    16.157355          13.658335  1.066340e+07    16.182328   \n",
       "3    15.591329          14.779397  6.025775e+06    15.611557   \n",
       "4    13.862877          12.102615  1.078927e+06    13.891479   \n",
       "\n",
       "   log_parea_pred_28       pred_29  log_pred_29  log_parea_pred_29  \n",
       "0          15.284537  1.458851e+07    16.495745          15.266638  \n",
       "1          13.138282  3.930381e+06    15.184247          13.139628  \n",
       "2          13.683307  1.053625e+07    16.170332          13.671312  \n",
       "3          14.799625  5.922934e+06    15.594343          14.782410  \n",
       "4          12.131217  1.092658e+06    13.904125          12.143863  \n",
       "\n",
       "[5 rows x 83 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check models scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_models = {idx_model: cal_score(cv['total_price'], cv[f'pred_{idx_model}'])\n",
    "                 for idx_model in idx_models}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0 model-01 5870.873059\n",
      " 1 model-02 5400.852164\n",
      " 2 model-03 5877.873452\n",
      " 3 model-04 5713.867808\n",
      " 4 model-05 5724.869598\n",
      " 5 model-06 5886.873769\n",
      " 6 model-07 5171.836449\n",
      " 7 model-08 5514.858826\n",
      " 8 model-09 5872.873118\n",
      " 9 model-10 5897.873845\n",
      "10 model-12 5486.856963\n",
      "11 model-13 5506.858055\n",
      "12 model-14 5908.873901\n",
      "13 model-15 5900.873836\n",
      "14 model-16 5907.874126\n",
      "15 model-17 5905.874165\n",
      "16 model-18 5908.874297\n",
      "17 model-19 5911.874156\n",
      "18 model-20 5908.874040\n",
      "19 model-21 5758.870702\n",
      "20 model-22 5752.870671\n",
      "21 model-24 5866.873835\n",
      "22 model-25 5908.874238\n",
      "23 model-26 5918.873998\n",
      "24 model-27 5892.873194\n",
      "25 model-28 5901.874202\n",
      "26 model-29 5884.873848\n"
     ]
    }
   ],
   "source": [
    "for i, (idx_model, s) in enumerate(scores_models.items()):\n",
    "    print('%2d'%i, 'model-%02d'%idx_model, '%.6f'%s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search subset model mean "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prefix(is_per_area, is_log):\n",
    "    if is_log:\n",
    "        if is_per_area:\n",
    "            col_prefix = 'log_parea_pred'\n",
    "        else:\n",
    "            col_prefix = 'log_pred'\n",
    "    else:\n",
    "        if is_per_area:\n",
    "            raise 'Not support'\n",
    "        else:\n",
    "            col_prefix = 'pred'\n",
    "    return col_prefix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_subset(df, idx_models, is_per_area, is_log):\n",
    "    col_prefix = get_prefix(is_per_area, is_log)\n",
    "    cols_models = [col_prefix+'_'+str(i) for i in idx_models]\n",
    "    return df[cols_models].mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_mean_subset(df, idx_models, is_per_area, is_log):\n",
    "    y_pred = mean_subset(df, idx_models, is_per_area, is_log)\n",
    "    y_pred_final = y_pred\n",
    "    if is_log:\n",
    "        y_pred_final = np.expm1(y_pred_final)\n",
    "    if is_per_area:\n",
    "        y_pred_final = y_pred_final * df['building_area']\n",
    "    return y_pred_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_mean_subset(cv, idx_models, is_per_area, is_log):\n",
    "    idx_models_subset = idx_models.copy()\n",
    "    result_list = []\n",
    "    y_pred_cv = predict_mean_subset(cv, idx_models_subset, is_per_area=is_per_area, is_log=is_log)\n",
    "    score = cal_score(cv['total_price'], y_pred_cv)\n",
    "    print('%.6f'%score, idx_models_subset)\n",
    "    result_list.append([score, idx_models_subset])\n",
    "    while len(idx_models_subset) > 1:\n",
    "        print('N models remained:', len(idx_models_subset))\n",
    "        result_list_subset = []\n",
    "        for i_rm in range(len(idx_models_subset)):\n",
    "            idx_models_subset_next = idx_models_subset[:i_rm]+idx_models_subset[i_rm+1:]\n",
    "            y_pred_cv = predict_mean_subset(cv, idx_models_subset_next, is_per_area=is_per_area, is_log=is_log)\n",
    "            score = cal_score(cv['total_price'], y_pred_cv)\n",
    "            print('%.6f'%score, idx_models_subset_next)\n",
    "            result_list_subset.append([score, idx_models_subset_next])\n",
    "        result_list.extend(result_list_subset)\n",
    "        result_list_subset.sort(key=lambda x: x[0], reverse=True)\n",
    "        print('Best of the round:', result_list_subset[0][0], result_list_subset[0][1])\n",
    "        idx_models_subset = result_list_subset[0][1]\n",
    "        print()\n",
    "    result_list.sort(key=lambda x: x[0], reverse=True)\n",
    "    return result_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Log mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5944.875504 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "N models remained: 27\n",
      "5944.875522 [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5942.875473 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.875506 [1, 2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5942.875505 [1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.875458 [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.875514 [1, 2, 3, 4, 5, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.875473 [1, 2, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.875460 [1, 2, 3, 4, 5, 6, 7, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875519 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875514 [1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5939.875494 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5939.875492 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875511 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875514 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875507 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875505 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875499 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875501 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875506 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.875462 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.875460 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5941.875451 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5944.875496 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5942.875505 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5942.875491 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5944.875499 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n",
      "5942.875495 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5945.875519275808 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 26\n",
      "5946.875535 [2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.875497 [1, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875518 [1, 2, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.875521 [1, 2, 3, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875473 [1, 2, 3, 4, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875527 [1, 2, 3, 4, 5, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5947.875496 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875484 [1, 2, 3, 4, 5, 6, 7, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875528 [1, 2, 3, 4, 5, 6, 7, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5942.875519 [1, 2, 3, 4, 5, 6, 7, 8, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.875516 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875525 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.875527 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875520 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875519 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875512 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875515 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875520 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.875476 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.875474 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5942.875464 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5946.875510 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5944.875519 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5943.875504 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5945.875513 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n",
      "5945.875509 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5947.875495884089 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 25\n",
      "5948.875520 [2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875432 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875504 [1, 2, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875502 [1, 2, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.875453 [1, 2, 3, 4, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875512 [1, 2, 3, 4, 5, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875423 [1, 2, 3, 4, 5, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875514 [1, 2, 3, 4, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875454 [1, 2, 3, 4, 5, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875453 [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875511 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875514 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875507 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875505 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.875498 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875502 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875507 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5939.875457 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.875456 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5942.875443 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5945.875496 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5945.875506 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5942.875489 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5945.875499 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5945.875493 [1, 2, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5953.875431794345 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 24\n",
      "5950.875465 [3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875447 [1, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875438 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875394 [1, 3, 4, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875455 [1, 3, 4, 5, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875294 [1, 3, 4, 5, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875459 [1, 3, 4, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875326 [1, 3, 4, 5, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875325 [1, 3, 4, 5, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875456 [1, 3, 4, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875460 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875452 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875450 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875442 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875446 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875452 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875399 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875397 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5952.875381 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5953.875441 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5951.875451 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5949.875434 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5952.875444 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n",
      "5951.875437 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5955.8754380840255 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 23\n",
      "5953.875473 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875454 [1, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875402 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875462 [1, 3, 5, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875291 [1, 3, 5, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875468 [1, 3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875328 [1, 3, 5, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875323 [1, 3, 5, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875464 [1, 3, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875468 [1, 3, 5, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875459 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875458 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875449 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875453 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875459 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875405 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875404 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5952.875385 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5952.875448 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5952.875458 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5949.875439 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5953.875451 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n",
      "5952.875444 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5959.875405001211 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 22\n",
      "5961.875441 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875420 [1, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875337 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875430 [1, 3, 5, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875258 [1, 3, 5, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875438 [1, 3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875297 [1, 3, 5, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875292 [1, 3, 5, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875434 [1, 3, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875438 [1, 3, 5, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875429 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875426 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875417 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875423 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875429 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875340 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 25, 26, 27, 28, 29]\n",
      "5955.875350 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 25, 26, 27, 28, 29]\n",
      "5956.875417 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5955.875427 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 27, 28, 29]\n",
      "5954.875408 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 28, 29]\n",
      "5956.875421 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 29]\n",
      "5955.875413 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5961.87544067905 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 21\n",
      "5956.875454 [5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875371 [3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875463 [3, 5, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875298 [3, 5, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5961.875474 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875340 [3, 5, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5948.875333 [3, 5, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875470 [3, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875474 [3, 5, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875465 [3, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875462 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875453 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875459 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875466 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5954.875374 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 25, 26, 27, 28, 29]\n",
      "5957.875383 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 25, 26, 27, 28, 29]\n",
      "5957.875453 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5959.875464 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 27, 28, 29]\n",
      "5957.875444 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 28, 29]\n",
      "5960.875457 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 29]\n",
      "5958.875449 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5961.875473922974 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 20\n",
      "5958.875486 [5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875404 [3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875496 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875339 [3, 5, 6, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875383 [3, 5, 6, 8, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875376 [3, 5, 6, 8, 12, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875502 [3, 5, 6, 8, 12, 13, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875506 [3, 5, 6, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875497 [3, 5, 6, 8, 12, 13, 14, 15, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875495 [3, 5, 6, 8, 12, 13, 14, 15, 16, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875485 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875492 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875499 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875408 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 25, 26, 27, 28, 29]\n",
      "5957.875414 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 25, 26, 27, 28, 29]\n",
      "5957.875485 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5957.875497 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 27, 28, 29]\n",
      "5954.875476 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 28, 29]\n",
      "5956.875489 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 29]\n",
      "5954.875482 [3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5960.87549607151 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 19\n",
      "5960.875504 [5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875423 [3, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875370 [3, 5, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875415 [3, 5, 8, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875408 [3, 5, 8, 12, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875524 [3, 5, 8, 12, 13, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875528 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875518 [3, 5, 8, 12, 13, 14, 15, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875516 [3, 5, 8, 12, 13, 14, 15, 16, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875506 [3, 5, 8, 12, 13, 14, 15, 16, 17, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875514 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875521 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875427 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 25, 26, 27, 28, 29]\n",
      "5960.875432 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 25, 26, 27, 28, 29]\n",
      "5961.875507 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5958.875519 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 27, 28, 29]\n",
      "5954.875497 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 28, 29]\n",
      "5959.875511 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 29]\n",
      "5959.875503 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5961.875506791447 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 18\n",
      "5960.875511 [5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5955.875433 [3, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5953.875392 [3, 5, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5955.875440 [3, 5, 8, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5951.875433 [3, 5, 8, 12, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5959.875531 [3, 5, 8, 12, 13, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5960.875536 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5959.875525 [3, 5, 8, 12, 13, 14, 15, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5957.875523 [3, 5, 8, 12, 13, 14, 15, 16, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5955.875512 [3, 5, 8, 12, 13, 14, 15, 16, 17, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5958.875520 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 20, 22, 24, 26, 27, 28, 29]\n",
      "5958.875528 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 26, 27, 28, 29]\n",
      "5956.875436 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 24, 26, 27, 28, 29]\n",
      "5955.875436 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 26, 27, 28, 29]\n",
      "5958.875526 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 27, 28, 29]\n",
      "5952.875503 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 28, 29]\n",
      "5957.875517 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 29]\n",
      "5960.875509 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "Best of the round: 5960.875535573506 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 17\n",
      "5958.875535 [5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5957.875461 [3, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5955.875437 [3, 5, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5955.875488 [3, 5, 8, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5955.875479 [3, 5, 8, 12, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5958.875554 [3, 5, 8, 12, 13, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5958.875548 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5957.875546 [3, 5, 8, 12, 13, 14, 16, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5956.875536 [3, 5, 8, 12, 13, 14, 16, 17, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "5957.875544 [3, 5, 8, 12, 13, 14, 16, 17, 18, 20, 22, 24, 26, 27, 28, 29]\n",
      "5958.875552 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 22, 24, 26, 27, 28, 29]\n",
      "5952.875464 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 24, 26, 27, 28, 29]\n",
      "5954.875458 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 26, 27, 28, 29]\n",
      "5958.875550 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 27, 28, 29]\n",
      "5958.875526 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 28, 29]\n",
      "5957.875541 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 29]\n",
      "5959.875533 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "Best of the round: 5959.875533138635 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "\n",
      "N models remained: 16\n",
      "5955.875527 [5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5957.875457 [3, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5955.875451 [3, 5, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5950.875505 [3, 5, 8, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5951.875496 [3, 5, 8, 12, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5956.875545 [3, 5, 8, 12, 13, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5958.875540 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5957.875537 [3, 5, 8, 12, 13, 14, 16, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5955.875528 [3, 5, 8, 12, 13, 14, 16, 17, 19, 20, 22, 24, 26, 27, 28]\n",
      "5955.875534 [3, 5, 8, 12, 13, 14, 16, 17, 18, 20, 22, 24, 26, 27, 28]\n",
      "5954.875542 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 22, 24, 26, 27, 28]\n",
      "5958.875460 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 24, 26, 27, 28]\n",
      "5954.875446 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 26, 27, 28]\n",
      "5954.875541 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 27, 28]\n",
      "5958.875511 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5955.875530 [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27]\n",
      "Best of the round: 5958.875540026772 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "\n",
      "N models remained: 15\n",
      "5955.875524 [5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5955.875460 [3, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5954.875480 [3, 5, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5951.875537 [3, 5, 8, 13, 14, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5954.875528 [3, 5, 8, 12, 14, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5956.875540 [3, 5, 8, 12, 13, 17, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5956.875531 [3, 5, 8, 12, 13, 14, 18, 19, 20, 22, 24, 26, 27, 28]\n",
      "5952.875522 [3, 5, 8, 12, 13, 14, 17, 19, 20, 22, 24, 26, 27, 28]\n",
      "5956.875529 [3, 5, 8, 12, 13, 14, 17, 18, 20, 22, 24, 26, 27, 28]\n",
      "5952.875538 [3, 5, 8, 12, 13, 14, 17, 18, 19, 22, 24, 26, 27, 28]\n",
      "5956.875463 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 24, 26, 27, 28]\n",
      "5951.875440 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 26, 27, 28]\n",
      "5953.875537 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 27, 28]\n",
      "5957.875505 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5955.875524 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 27]\n",
      "Best of the round: 5957.875504560616 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "\n",
      "N models remained: 14\n",
      "5951.875473 [5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5953.875421 [3, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5950.875471 [3, 5, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5948.875534 [3, 5, 8, 13, 14, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5947.875524 [3, 5, 8, 12, 14, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5953.875483 [3, 5, 8, 12, 13, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5951.875477 [3, 5, 8, 12, 13, 14, 18, 19, 20, 22, 24, 26, 28]\n",
      "5952.875468 [3, 5, 8, 12, 13, 14, 17, 19, 20, 22, 24, 26, 28]\n",
      "5952.875472 [3, 5, 8, 12, 13, 14, 17, 18, 20, 22, 24, 26, 28]\n",
      "5952.875482 [3, 5, 8, 12, 13, 14, 17, 18, 19, 22, 24, 26, 28]\n",
      "5953.875421 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 24, 26, 28]\n",
      "5948.875387 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 26, 28]\n",
      "5951.875481 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 28]\n",
      "5950.875468 [3, 5, 8, 12, 13, 14, 17, 18, 19, 20, 22, 24, 26]\n",
      "Best of the round: 5953.875482644671 [3, 5, 8, 12, 13, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "\n",
      "N models remained: 13\n",
      "5948.875427 [5, 8, 12, 13, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5951.875394 [3, 8, 12, 13, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5951.875485 [3, 5, 12, 13, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5948.875556 [3, 5, 8, 13, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5946.875546 [3, 5, 8, 12, 17, 18, 19, 20, 22, 24, 26, 28]\n",
      "5950.875428 [3, 5, 8, 12, 13, 18, 19, 20, 22, 24, 26, 28]\n",
      "5949.875420 [3, 5, 8, 12, 13, 17, 19, 20, 22, 24, 26, 28]\n",
      "5952.875425 [3, 5, 8, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5951.875436 [3, 5, 8, 12, 13, 17, 18, 19, 22, 24, 26, 28]\n",
      "5948.875392 [3, 5, 8, 12, 13, 17, 18, 19, 20, 24, 26, 28]\n",
      "5945.875341 [3, 5, 8, 12, 13, 17, 18, 19, 20, 22, 26, 28]\n",
      "5951.875434 [3, 5, 8, 12, 13, 17, 18, 19, 20, 22, 24, 28]\n",
      "5951.875421 [3, 5, 8, 12, 13, 17, 18, 19, 20, 22, 24, 26]\n",
      "Best of the round: 5952.875425237741 [3, 5, 8, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "\n",
      "N models remained: 12\n",
      "5946.875335 [5, 8, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5951.875327 [3, 8, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5951.875474 [3, 5, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5948.875552 [3, 5, 8, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5948.875544 [3, 5, 8, 12, 17, 18, 20, 22, 24, 26, 28]\n",
      "5946.875331 [3, 5, 8, 12, 13, 18, 20, 22, 24, 26, 28]\n",
      "5947.875325 [3, 5, 8, 12, 13, 17, 20, 22, 24, 26, 28]\n",
      "5946.875336 [3, 5, 8, 12, 13, 17, 18, 22, 24, 26, 28]\n",
      "5948.875321 [3, 5, 8, 12, 13, 17, 18, 20, 24, 26, 28]\n",
      "5936.875244 [3, 5, 8, 12, 13, 17, 18, 20, 22, 26, 28]\n",
      "5946.875335 [3, 5, 8, 12, 13, 17, 18, 20, 22, 24, 28]\n",
      "5947.875321 [3, 5, 8, 12, 13, 17, 18, 20, 22, 24, 26]\n",
      "Best of the round: 5951.875474419962 [3, 5, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "\n",
      "N models remained: 11\n",
      "5946.875441 [5, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5954.875426 [3, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5940.875393 [3, 5, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5940.875376 [3, 5, 12, 17, 18, 20, 22, 24, 26, 28]\n",
      "5948.875443 [3, 5, 12, 13, 18, 20, 22, 24, 26, 28]\n",
      "5951.875435 [3, 5, 12, 13, 17, 20, 22, 24, 26, 28]\n",
      "5946.875450 [3, 5, 12, 13, 17, 18, 22, 24, 26, 28]\n",
      "5951.875425 [3, 5, 12, 13, 17, 18, 20, 24, 26, 28]\n",
      "5947.875335 [3, 5, 12, 13, 17, 18, 20, 22, 26, 28]\n",
      "5946.875446 [3, 5, 12, 13, 17, 18, 20, 22, 24, 28]\n",
      "5947.875432 [3, 5, 12, 13, 17, 18, 20, 22, 24, 26]\n",
      "Best of the round: 5954.8754258482295 [3, 12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "\n",
      "N models remained: 10\n",
      "5953.875388 [12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5940.875385 [3, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5941.875367 [3, 12, 17, 18, 20, 22, 24, 26, 28]\n",
      "5951.875394 [3, 12, 13, 18, 20, 22, 24, 26, 28]\n",
      "5950.875383 [3, 12, 13, 17, 20, 22, 24, 26, 28]\n",
      "5947.875405 [3, 12, 13, 17, 18, 22, 24, 26, 28]\n",
      "5942.875218 [3, 12, 13, 17, 18, 20, 24, 26, 28]\n",
      "5948.875274 [3, 12, 13, 17, 18, 20, 22, 26, 28]\n",
      "5946.875400 [3, 12, 13, 17, 18, 20, 22, 24, 28]\n",
      "5948.875385 [3, 12, 13, 17, 18, 20, 22, 24, 26]\n",
      "Best of the round: 5953.875387802076 [12, 13, 17, 18, 20, 22, 24, 26, 28]\n",
      "\n",
      "N models remained: 9\n",
      "5939.875417 [13, 17, 18, 20, 22, 24, 26, 28]\n",
      "5949.875393 [12, 17, 18, 20, 22, 24, 26, 28]\n",
      "5950.875311 [12, 13, 18, 20, 22, 24, 26, 28]\n",
      "5950.875303 [12, 13, 17, 20, 22, 24, 26, 28]\n",
      "5950.875329 [12, 13, 17, 18, 22, 24, 26, 28]\n",
      "5947.875129 [12, 13, 17, 18, 20, 24, 26, 28]\n",
      "5948.875190 [12, 13, 17, 18, 20, 22, 26, 28]\n",
      "5946.875324 [12, 13, 17, 18, 20, 22, 24, 28]\n",
      "5952.875306 [12, 13, 17, 18, 20, 22, 24, 26]\n",
      "Best of the round: 5952.875305668161 [12, 13, 17, 18, 20, 22, 24, 26]\n",
      "\n",
      "N models remained: 8\n",
      "5940.875445 [13, 17, 18, 20, 22, 24, 26]\n",
      "5945.875416 [12, 17, 18, 20, 22, 24, 26]\n",
      "5940.875141 [12, 13, 18, 20, 22, 24, 26]\n",
      "5942.875134 [12, 13, 17, 20, 22, 24, 26]\n",
      "5939.875153 [12, 13, 17, 18, 22, 24, 26]\n",
      "5936.874978 [12, 13, 17, 18, 20, 24, 26]\n",
      "5938.875020 [12, 13, 17, 18, 20, 22, 26]\n",
      "5940.875149 [12, 13, 17, 18, 20, 22, 24]\n",
      "Best of the round: 5945.875415893358 [12, 17, 18, 20, 22, 24, 26]\n",
      "\n",
      "N models remained: 7\n",
      "5913.874947 [17, 18, 20, 22, 24, 26]\n",
      "5942.875415 [12, 18, 20, 22, 24, 26]\n",
      "5943.875403 [12, 17, 20, 22, 24, 26]\n",
      "5941.875429 [12, 17, 18, 22, 24, 26]\n",
      "5938.875185 [12, 17, 18, 20, 24, 26]\n",
      "5943.875240 [12, 17, 18, 20, 22, 26]\n",
      "5942.875420 [12, 17, 18, 20, 22, 24]\n",
      "Best of the round: 5943.875402873843 [12, 17, 20, 22, 24, 26]\n",
      "\n",
      "N models remained: 6\n",
      "5916.874970 [17, 20, 22, 24, 26]\n",
      "5933.875311 [12, 20, 22, 24, 26]\n",
      "5937.875343 [12, 17, 22, 24, 26]\n",
      "5936.875108 [12, 17, 20, 24, 26]\n",
      "5935.875144 [12, 17, 20, 22, 26]\n",
      "5934.875330 [12, 17, 20, 22, 24]\n",
      "Best of the round: 5937.875342577739 [12, 17, 22, 24, 26]\n",
      "\n",
      "N models remained: 5\n",
      "5916.875024 [17, 22, 24, 26]\n",
      "5926.875034 [12, 22, 24, 26]\n",
      "5936.874933 [12, 17, 24, 26]\n",
      "5920.874890 [12, 17, 22, 26]\n",
      "5926.875035 [12, 17, 22, 24]\n",
      "Best of the round: 5936.874933494015 [12, 17, 24, 26]\n",
      "\n",
      "N models remained: 4\n",
      "5930.874732 [17, 24, 26]\n",
      "5911.874240 [12, 24, 26]\n",
      "5911.874118 [12, 17, 26]\n",
      "5911.874260 [12, 17, 24]\n",
      "Best of the round: 5930.874731871537 [17, 24, 26]\n",
      "\n",
      "N models remained: 3\n",
      "5920.874660 [24, 26]\n",
      "5918.874232 [17, 26]\n",
      "5918.874738 [17, 24]\n",
      "Best of the round: 5920.874660106518 [24, 26]\n",
      "\n",
      "N models remained: 2\n",
      "5918.873998 [26]\n",
      "5866.873835 [24]\n",
      "Best of the round: 5918.873998161249 [26]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "result_list_log = search_mean_subset(cv, idx_models, is_per_area=False, is_log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[5961.875506791447,\n",
       "  [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]],\n",
       " [5961.875473922974,\n",
       "  [3,\n",
       "   5,\n",
       "   6,\n",
       "   8,\n",
       "   12,\n",
       "   13,\n",
       "   14,\n",
       "   15,\n",
       "   16,\n",
       "   17,\n",
       "   18,\n",
       "   19,\n",
       "   20,\n",
       "   22,\n",
       "   24,\n",
       "   25,\n",
       "   26,\n",
       "   27,\n",
       "   28,\n",
       "   29]],\n",
       " [5961.87544067905,\n",
       "  [3,\n",
       "   5,\n",
       "   6,\n",
       "   8,\n",
       "   10,\n",
       "   12,\n",
       "   13,\n",
       "   14,\n",
       "   15,\n",
       "   16,\n",
       "   17,\n",
       "   18,\n",
       "   19,\n",
       "   20,\n",
       "   22,\n",
       "   24,\n",
       "   25,\n",
       "   26,\n",
       "   27,\n",
       "   28,\n",
       "   29]],\n",
       " [5960.875535573506,\n",
       "  [3, 5, 8, 12, 13, 14, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]],\n",
       " [5960.8755211020325,\n",
       "  [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5960.875511458838,\n",
       "  [5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]],\n",
       " [5960.875509209059,\n",
       "  [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28]],\n",
       " [5960.875503627666,\n",
       "  [5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5960.87549607151,\n",
       "  [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5960.875474232222,\n",
       "  [3,\n",
       "   5,\n",
       "   6,\n",
       "   8,\n",
       "   10,\n",
       "   12,\n",
       "   13,\n",
       "   14,\n",
       "   16,\n",
       "   17,\n",
       "   18,\n",
       "   19,\n",
       "   20,\n",
       "   22,\n",
       "   24,\n",
       "   25,\n",
       "   26,\n",
       "   27,\n",
       "   28,\n",
       "   29]]]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(result_list_log[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score and subset: 5961.875507 [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n",
      "model-03 5877.873452\n",
      "model-05 5724.869598\n",
      "model-08 5514.858826\n",
      "model-12 5486.856963\n",
      "model-13 5506.858055\n",
      "model-14 5908.873901\n",
      "model-15 5900.873836\n",
      "model-16 5907.874126\n",
      "model-17 5905.874165\n",
      "model-18 5908.874297\n",
      "model-19 5911.874156\n",
      "model-20 5908.874040\n",
      "model-22 5752.870671\n",
      "model-24 5866.873835\n",
      "model-26 5918.873998\n",
      "model-27 5892.873194\n",
      "model-28 5901.874202\n",
      "model-29 5884.873848\n"
     ]
    }
   ],
   "source": [
    "best_models_subset_log = result_list_log[0][1]\n",
    "print('Best score and subset:', '%.6f'%result_list_log[0][0], best_models_subset_log)\n",
    "for idx_model in best_models_subset_log:\n",
    "    print('model-%02d'%idx_model, '%.6f'%scores_models[idx_model])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(best_models_subset_log)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# per area or not for the mean ensemble model are the same\n",
    "#result_list_per_area_log = search_mean_subset(cv, idx_models, is_per_area=True, is_log=True)\n",
    "#result_list_per_area_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Linear mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5939.874652 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "N models remained: 27\n",
      "5940.874642 [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.874699 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5938.874627 [1, 2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5938.874641 [1, 2, 3, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874588 [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5938.874633 [1, 2, 3, 4, 5, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5942.875025 [1, 2, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.874638 [1, 2, 3, 4, 5, 6, 7, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5942.874640 [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874633 [1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.874677 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874685 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874630 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874633 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874626 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874624 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5939.874618 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.874621 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874625 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.874589 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874587 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5936.874575 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5940.874616 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5942.874625 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5940.874612 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5941.874618 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n",
      "5942.874614 [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5944.8746991570115 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 26\n",
      "5942.874700 [3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5942.874683 [1, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874694 [1, 3, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.874644 [1, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.874690 [1, 3, 4, 5, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875058 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.874646 [1, 3, 4, 5, 6, 7, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5942.874696 [1, 3, 4, 5, 6, 7, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874691 [1, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.874685 [1, 3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874697 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.874688 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.874691 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874684 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5940.874682 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.874675 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.874679 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.874684 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5941.874646 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5943.874644 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5938.874627 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5941.874674 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5940.874683 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5941.874669 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5940.874676 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n",
      "5940.874671 [1, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5949.875058335599 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 25\n",
      "5949.875081 [3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875064 [1, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875069 [1, 3, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875022 [1, 3, 4, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875071 [1, 3, 4, 5, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.874979 [1, 3, 4, 5, 6, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875078 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875074 [1, 3, 4, 5, 6, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875017 [1, 3, 4, 5, 6, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875029 [1, 3, 4, 5, 6, 8, 9, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5948.875071 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875074 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5948.875066 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875064 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875057 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875061 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875066 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875023 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875021 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5949.875002 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5948.875056 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5949.875065 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5947.875051 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5949.875058 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5950.875052 [1, 3, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5953.8750781723775 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 24\n",
      "5950.875100 [3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875082 [1, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875090 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875042 [1, 3, 4, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875089 [1, 3, 4, 5, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875003 [1, 3, 4, 5, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875093 [1, 3, 4, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5944.875043 [1, 3, 4, 5, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5947.875055 [1, 3, 4, 5, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875090 [1, 3, 4, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875093 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875085 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875083 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875076 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875081 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 20, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875085 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875043 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875041 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28, 29]\n",
      "5952.875020 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29]\n",
      "5954.875075 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 26, 27, 28, 29]\n",
      "5954.875085 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 27, 28, 29]\n",
      "5950.875070 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 28, 29]\n",
      "5953.875077 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 29]\n",
      "5954.875072 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5956.875085383532 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 23\n",
      "5952.875107 [3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875088 [1, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875099 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875050 [1, 3, 4, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875096 [1, 3, 4, 5, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875016 [1, 3, 4, 5, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875098 [1, 3, 4, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875058 [1, 3, 4, 5, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875071 [1, 3, 4, 5, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875095 [1, 3, 4, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875099 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875090 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875088 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875081 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875085 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875051 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875049 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 24, 25, 26, 27, 28, 29]\n",
      "5948.875023 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 25, 26, 27, 28, 29]\n",
      "5950.875079 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 26, 27, 28, 29]\n",
      "5951.875089 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5951.875074 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 28, 29]\n",
      "5951.875081 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 29]\n",
      "5950.875076 [1, 3, 4, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5956.875099353037 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 22\n",
      "5952.875123 [3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875103 [1, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875065 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875111 [1, 3, 5, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875026 [1, 3, 5, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875115 [1, 3, 5, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875073 [1, 3, 5, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5945.875082 [1, 3, 5, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875111 [1, 3, 5, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875115 [1, 3, 5, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875106 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875104 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875096 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875100 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875065 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875063 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 24, 25, 26, 27, 28, 29]\n",
      "5951.875035 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 25, 26, 27, 28, 29]\n",
      "5951.875094 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 26, 27, 28, 29]\n",
      "5954.875104 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5948.875087 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 28, 29]\n",
      "5954.875097 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 29]\n",
      "5953.875091 [1, 3, 5, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5955.875064928247 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 21\n",
      "5956.875088 [3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875067 [1, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875076 [1, 3, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5948.874995 [1, 3, 6, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875082 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5949.875043 [1, 3, 6, 8, 10, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875055 [1, 3, 6, 8, 10, 12, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875078 [1, 3, 6, 8, 10, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875082 [1, 3, 6, 8, 10, 12, 13, 14, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875072 [1, 3, 6, 8, 10, 12, 13, 14, 15, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875070 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875061 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875066 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875001 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5948.874999 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 24, 25, 26, 27, 28, 29]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5953.874997 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 25, 26, 27, 28, 29]\n",
      "5953.875060 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 26, 27, 28, 29]\n",
      "5955.875071 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5954.875053 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 28, 29]\n",
      "5955.875063 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 29]\n",
      "5954.875057 [1, 3, 6, 8, 10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5960.87508175082 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 20\n",
      "5959.875104 [3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875082 [1, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875092 [1, 3, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875020 [1, 3, 6, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875073 [1, 3, 6, 8, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875084 [1, 3, 6, 8, 12, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5960.875091 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875095 [1, 3, 6, 8, 12, 13, 14, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875086 [1, 3, 6, 8, 12, 13, 14, 15, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875084 [1, 3, 6, 8, 12, 13, 14, 15, 16, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5957.875075 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875081 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5948.875017 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875015 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 24, 25, 26, 27, 28, 29]\n",
      "5955.875010 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 25, 26, 27, 28, 29]\n",
      "5956.875074 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 26, 27, 28, 29]\n",
      "5956.875086 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5952.875067 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 28, 29]\n",
      "5955.875077 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 29]\n",
      "5954.875071 [1, 3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5960.875091343384 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 19\n",
      "5956.875112 [3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5959.875089 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875099 [1, 3, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875042 [1, 3, 6, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5958.875098 [1, 3, 6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875110 [1, 3, 6, 8, 12, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875100 [1, 3, 6, 8, 12, 13, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875090 [1, 3, 6, 8, 12, 13, 15, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875088 [1, 3, 6, 8, 12, 13, 15, 16, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875080 [1, 3, 6, 8, 12, 13, 15, 16, 17, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875086 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875026 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875024 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 24, 25, 26, 27, 28, 29]\n",
      "5954.875015 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 25, 26, 27, 28, 29]\n",
      "5953.875079 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 26, 27, 28, 29]\n",
      "5954.875092 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5951.875071 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 28, 29]\n",
      "5953.875083 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 29]\n",
      "5952.875076 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5959.875089024389 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 18\n",
      "5959.875104 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875090 [1, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875054 [1, 6, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875111 [1, 6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5948.875123 [1, 6, 8, 12, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875095 [1, 6, 8, 12, 13, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875085 [1, 6, 8, 12, 13, 15, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875082 [1, 6, 8, 12, 13, 15, 16, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875074 [1, 6, 8, 12, 13, 15, 16, 17, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875081 [1, 6, 8, 12, 13, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875019 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875017 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 24, 25, 26, 27, 28, 29]\n",
      "5953.875006 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 25, 26, 27, 28, 29]\n",
      "5951.875073 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 26, 27, 28, 29]\n",
      "5952.875087 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5950.875066 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 28, 29]\n",
      "5952.875077 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 29]\n",
      "5954.875070 [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5959.875103957208 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 17\n",
      "5954.875096 [8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875085 [6, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5954.875145 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875156 [6, 8, 12, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875106 [6, 8, 12, 13, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875096 [6, 8, 12, 13, 15, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875093 [6, 8, 12, 13, 15, 16, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875085 [6, 8, 12, 13, 15, 16, 17, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875093 [6, 8, 12, 13, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5950.875028 [6, 8, 12, 13, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5951.875027 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 24, 25, 26, 27, 28, 29]\n",
      "5953.875013 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 25, 26, 27, 28, 29]\n",
      "5951.875085 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 26, 27, 28, 29]\n",
      "5950.875100 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5951.875077 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 28, 29]\n",
      "5952.875088 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 29]\n",
      "5954.875082 [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5954.87514541573 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "\n",
      "N models remained: 16\n",
      "5952.875158 [8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5946.875011 [6, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5948.875088 [6, 8, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875173 [6, 8, 13, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875162 [6, 8, 13, 15, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5953.875159 [6, 8, 13, 15, 16, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875149 [6, 8, 13, 15, 16, 17, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5952.875160 [6, 8, 13, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "5955.875082 [6, 8, 13, 15, 16, 17, 18, 19, 22, 24, 25, 26, 27, 28, 29]\n",
      "5956.875079 [6, 8, 13, 15, 16, 17, 18, 19, 21, 24, 25, 26, 27, 28, 29]\n",
      "5948.875066 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 25, 26, 27, 28, 29]\n",
      "5954.875151 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 26, 27, 28, 29]\n",
      "5956.875166 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5952.875143 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 28, 29]\n",
      "5956.875154 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 29]\n",
      "5952.875147 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28]\n",
      "Best of the round: 5956.875166152039 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "\n",
      "N models remained: 15\n",
      "5948.875178 [8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5947.875042 [6, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5948.875123 [6, 8, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5953.875191 [6, 8, 13, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5953.875180 [6, 8, 13, 15, 17, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5953.875176 [6, 8, 13, 15, 16, 18, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5953.875166 [6, 8, 13, 15, 16, 17, 19, 21, 22, 24, 25, 27, 28, 29]\n",
      "5952.875174 [6, 8, 13, 15, 16, 17, 18, 21, 22, 24, 25, 27, 28, 29]\n",
      "5956.875103 [6, 8, 13, 15, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5955.875100 [6, 8, 13, 15, 16, 17, 18, 19, 21, 24, 25, 27, 28, 29]\n",
      "5950.875080 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 25, 27, 28, 29]\n",
      "5953.875165 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 27, 28, 29]\n",
      "5950.875157 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 28, 29]\n",
      "5953.875170 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 29]\n",
      "5949.875162 [6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 27, 28]\n",
      "Best of the round: 5956.87510264205 [6, 8, 13, 15, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "\n",
      "N models remained: 14\n",
      "5952.875111 [8, 13, 15, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5946.874979 [6, 13, 15, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5944.875068 [6, 8, 15, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5956.875129 [6, 8, 13, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5954.875116 [6, 8, 13, 15, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5952.875112 [6, 8, 13, 15, 16, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5954.875101 [6, 8, 13, 15, 16, 17, 19, 22, 24, 25, 27, 28, 29]\n",
      "5954.875111 [6, 8, 13, 15, 16, 17, 18, 22, 24, 25, 27, 28, 29]\n",
      "5947.874966 [6, 8, 13, 15, 16, 17, 18, 19, 24, 25, 27, 28, 29]\n",
      "5952.875008 [6, 8, 13, 15, 16, 17, 18, 19, 22, 25, 27, 28, 29]\n",
      "5953.875102 [6, 8, 13, 15, 16, 17, 18, 19, 22, 24, 27, 28, 29]\n",
      "5950.875093 [6, 8, 13, 15, 16, 17, 18, 19, 22, 24, 25, 28, 29]\n",
      "5953.875106 [6, 8, 13, 15, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5954.875099 [6, 8, 13, 15, 16, 17, 18, 19, 22, 24, 25, 27, 28]\n",
      "Best of the round: 5956.875129449494 [6, 8, 13, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "\n",
      "N models remained: 13\n",
      "5949.875132 [8, 13, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5948.875027 [6, 13, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5948.875123 [6, 8, 16, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5953.875134 [6, 8, 13, 17, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5952.875131 [6, 8, 13, 16, 18, 19, 22, 24, 25, 27, 28, 29]\n",
      "5953.875120 [6, 8, 13, 16, 17, 19, 22, 24, 25, 27, 28, 29]\n",
      "5953.875133 [6, 8, 13, 16, 17, 18, 22, 24, 25, 27, 28, 29]\n",
      "5948.874983 [6, 8, 13, 16, 17, 18, 19, 24, 25, 27, 28, 29]\n",
      "5949.875028 [6, 8, 13, 16, 17, 18, 19, 22, 25, 27, 28, 29]\n",
      "5953.875123 [6, 8, 13, 16, 17, 18, 19, 22, 24, 27, 28, 29]\n",
      "5946.875113 [6, 8, 13, 16, 17, 18, 19, 22, 24, 25, 28, 29]\n",
      "5954.875127 [6, 8, 13, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5951.875119 [6, 8, 13, 16, 17, 18, 19, 22, 24, 25, 27, 28]\n",
      "Best of the round: 5954.875127229536 [6, 8, 13, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "\n",
      "N models remained: 12\n",
      "5954.875123 [8, 13, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5951.875055 [6, 13, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5949.875159 [6, 8, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5954.875122 [6, 8, 13, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5950.875119 [6, 8, 13, 16, 18, 19, 22, 24, 25, 27, 29]\n",
      "5950.875107 [6, 8, 13, 16, 17, 19, 22, 24, 25, 27, 29]\n",
      "5948.875119 [6, 8, 13, 16, 17, 18, 22, 24, 25, 27, 29]\n",
      "5947.874969 [6, 8, 13, 16, 17, 18, 19, 24, 25, 27, 29]\n",
      "5948.875010 [6, 8, 13, 16, 17, 18, 19, 22, 25, 27, 29]\n",
      "5951.875107 [6, 8, 13, 16, 17, 18, 19, 22, 24, 27, 29]\n",
      "5943.875099 [6, 8, 13, 16, 17, 18, 19, 22, 24, 25, 29]\n",
      "5950.875104 [6, 8, 13, 16, 17, 18, 19, 22, 24, 25, 27]\n",
      "Best of the round: 5954.875122551166 [8, 13, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "\n",
      "N models remained: 11\n",
      "5948.875080 [13, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5947.875194 [8, 16, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5949.875103 [8, 13, 17, 18, 19, 22, 24, 25, 27, 29]\n",
      "5950.875099 [8, 13, 16, 18, 19, 22, 24, 25, 27, 29]\n",
      "5949.875087 [8, 13, 16, 17, 19, 22, 24, 25, 27, 29]\n",
      "5950.875103 [8, 13, 16, 17, 18, 22, 24, 25, 27, 29]\n",
      "5947.874939 [8, 13, 16, 17, 18, 19, 24, 25, 27, 29]\n",
      "5946.874985 [8, 13, 16, 17, 18, 19, 22, 25, 27, 29]\n",
      "5949.875090 [8, 13, 16, 17, 18, 19, 22, 24, 27, 29]\n",
      "5942.875081 [8, 13, 16, 17, 18, 19, 22, 24, 25, 29]\n",
      "5948.875087 [8, 13, 16, 17, 18, 19, 22, 24, 25, 27]\n",
      "Best of the round: 5950.87510276285 [8, 13, 16, 17, 18, 22, 24, 25, 27, 29]\n",
      "\n",
      "N models remained: 10\n",
      "5944.875111 [13, 16, 17, 18, 22, 24, 25, 27, 29]\n",
      "5947.875237 [8, 16, 17, 18, 22, 24, 25, 27, 29]\n",
      "5946.875057 [8, 13, 17, 18, 22, 24, 25, 27, 29]\n",
      "5948.875052 [8, 13, 16, 18, 22, 24, 25, 27, 29]\n",
      "5946.875040 [8, 13, 16, 17, 22, 24, 25, 27, 29]\n",
      "5947.874891 [8, 13, 16, 17, 18, 24, 25, 27, 29]\n",
      "5946.874932 [8, 13, 16, 17, 18, 22, 25, 27, 29]\n",
      "5950.875038 [8, 13, 16, 17, 18, 22, 24, 27, 29]\n",
      "5946.875029 [8, 13, 16, 17, 18, 22, 24, 25, 29]\n",
      "5947.875038 [8, 13, 16, 17, 18, 22, 24, 25, 27]\n",
      "Best of the round: 5950.875037835753 [8, 13, 16, 17, 18, 22, 24, 27, 29]\n",
      "\n",
      "N models remained: 9\n",
      "5946.875115 [13, 16, 17, 18, 22, 24, 27, 29]\n",
      "5946.875260 [8, 16, 17, 18, 22, 24, 27, 29]\n",
      "5948.874941 [8, 13, 17, 18, 22, 24, 27, 29]\n",
      "5948.874935 [8, 13, 16, 18, 22, 24, 27, 29]\n",
      "5948.874922 [8, 13, 16, 17, 22, 24, 27, 29]\n",
      "5946.874783 [8, 13, 16, 17, 18, 24, 27, 29]\n",
      "5941.874808 [8, 13, 16, 17, 18, 22, 27, 29]\n",
      "5943.874907 [8, 13, 16, 17, 18, 22, 24, 29]\n",
      "5945.874916 [8, 13, 16, 17, 18, 22, 24, 27]\n",
      "Best of the round: 5948.874941166913 [8, 13, 17, 18, 22, 24, 27, 29]\n",
      "\n",
      "N models remained: 8\n",
      "5945.875121 [13, 17, 18, 22, 24, 27, 29]\n",
      "5947.875292 [8, 17, 18, 22, 24, 27, 29]\n",
      "5940.874745 [8, 13, 18, 22, 24, 27, 29]\n",
      "5939.874734 [8, 13, 17, 22, 24, 27, 29]\n",
      "5938.874613 [8, 13, 17, 18, 24, 27, 29]\n",
      "5936.874625 [8, 13, 17, 18, 22, 27, 29]\n",
      "5938.874720 [8, 13, 17, 18, 22, 24, 29]\n",
      "5938.874734 [8, 13, 17, 18, 22, 24, 27]\n",
      "Best of the round: 5947.875291745064 [8, 17, 18, 22, 24, 27, 29]\n",
      "\n",
      "N models remained: 7\n",
      "5926.874982 [17, 18, 22, 24, 27, 29]\n",
      "5952.875275 [8, 18, 22, 24, 27, 29]\n",
      "5949.875259 [8, 17, 22, 24, 27, 29]\n",
      "5946.875077 [8, 17, 18, 24, 27, 29]\n",
      "5950.875100 [8, 17, 18, 22, 27, 29]\n",
      "5943.875254 [8, 17, 18, 22, 24, 29]\n",
      "5954.875264 [8, 17, 18, 22, 24, 27]\n",
      "Best of the round: 5954.875263694169 [8, 17, 18, 22, 24, 27]\n",
      "\n",
      "N models remained: 6\n",
      "5930.875004 [17, 18, 22, 24, 27]\n",
      "5952.875172 [8, 18, 22, 24, 27]\n",
      "5958.875161 [8, 17, 22, 24, 27]\n",
      "5947.875016 [8, 17, 18, 24, 27]\n",
      "5948.874995 [8, 17, 18, 22, 27]\n",
      "5943.875104 [8, 17, 18, 22, 24]\n",
      "Best of the round: 5958.8751605479 [8, 17, 22, 24, 27]\n",
      "\n",
      "N models remained: 5\n",
      "5927.875003 [17, 22, 24, 27]\n",
      "5944.874839 [8, 22, 24, 27]\n",
      "5944.874791 [8, 17, 24, 27]\n",
      "5938.874693 [8, 17, 22, 27]\n",
      "5940.874801 [8, 17, 22, 24]\n",
      "Best of the round: 5944.874839313047 [8, 22, 24, 27]\n",
      "\n",
      "N models remained: 4\n",
      "5922.874884 [22, 24, 27]\n",
      "5929.874110 [8, 24, 27]\n",
      "5913.873831 [8, 22, 27]\n",
      "5916.873857 [8, 22, 24]\n",
      "Best of the round: 5929.874109566711 [8, 24, 27]\n",
      "\n",
      "N models remained: 3\n",
      "5927.874626 [24, 27]\n",
      "5881.871710 [8, 27]\n",
      "5856.871694 [8, 24]\n",
      "Best of the round: 5927.874625934677 [24, 27]\n",
      "\n",
      "N models remained: 2\n",
      "5892.873194 [27]\n",
      "5866.873835 [24]\n",
      "Best of the round: 5892.873193505994 [27]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "result_list_none = search_mean_subset(cv, idx_models, is_per_area=False, is_log=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[5960.875091343384,\n",
       "  [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5960.875082147267,\n",
       "  [1, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5960.87508175082,\n",
       "  [1,\n",
       "   3,\n",
       "   6,\n",
       "   8,\n",
       "   12,\n",
       "   13,\n",
       "   14,\n",
       "   15,\n",
       "   16,\n",
       "   17,\n",
       "   18,\n",
       "   19,\n",
       "   21,\n",
       "   22,\n",
       "   24,\n",
       "   25,\n",
       "   26,\n",
       "   27,\n",
       "   28,\n",
       "   29]],\n",
       " [5959.875104006798,\n",
       "  [3, 6, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5959.875103957208,\n",
       "  [6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5959.875091803407,\n",
       "  [1, 3, 8, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5959.875089024389,\n",
       "  [1, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5958.8751605479, [8, 17, 22, 24, 27]],\n",
       " [5958.875098731293,\n",
       "  [1, 3, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]],\n",
       " [5958.875097569281,\n",
       "  [1, 3, 6, 8, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]]]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(result_list_none[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score and subset: 5960.875091 [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n",
      "model-01 5870.873059\n",
      "model-03 5877.873452\n",
      "model-06 5886.873769\n",
      "model-08 5514.858826\n",
      "model-12 5486.856963\n",
      "model-13 5506.858055\n",
      "model-15 5900.873836\n",
      "model-16 5907.874126\n",
      "model-17 5905.874165\n",
      "model-18 5908.874297\n",
      "model-19 5911.874156\n",
      "model-21 5758.870702\n",
      "model-22 5752.870671\n",
      "model-24 5866.873835\n",
      "model-25 5908.874238\n",
      "model-26 5918.873998\n",
      "model-27 5892.873194\n",
      "model-28 5901.874202\n",
      "model-29 5884.873848\n"
     ]
    }
   ],
   "source": [
    "best_models_subset_none = result_list_none[0][1]\n",
    "print('Best score and subset:', '%.6f'%result_list_none[0][0], best_models_subset_none)\n",
    "for idx_model in best_models_subset_none:\n",
    "    print('model-%02d'%idx_model, '%.6f'%scores_models[idx_model])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute submission for mean log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict for mean log with [3, 5, 8, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 24, 26, 27, 28, 29]\n"
     ]
    }
   ],
   "source": [
    "print('Predict for mean log with', best_models_subset_log)\n",
    "test['total_price'] = predict_mean_subset(test, best_models_subset_log, is_per_area=False, is_log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/matplotlib/axes/_axes.py:6571: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHVNJREFUeJzt3Xt4VfWd7/H3F4wGBEGgZqo4E+qlB7yhBLBandB6ibUP2KktltYjnQKdqfTo1MmUnlan2ulzqDnjCNZLOdTDODaixerhOTAjxhr1TL0BxQsGBXnSGtCgWJEoUQLf88daWeyEhOyd7LXXvnxez8PjXvfvb2e7v/v3+631+5m7IyIiAjAo6QBERCR/KCmIiEhESUFERCJKCiIiElFSEBGRiJKCiIhElBRERCSipCAiIhElBRERiRyWdACZGjNmjFdWVmZ83AcffMCRRx6Z/YDynMpdWlTu0pJJudetW/eOu3+ir/0KLilUVlaydu3ajI9rbGykuro6+wHlOZW7tKjcpSWTcpvZH9LZT81HIiISUVIQEZGIkoKIiEQKrk9BRIrT3r17aWlpob29PeNjR4wYQVNTUwxR5beeyl1eXs7YsWMpKyvr1zmVFEQkL7S0tDB8+HAqKysxs4yO3b17N8OHD48psvzVvdzuzs6dO2lpaWHcuHH9Oqeaj0QkL7S3tzN69OiME4IcYGaMHj26X7WtTkoKIpI3lBAGbqDvoZKCiIhE1KcgInmpcsGqrJ6veeGlh9z+3nvvUV9fz3e+852Mz/2FL3yB+vp6Ro4cmdb+Dz/8MCeffDITJkzI+FpxU01BCl/9zOCfyAC899573HHHHT1u6+joOOSxq1evTjshQJAUXnnllX5dK25KCiIiwIIFC3j99deZOHEitbW1NDY2ct555zF9+vToF/1ll13GpEmTOOWUU1iyZEl0bGVlJe+88w7Nzc2MHz+euXPncsopp3DRRRexZ8+eLtf53e9+x8qVK6mtrWXixIm8/vrrVFdXc+2111JVVcWiRYuYPXs2K1asiI4ZNmxY9Lquro7Jkydz+umn89Of/jTr74OSgogIsHDhQk444QQ2bNhAXV0dAOvXr2fRokW89tprANx9992sW7eOtWvXsnjxYnbu3HnQeTZv3szVV1/Nxo0bGTlyJA8++GCX7eeccw7Tp0+nrq6ODRs2cMIJJwDw8ccfs3btWq677rpeY1yzZg2bN2/mueeeY8OGDWzYsIEnn3wyW28BoD4FEZFeTZkypcv9/osXL+ahhx4C4I033mDz5s2MHj26yzHjxo1j4sSJAEyaNInm5ua0rjVzZt9NoGvWrGHNmjWceeaZALz//vts3ryZ888/P61rpENJQUSkF6nDUjc2NtLQ0MDTTz/N0KFDqa6u7vF5gCOOOCJ6PXjw4IOaj9K51mGHHcb+/fsB2L9/Px9//DEQPJz2gx/8gG9/+9tAPA/tqflIRAQYPnw4u3fv7nX7rl27OProoxk6dCibNm3imWeeie1alZWVrFu3DoCVK1eyd+9eAC6++GLuvvtu2traANi+fTs7duzodxw9UU1BRPJSX7eQpsrGL+bRo0dz7rnncuqpp3LJJZdw6aVdr19TU8Ndd93F+PHj+fSnP83ZZ5/d72tdccUVzJ07l8WLF3fpUO40d+5cZsyYwRlnnEFNTU1Ui7joootoamriM5/5DABDhgzhvvvu45hjjul3LN2Zu2ftZLlQVVXlmmQnfSVR7s7bUWfdH60qiXL3oJDL3dTUxPjx4/t1rMY+6qqn99LM1rl7VV/nVPORFKd3t+rZBZF+UFIQEZGIkoKIiESUFEREJKKkICIiESUFERGJ6DkFEclPGdw9NqSjAw7r4+ss5Zblngxk6Oxbb72VefPmMXTo0IyPzTeqKUjRqVywit3tHTQ0tSYdihSQQw2d3Zdbb72VDz/8MMsRJUM1BRERug6dfeGFF3LMMcfwwAMP8NFHH/GlL32JG2+8kQ8++ICvfvWrtLS0sG/fPq6//npaW1vZvn0706ZNY8yYMTz++ONJF2VAlBRERAiGzn755ZfZsGEDa9asYcWKFTz33HO4O9OnT+fJJ5/k7bff5thjj2XVqmBWuF27djFixAhuueUWHn/8ccaMGZNwKQZOzUciIt2kDlF91llnsWnTJjZv3sxpp53Go48+yve//32eeuopRowYkXSoWRdrTcHMaoBFwGBgqbsv7GW/LwMrgMnunvnARlLyGppamZPlOX2ldHUfojrV+vXrWb16NT/60Y/4/Oc/zw033JBAhPGJraZgZoOB24FLgAnA18zsoFmqzWw4cA3wbFyxiIj0JXU46+5DVG/bto0dO3awfft2hg4dyje+8Q1qa2tZv379QccWujhrClOALe6+FcDMlgMzgO6zVf8E+BlQG2MsIlJo+riFNNWeGIbOnjVrVjRE9bBhw7j33nvZsmULtbW1DBo0iLKyMu68804A5s2bR01NDccee2zBdzTHNnS2mV0O1Lj7nHD5SmCqu89P2ecs4Ifu/mUzawT+vqfmIzObB8wDqKiomLR8+fKM42lra+sy+XWpKIlyv7uV3e0dNHtFtOrPh+xlUPu7DD/25AQDy71C/nuPGDGCE088sV/H7tu3j8GDB2c5ovzXW7m3bNnCrl27uqybNm1aWkNnJ3b3kZkNAm4BZve1r7svAZZAMJ9Cf8aLL+Rx5geiJMpdfycNTa2MB+bsDSqct53eypCmeqpnNSYaWq4V8t+7qamp37/2NZ9CV+Xl5dE8zpmK8+6jbcDxKctjw3WdhgOnAo1m1gycDaw0sz4zmYiIxCPOpPA8cJKZjTOzw4ErgJWdG919l7uPcfdKd68EngGm6+6jPFI/s/AnqimGMpSQQpsJMh8N9D2MLSm4ewcwH3gEaAIecPeNZnaTmU2P67oiUpjKy8vZuXOnEsMAuDs7d+6kvLy83+eItU/B3VcDq7ut6/GmXnevjjMWEclvY8eOpaWlhbfffjvjY9vb2wf0RVioeip3eXk5Y8eO7fc5NcyFDExn00wGtw8mon5m/sdY4srKyhg3bly/jm1sbOx3x2ohi6PcGuZCskPt9iJFQTUFyb4c1R4qw2EtlpZpiGyRbFFNQUREIkoKIiISUVKQwqK+C5FYKSlIZvQwmEhRU1IQEZGIkoLklmoaInlNt6RKQcl0hrWGpgO3q14QR0AiRUZJQYrK0rI6APYw6+CNhfL0tUiC1HwkBWdpWV305d8vasIS6ZWSgoiIRNR8JAVrQLUFEemRagoyYA1NrTQ0tUZjER1ETTUiBUNJQUREImo+koyl3haqEUpFiotqCiIiElFSEBGRiJqPpGR0ebp5fEWCkYjkLyUFyarKBauifgYNKyFSeNR8JCIiEdUUJDapcyhfkPKsQurdS80LL00kNhHpmZKC9IueJhYpTmo+kr5pADmRkqGaguRE6p0/IpK/VFMQEZGIagqSls7O4UyGtVC/g0jhUU1BREQiSgoiIhJRUhARkYiSgoiIRNTRLFnT747lzmcgZt2fvWBEpF9UU5BAGg+o6W4ikeKnmoIkIkow9fckG4iIdKGagoiIRJQUREQkEmvzkZnVAIuAwcBSd1/YbfvfAFcD+4A2YJ67vxJnTHJoncNdAxk9vdxfXWZD6yWOVEvLYg5IpMTFVlMws8HA7cAlwATga2Y2odtu9e5+mrtPBG4GbokrHklPop3JORyNtaGplcoFq3pNPiKlKs7moynAFnff6u4fA8uBGak7uPv7KYtHAh5jPCIi0oc4m4+OA95IWW4BpnbfycyuBr4HHA58LsZ4RESkD+Yez49zM7scqHH3OeHylcBUd5/fy/6zgIvd/aoets0D5gFUVFRMWr58ecbxtLW1MWzYsIyPK3Rpl/vdrQDsbu+g2SsAqLTczoEwvDz8jTLqU7y0bVeP+6Qb0/7yUQxqf/eQ+3SW87TjRqQfZJ7T57y0ZFLuadOmrXP3qr72i7OmsA04PmV5bLiuN8uBO3va4O5LgCUAVVVVXl1dnXEwjY2N9Oe4Qpd2ueuDt76hqZV/3lsLwNKy+hgjO1j1+Irwxf3MjuZ3Dvo45mQY057xsxjSdOh9O8vZ/PXqfkSbn/Q5Ly1xlDvOpPA8cJKZjSNIBlcAs1J3MLOT3H1zuHgpsBmRHEvtbG5eeGmCkYgkL7ak4O4dZjYfeITgltS73X2jmd0ErHX3lcB8M7sA2Av8CTio6UhERHIn1ucU3H01sLrbuhtSXl8T5/VFRCQzeqJZREQiGhBP8kbn081z9ECZSGJUUxARkYhqCnIQzZsgUrpUUxARkYhqCqWsfmbBTIGp2otIbqimICIiESUFERGJKCmIiEhESUFERCJKCiIiElFSEBGRiJKCiIhElBQkb8X9bMLSsjo9/yDSjZKC5J1cf1krMYgckPYTzWZ2BnBeuPiUu78QT0giIpKUtJKCmV0DzAV+E66618yWuPttsUUmkkOdtYXKBQfWaWpOKUXp1hS+BUx19w8AzOxnwNOAkoKISBFJt0/BgH0py/vCdSIiUkTSrSn8b+BZM3soXL4M+GU8IYmISFLSSgrufouZNQKfDVd9091/H1tUklOVC1axtKw16TDyytKyOqi/p2CGFhfJlkMmBTM7yt3fN7NRQHP4r3PbKHd/N97wJHb1M5UQRCTSV02hHvgisA7wlPUWLn8qprhERCQBh0wK7v7F8L/jchOOiIgkKa27j8zssXTWiYhIYeurT6EcGAqMMbOjOXAb6lHAcTHHJpJzGvJCSl1ffQrfBq4FjiXoV+hMCu8DP48xLhERSUBffQqLgEVm9l0NaSEiUvzSfU7hNjM7FZgAlKesvyeuwEREJPfS7Wj+R4Jxjm4DpgE3A9NjjEskP9TPTDoCkZxKd+yjy4HPA2+5+zeBM4ARsUUlkgcamlppaGqlcsGqpEMRyZl0k0K7u+8HOszsKGAHcHx8YYmISBL67FMwMwNeNLORwP8iuAupjWDobBERKSJ9JgV3dzOb4u7vAXeZ2X8AR7n7i/GHJyIiuZTu0NnrzWyyuz/v7s1xBiS5oZFRRaQn6SaFqcDXzewPwAeEA+K5++mxRSYiIjmXblK4ONYoRPKY5laQUpLW3Ufu/oee/vV1nJnVmNmrZrbFzBb0sP17ZvaKmb1oZo+Z2V/0pxAiIpId6d6SmjEzGwzcDlxC8CT018xsQrfdfg9Uhc1QKwgeihMRkYTElhSAKcAWd9/q7h8Dy4EZqTu4++Pu/mG4+AwwNsZ4RESkD3EmheOAN1KWWzj0cNvfAv49xnhERKQP5u5979WfE5tdDtS4+5xw+UpgqrvP72HfbwDzgb9094962D4PmAdQUVExafny5RnH09bWxrBhwzI+rtD1Vu6Xtu2i0or3ltT95aMY1J69KcSHlx8Go/J/9ll9zktLJuWeNm3aOnev6mu/dO8+6o9tdB0KY2y4rgszuwD4Ib0kBAB3XwIsAaiqqvLq6uqMg2lsbKQ/xxW63so9e8EqlpbV5z6gHNkzfhZDmrJXvg5gzt5aAJoXXpq182abPuelJY5yx9l89DxwkpmNM7PDgSuAlak7mNmZwC+A6e6+I8ZYREQkDbElBXfvIGgSegRoAh5w941mdpOZdQ67XQcMA35tZhvMbGUvpxPJPxpWW4pQnM1HuPtqYHW3dTekvL4gzuuLiEhm4mw+EhGRAhNrTUGkmCwtqwtf5W9Hs8hAKSmUEM0gJiJ9UfORiIhEVFMoQQeaQUREulJSKCFKBiLSFzUfiYhIRDUFkQx1dtgvLWtFD9pIsVFNQUREIkoKIiISUfNRKdAYPSKSJtUURAaifqaSrhQVJQUREYmo+ajIvbRtFx2bineGNRHJLtUUREQkoqQgIiIRJQUREYkoKYiISERJQUREIkoKItmg5xWkSOiWVJEsaWhqheurAZizt5bmhZq2UwqPagoiIhJRTUEkQ5qsSIqZkoJITDrnXQDUlCQFQ81HIiISUU1BZAAamnofV6qzmWnO3tpchSMyYKopiIhIRDWFIpTaln3b6RohVUTSp5qCiIhElBRERCSipCAiIhElBRERiaijuUh13g65h1kJRyIihUQ1BZEYaCgMKVRKCiK5oqG1pQAoKYjkQOWCVTQ0tXZ5hkQkHykpFAtN8iIiWRBrUjCzGjN71cy2mNmCHrafb2brzazDzC6PMxaRpCwtqzu4j0FJXPJUbEnBzAYDtwOXABOAr5nZhG67/RGYDdTHFYeIiKQvzltSpwBb3H0rgJktB2YAr3Tu4O7N4bb9McYhIiJpirP56DjgjZTllnCdiIjkKXP3eE4c9BHUuPuccPlKYKq7z+9h32XA/3X3Fb2cax4wD6CiomLS8uXLM46nra2NYcOGZXxcwXh3a/Ryd3tH9Hp/+SgGtb+bRESJytdyN3sFpx034sDfa9Snsnr+ov+c90Ll7tu0adPWuXtVX/vF2Xy0DTg+ZXlsuC5j7r4EWAJQVVXl1dXVGZ+jsbGR/hxXMOrvjF6mTvyyZ/wshjSVXpdNvpZ7PNCx6cDyBT9pzOr5i/5z3guVO3viTArPAyeZ2TiCZHAFaMyFuBxqBjARkXTF1qfg7h3AfOARoAl4wN03mtlNZjYdwMwmm1kL8BXgF2a2Ma54RESkb7EOiOfuq4HV3dbdkPL6eYJmJRERyQN6ollERCJKCiIiElFSEBGRiJKCiIhENPNaAeschlkTuohItigpiCQoNbFn+0E2kf5Q85GIiESUFEREJKKkICIiESUFERGJKCmIiEhEdx+J5InOO5EAmhdemmAkUsqUFApM6heHiEi2qflIREQiqikUMD3JXFw6/55z9tYmHImUMiUFkQT1lNiXltVB/T0w6/4EIpJSp+YjERGJKCmI5Kv6mcE/kRxS85FIHmpoao1eX5BgHFJ6VFMoJPUz1bksIrFSUhARkYiajwqQagsiEhclhQJwYCKW1j72FBEZGDUfiYhIRDUFkTyngfIkl1RTEBGRiGoKear7aKjqXC5dS8vqNB6S5IxqCiKFRE85S8xUU8hTqhlIqgOfh4pE45Dip5qCiIhEVFPIN2oaEJEEqaYgUkAamlppaGrVtKwSG9UU8kjlglV6allEEqWaQh5R57KkK5idTU2Nkn1KCiIFqrMZSU1Jkk1qPkpY0GSkGoJkQf1MKL8YqE46EilgSgoJ0C87EclXsTYfmVmNmb1qZlvMbEEP248ws/vD7c+aWWWc8eSbpWV1qiVIVlQuWEVDUyu72zuSDkUKXGw1BTMbDNwOXAi0AM+b2Up3fyVlt28Bf3L3E83sCuBnQNH0nqU2DaWOXaNEINnU/fPUcH01wEHjJWmEVUlHnM1HU4At7r4VwMyWAzOA1KQwA/hx+HoF8HMzM3f3GONKlBKCZNOhPk+p2+bsrT2o/6ozaUTJovNupln3Zz9QKRhxJoXjgDdSlluAqb3t4+4dZrYLGA28E2Nc/TaQvgAlA0nSoT5/nTWLyPXVOatl9DZXhOaQSE5BdDSb2TxgXrjYZmav9uM0Y0gg2VyY6wse5IlEyp08lTuNfYHePqNPdFmynw0kpvT0do00r12if++Myv0X6ewUZ1LYBhyfsjw2XNfTPi1mdhgwAtjZ/UTuvgRYMpBgzGytu1cN5ByFSOUuLSp3aYmj3HHeffQ8cJKZjTOzw4ErgJXd9lkJXBW+vhz4bTH3J4iI5LvYagphH8F84BFgMHC3u280s5uAte6+Evgl8G9mtgV4lyBxiIhIQmLtU3D31cDqbutuSHndDnwlzhhSDKj5qYCp3KVF5S4tWS+3qbVGREQ6aUA8ERGJFGVSMLO7zWyHmb2csm6UmT1qZpvD/x6dZIxx6KXcdWa2ycxeNLOHzGxkkjHGoadyp2y7zszczMYkEVuceiu3mX03/JtvNLObk4ovLr18ziea2TNmtsHM1prZlCRjjIOZHW9mj5vZK+Hf9ppwfVa/24oyKQDLgJpu6xYAj7n7ScBj4XKxWcbB5X4UONXdTwdeA36Q66ByYBkHlxszOx64CPhjrgPKkWV0K7eZTSMYKeAMdz8F+J8JxBW3ZRz8974ZuNHdJwI3hMvFpgO4zt0nAGcDV5vZBLL83VaUScHdnyS4mynVDOBfw9f/ClyW06ByoKdyu/sad+8cJe0ZgudFikovf2+AfwH+ASjKjrNeyv23wEJ3/yjcZ0fOA4tZL+V24Kjw9Qhge06DygF3f9Pd14evdwNNBKNCZPW7rSiTQi8q3P3N8PVbQEWSwSTkr4F/TzqIXDCzGcA2d38h6Vhy7GTgvHDU4SfMbHLSAeXItUCdmb1BUDsqxhpxJBxR+kzgWbL83VZKSSESPiBXlL8ee2NmPySofv4q6VjiZmZDgf9O0IxQag4DRhE0L9QCD5iZJRtSTvwt8HfufjzwdwTPQBUlMxsGPAhc6+7vp27LxndbKSWFVjP7JED436KrVvfGzGYDXwS+XiJPjJ8AjANeMLNmgiaz9Wb2Z4lGlRstwG888Bywn2B8nGJ3FfCb8PWvCUZpLjpmVkaQEH7l7p3lzep3WyklhdQhNa4C/k+CseSMmdUQtKtPd/cPk44nF9z9JXc/xt0r3b2S4IvyLHd/K+HQcuFhYBqAmZ0MHE5pDBS3HfjL8PXngM0JxhKLsMb3S6DJ3W9J2ZTd7zZ3L7p/wH3Am8Begi+EbxEMyf0YwYelARiVdJw5KvcWguHJN4T/7ko6zlyUu9v2ZmBM0nHm6O99OHAv8DKwHvhc0nHmqNyfBdYBLxC0s09KOs4Yyv1ZgqahF1P+f/5Ctr/b9ESziIhESqn5SERE+qCkICIiESUFERGJKCmIiEhESUFERCJKCpJ3zKxtgMevMLNPZXjM35jZfx3ANe8ys3P7e3yczOx8M1tvZh1mdnnK+k+Y2X8kGZvkHyUFKSpmdgow2N23ZnDMYe5+l7vfM4BLn00w4GBa1xvAdbqfq9rMlvWx2x+B2UB96kp3fxt4M1+TmSRDSUHylgXqzOxlM3vJzGaG6weZ2R3hnAGPmtnqlF/AXyfliU4zazOzfwnHn3/MzD4Rrm80s1vNbC1wjZn92Mz+Ptx2opk1mNkL4S/sE8L1tWb2fDg3xY0p1xgPvObu+8xsbrjPC2b2YDgOE2a2LKxNPAvcbGZHhvMCPGdmvw8H8MPMKs3sqfC6683snIG+j+7e7O4vEgx50d3D4XsmAigpSH77K2AicAZwAcEomJ8M11cCE4Argc+kHHMuwZOtnY4E1nowt8ATwD+mbDvc3avc/Z+7XfdXwO3ufgZwDsGv6YuAkwjG1JkITDKz88P9LwE6m2F+4+6Tw2ObCJ627TQWOMfdvwf8EPitu08hGJaizsyOJBi35kJ3PwuYCSxO763qt7XAeTFfQwpI1qqxIjH4LHCfu+8jGPTrCWByuP7X7r4feMvMHk855pPA2ynL+4H7w9f3cmDQNFLWR8xsOHCcuz8E4O7t4fqLCCbs+X246zCCJPEkcDHwzXD9qWb2T8DIcJ9HUk7/67AshOea3lk7AcqBPycYw+fnZjYR2EcwFPZBwhrHEeE1RpnZhnDT9939kZ6O6cUO4NgM9pcip6QgxWYPwRdsb1LHdfkgg/Ma8D/c/RddVgbNQyPdvXNSl2XAZe7+Qjg6bXUv1zPgy+7+arfz/RhoJagdDQLaeyyE+9Rw/2pgtrvPzqAsqcoJ3jMRQM1Hkt+eAmaa2eCwL+B84DngP4Evh30LFXT94m0CTkxZHgR09jfMAv7foS7owYxWLWZ2GYCZHRF+8T8C/HU4lj1mdpyZHUPQ9JNaUxlO0NxUxqHb6h8Bvts514GZnRmuHwG8GdaCrgQGHyreLDiZYPA8EUBJQfLbQwQjQr4A/Bb4Bw+Gv36QYHTMVwiahNYDu8JjVnHwr/MpFkzy/jngpjSueyXw38zsReB3wJ+5+xqCu3eeNrOXgBUECSC1PwHgeoJROv8T2HSIa/wEKANeNLON4TLAHcBVZvYC8F/IrDbTIzObbGYtwFeAX4TX6zSN4D0TAdAoqVKYzGyYu7eZ2WiC2sO57v6WmQ0h+OV+bng3UJu7D4sxjvXAVHffG9c14mRmTwIz3P1PScci+UFJQQqSmTUSdOYeDtzs7stStl1MMBHJH+NOCoUsbJI7190fTjoWyR9KCiIiElGfgoiIRJQUREQkoqQgIiIRJQUREYkoKYiISERJQUREIv8fkFlSHUEy3bMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHHpJREFUeJzt3X94VdWd7/H3V4xGBFGhZKo4E7TaGxRFidjW1glT1Fj7gL1jxUa9ZaZAOxWvzti06R1/VPvHpc2tI8xYLTdlHMdm0GL18gzMFXFIdW79wY9iFaJCedIabGPFgRJKhMj3/nF2NjvhJDlJzj77/Pi8nsfH/WPtvb8rJOd71lp7r23ujoiICMAxSQcgIiL5Q0lBRERCSgoiIhJSUhARkZCSgoiIhJQUREQkpKQgIiIhJQUREQkpKYiISOjYpAMYqgkTJnhlZWXaffv37+fEE0/MbUAJK7U6q77Fr9TqnKv6btq06V13/9Bg5QouKVRWVrJx48a0+1paWqipqcltQAkrtTqrvsWv1Oqcq/qa2a8yKafuIxERCSkpiIhISElBRERCBTemICLF6dChQ7S3tzNu3DhaW1uTDidnsl3f8vJyJk2aRFlZ2bCOV1IQkbzQ3t7O2LFjGT9+PCeddFLS4eTMvn37GDt2bFbO5e7s3r2b9vZ2Jk+ePKxzqPtIRPJCV1cX48ePx8ySDqVgmRnjx4+nq6tr2OdQUhCRvKGEMHIj/RkqKYiISEhjCiKSlyobVmf1fG2Lrx5w/549e2hubuarX/3qkM/9mc98hubmZk4++eSMyj/11FOcc845TJkyZcjXiptaCpKIyobV4X8i+WDPnj18//vfT7uvu7t7wGPXrFmTcUKAVFLYtm3bsK4VNyUFERGgoaGBX/7yl0ybNo36+npaWlr41Kc+xezZs8Nv9Ndccw3Tp0/n3HPPZdmyZeGxlZWVvPvuu7S1tVFVVcWCBQs499xzueKKKzhw4ECv6/zsZz9j1apV1NfXM23aNHbu3ElNTQ233XYb1dXVLFmyhHnz5rFy5crwmDFjxoTLjY2NXHzxxZx//vncfffdWf85KCmIiACLFy/mrLPOYsuWLTQ2NgKwefNmlixZwptvvgnA8uXL2bRpExs3bmTp0qXs3r37qPNs376dm2++ma1bt3LyySfzxBNP9Nr/iU98gtmzZ9PY2MiWLVs488wzATh48CAbN27k9ttv7zfGtWvXsn37dl5++WW2bNnCpk2beO6557L1IwA0piAi0q8ZM2b0ut9/6dKlPPnkkwC89dZbbN++nfHjx/c6ZvLkyUybNg2A6dOn09bWltG15s6dO2iZtWvXsnbtWi688EIAOjs72b59O5dddllG18iEkoKISD+iU1q3tLSwbt06XnjhBUaPHk1NTU3a5wGOP/74cHnUqFFHdR9lcq1jjz2Ww4cPA3D48GEOHjwIpB5O++Y3v8mXv/zlYdUnE+o+EhEBxo4dy759+/rdv3fvXk455RRGjx7N66+/zosvvhjbtSorK9m0aRMAq1at4tChQwBceeWVLF++nM7OTgB27drFO++8M+w40lFLQUTy0mC3kGbb+PHjufTSSznvvPO46qqruPrq3tevra3loYceoqqqio9+9KN87GMfG/a1rr/+ehYsWMDSpUt5+OGHj9q/YMEC5syZwwUXXEBtbW3YirjiiitobW3l4x//OJAagH700UeZOHHisGPpy9w9ayfLherqatdLdo4o1DpHb0Udyh9/odZ3uEqpvq2trVRVVWV1LqBCEEd9e36WUWa2yd2rBztW3UciIhJSUhARkZCSgoiIhJQUREQkpKQgIiIhJQUREQnpOQURyU/Ng0/7MCR1jw24eyRTZ99///0sXLiQ0aNHDze6vKGWgogIA0+dPZj777+fP/zhD1mOKBlqKYiI0Hvq7Msvv5yJEyfy+OOP8/777/O5z32Oe+65h/3793PdddfR3t7OBx98wJ133klHRwdvv/02M2fOZMKECaxfvz7pqoyIkoKICKmps1977TW2bNnC2rVrWblyJS+//DLuzuzZs3nuuef43e9+x2mnncbq1akn8vfu3cu4ceO47777WL9+PRMmTEi4FiOn7iMRkT6iU1RfdNFFvP7662zfvp2pU6fyzDPP8I1vfIPnn3+ecePGJR1q1qmlICLSx0BTVG/evJk1a9Zwxx138OlPf5q77rorgQjjo5aCiAi9p7Pub4rqt99+m9GjR3PjjTdSX1/P5s2bjzq20KmlICL5aZBbSLOt79TZdXV1R01RvWPHDurr6znmmGMoKyvjwQcfBGDhwoXU1tZy2mmnaaBZRKRYNDc391q/9dZbe62fddZZXHnllUcdd8stt3DLLbfEGluuqPtIRERCSgoiIhKKNSmYWa2ZvWFmO8ysYYByf25mbmaDvhVIRIpXob0JMh+N9GcYW1Iws1HAA8BVwBTgC2Y2JU25scCtwEtxxSIi+a+8vJzdu3crMYyAu7N7927Ky8uHfY44B5pnADvcfSeAma0A5gDb+pT7NvAdoD7GWEQkz02aNIn29nb27Nkzog+1QtPV1ZXV+paXlzNp0qRhHx9nUjgdeCuy3g5cEi1gZhcBZ7j7ajNTUhApYWVlZUyePJmWlhYuvPDCpMPJmXyrb2K3pJrZMcB9wLwMyi4EFgJUVFTQ0tKStlxnZ2e/+4pVodb59qnd4fJQ4i/U+g5XqdUXSq/O+VbfOJPCLuCMyPqkYFuPscB5QIuZAfwRsMrMZrv7xuiJ3H0ZsAygurraa2pq0l6wpaWF/vYVq0Kt87yG1eFy2w01GR9XqPUdrlKrL5RenfOtvnHefbQBONvMJpvZccD1wKqene6+190nuHulu1cCLwJHJQQREcmd2FoK7t5tZouAp4FRwHJ332pm9wIb3X3VwGeQotU8l6ayDuYfqu+1LZTj6Q1E5IhYxxTcfQ2wps+2tFMKuntNnLGIiMjgNPeR5Ea237crIrHQNBciIhJSUhARkZCSgoiIhJQUREQkpIFmyYrK6MNoi69OMBIRGQklBYlVT7JoKutgVlVFwtGIyGDUfSQiIiG1FCQ+wZPLIlI4lBQkcZUNq3slj/lBl5PGJkRyT0lBsi46jjCQprLGXIQjIkOgMQUREQkpKYiISEhJQUREQkoKIiIS0kCzZJUGj0UKm1oKIiISUlIQEZGQuo8kK9RtJFIc1FIQEZGQWgoyJJoiW6S4qaUgIiIhJQUREQmp+0gGFe0ySjqG26d2U5NsKCJFTS0FEREJKSmIiEhI3UcyqOgzCPMP1efues2PQN1jsV9PRI5QS0FEREJqKUjOrGvV+5pF8p1aCiIiElJLQYaveW7SEYhIlqmlICIiISUFEREJKSmIiEhIYwoyJL3fm1AR/wWDcYumso6cPCMhUurUUhARkVCsLQUzqwWWAKOAJndf3Gf/V4CbgQ+ATmChu2+LMybpn96VICKxtRTMbBTwAHAVMAX4gplN6VOs2d2nuvs04LvAfXHFIyIig4uz+2gGsMPdd7r7QWAFMCdawN1/H1k9EfAY4xERkUGYezyfw2Z2LVDr7vOD9ZuAS9x9UZ9yNwN/AxwH/Jm7b09zroXAQoCKiorpK1asSHvNzs5OxowZk9V65Lts1vnVXXvD5amnjwuX9739ZtryY8uP9D7u6+rOSgwDnb/NK6g4ASaeOm6Ao4qLfqeLX67qO3PmzE3uXj1YucTvPnL3B4AHzKwOuAP4Ypoyy4BlANXV1V5TU5P2XC0tLfS3r1hls87zomMKNxw557o7v5W2fDQNnJCVCHqrqTpyd9O61g6+d6ie26d2c12kvsU+DqLf6eKXb/WNMynsAs6IrE8KtvVnBfBgjPFIBo7ccpq/H7D58CY4kWIVZ1LYAJxtZpNJJYPrgbpoATM7O9JddDVwVNeRyEB6PzeRv4lMpFDElhTcvdvMFgFPk7oldbm7bzWze4GN7r4KWGRms4BDwH+SputIpEdTWSMHrA44PelQRIpWrGMK7r4GWNNn212R5VvjvL6MgGZAFSlJeqJZRERCSgoiIhJSUhARkVDizymIDFXvO45EJJvUUhARkZCSgoiIhJQUREQkpKQgIiIhJQUREQkpKYiISCjjW1LN7ALgU8Hq8+7+SjwhiYhIUjJKCmZ2K7AA+Emw6VEzW+bufx9bZFLy1rV2JB2CSMnJtKXwJVJvTdsPYGbfAV4AlBSKQTD5XVOZPoRFSl2mYwoGfBBZ/yDYJiIiRSTTlsI/Ai+Z2ZPB+jXAD+MJSUREkpJRUnD3+8ysBfhksOkv3P3nsUUlIiKJGDApmNlJ7v57MzsVaAv+69l3qru/F294IiKSS4O1FJqBzwKbAI9st2D9zJjiEhGRBAyYFNz9s8H/J+cmHBERSVJGdx+Z2bOZbBMRkcI22JhCOTAamGBmp3DkNtSTgNNjjk1ERHJssDGFLwO3AaeRGlfoSQq/B/4hxrgkRyobVhfPQ2vBQ3gA1D2WXBwiBWywMYUlwBIzu0VTWoiIFL9Mn1P4ezM7D5gClEe2PxJXYCIiknuZToh3N1BDKimsAa4C/gNQUigk0e6VQNF0HdF7Ar1ZCcYhUsgynebiWuAC4Ofu/hdmVgE8Gl9YEifNPioi/cl0Qrwudz8MdJvZScA7wBnxhSUiIkkYtKVgZgb8wsxOBv43qbuQOklNnS2Sl9bdWQPA/EP1tC2+OtlgRArIoEnB3d3MZrj7HuAhM/u/wEnu/ov4wxMRkVzKdExhs5ld7O4b3L0tzoAky9IMLouI9CfTpHAJcIOZ/QrYTzAhnrufH1tkMjJKBiIyDJkmhStjjUJERPJCpg+v/SruQGRoKhtWA3D71G5qkg1FRIpIprekiohICci0+0gKRE8Loqmsg1lVFQlHk396fj6AblUVSSPWloKZ1ZrZG2a2w8wa0uz/GzPbZma/MLNnzexP4oxHSk9TWWNq0F0D7yIZiS0pmNko4AFS8yRNAb5gZlP6FPs5UB3cxbQS+G5c8YiIyODi7D6aAexw950AZrYCmANs6yng7usj5V8EbowxnpKm+Y5EJBNxdh+dDrwVWW9n4Le1fQn4txjjERGRQZi7x3Nis2uBWnefH6zfBFzi7ovSlL0RWAT8qbu/n2b/QmAhQEVFxfQVK1akvWZnZydjxozJXiXy2Ku79gJQcQJMPHXcUdsrbWQtg7HlRxqR+7q6R3SubDpcfirHdL03rGPbvPfA+9TTx/VTMn+U0u90j1Krc67qO3PmzE3uXj1YuTi7j3bReybVScG2XsxsFvC39JMQANx9GbAMoLq62mtqatJesKWlhf72FZt5kecUruupc/Ncul/PTjdRTeTOpXzqejpQVccJrc3DOrYqsjz/UD1tN9RkJaY4ldLvdI9Sq3O+1TfOpLABONvMJpNKBtcDddECZnYh8ANSLYp3YoylaFVaRyx31uRTIhCR3IltTMHdu0l1CT0NtAKPu/tWM7vXzGYHxRqBMcCPzWyLma2KKx4RERlcrA+vufsaUq/vjG67K7KstyaKiOQRTXMhIiIhJQUREQkpKYiISEhJQUREQkoKIiISUlIQEZGQkoKIiIT0kp1C1DyXprLUE8cHej8kLiIyImopSEnSy3dE0lNSEBGRkJKCiIiElBRERCSkpCAiIiElBRERCemWVBGgMniTHUDb4qt735VU91gCEYkkQy0FEREJqaUgJavnlaPzI60EkVKnpCDSR2XD6vCJ8VlVFQlHI5JbSgqFRE/fikjMlBSk5DWVNYbL8w/VJxiJSPKUFEQioglCpBTp7iMREQmppSCSoaOeZRApQmopiIhISElBRERC6j7Kd7oNNXnBv0FTWYfuTpKip5aCiIiE1FLIExrEFJF8oJaCiIiE1FLIcz2TtkH/8/BEy4iIjIRaCiIiElJLoYCoRSAicVNSEBmCcG6k5kf0RjYpSuo+Ehmu5rl6jkSKjpKCiIiE1H2Uj/Tts+DoORMpFrG2FMys1szeMLMdZtaQZv9lZrbZzLrN7No4YxERkcHF1lIws1HAA8DlQDuwwcxWufu2SLFfA/OAr8UVh0gconeCzY+0EkQKXZzdRzOAHe6+E8DMVgBzgDApuHtbsO9wjHGIDJtuA5ZSY+4ez4lT3UG17j4/WL8JuMTdF6Up+zDwr+6+sp9zLQQWAlRUVExfsWJF2mt2dnYyZsyY7FQgx17dtTdcnnrC7nB5X1f3gMcdLj+VY7reiy2ufJOP9W3z3k+aTz19XNbOXci/08NVanXOVX1nzpy5yd2rBytXEAPN7r4MWAZQXV3tNTU1acu1tLTQ3758Ny/SBdFU9ki4fMIgxx2oquOE1uaYoso/+Vjf7/WZTrvthpqsnbuQf6eHq9TqnG/1jXOgeRdwRmR9UrBNRETyVJwthQ3A2WY2mVQyuB6oi/F6IokIn3IGvYRHCl5sLQV37wYWAU8DrcDj7r7VzO41s9kAZnaxmbUDnwd+YGZb44pHREQGF+uYgruvAdb02XZXZHkDqW4lERHJAwUx0FwKol0QUpz01LMUAiUFkSzTh78UMk2IJyIiIbUUckzfIotb3zuRKjUFhhQYtRRERCSklkKSNEV2UdPzC1KI1FIQybGmskZ9IZC8pZaCSA7olmMpFEoKuRJ8M2wq61BXgojkLXUfiYhISElBRERCSgoiIhLSmEIOVDaspqnsyGsdjww6VqQ/QEQkIWopiIhISElBJAHrWjuobFitaTAk7ygpiIhISGMKIgnTJImST5QURBKiuZEkH6n7SEREQmopiOQRdSVJ0pQUErSutWPwQlIS1JUk+ULdRyIiElJLIYvU9Jese28nND+YWq57LNlYpCSopSCSpyobVrOvq1vdjJJTSgoiBUBPQEuuqPsoTpEX64hkKjrofIC6BCORUqSkkGU9f9Dr7tTrF0Wk8CgpDIMGlCUJR1oQ+p2T+CgpiBSJdXfWhMuzvt2SWBxS2JQUMqDBPckrwVgV6AFIyT4lBZECo0QgcVJSGCJNRyCFQONeMlxKCiPU88fXVNZIU1nCwYiIjJCSQgairYNMtovki6ayxvD26PmH6tO3GiJjFJpKQ5QURIpQui8sTWWN0PxIaqXusUgrt4NZVRW5DE/yWKxJwcxqgSXAKKDJ3Rf32X888AgwHdgNzHX3tjhjylT09j6RYtEzSD2/zx114eB1n9/7WVUVQ2s9qNVR8GJLCmY2CngAuBxoBzaY2Sp33xYp9iXgP939I2Z2PfAdYO7RZ8uNyobV6hKSkpDp7/m61o4wgUS7nqID2dFzqcVR+OJsKcwAdrj7TgAzWwHMAaJJYQ7wrWB5JfAPZmbu7jHGlRJ8o1nX2qG7iEQGkG7qlqay9HffRW+Xnd9P4ug5ru+2Acc7stHqUCsmI3EmhdOBtyLr7cAl/ZVx924z2wuMB96NJaJIIohS60Bk6Ab7uxnKDRrRAfEDVXWsu/Nb4b5o66Pvg6TpztWrtRL58O/1dx99+jtd6yaTpJHNhJVHLK4v5WZ2LVDr7vOD9ZuAS9x9UaTMa0GZ9mD9l0GZd/ucayGwMFj9KPBGP5edQFwJJX+VWp1V3+JXanXOVX3/xN0/NFihOFsKu4AzIuuTgm3pyrSb2bHAOFIDzr24+zJg2WAXNLON7l497IgLUKnVWfUtfqVW53yrb5wv2dkAnG1mk83sOOB6YFWfMquALwbL1wL/npPxBBERSSu2lkIwRrAIeJrULanL3X2rmd0LbHT3VcAPgX82sx3Ae6QSh4iIJCTW5xTcfQ2wps+2uyLLXcDns3jJQbuYilCp1Vn1LX6lVue8qm9sA80iIlJ44hxTEBGRAlOwScHMlpvZO8FtrT3bGs3sdTP7hZk9aWYnJxljtqWrc2Tf7WbmZjYhidji0F99zeyW4N95q5l9N6n4sq2f3+lpZvaimW0xs41mNiPJGLPJzM4ws/Vmti34t7w12H6qmT1jZtuD/5+SdKzZMEB98+pzq2CTAvAwUNtn2zPAee5+PvAm8M1cBxWzhzm6zpjZGcAVwK9zHVDMHqZPfc1sJqkn4S9w93OB/5VAXHF5mKP/fb8L3OPu04C7gvVi0Q3c7u5TgI8BN5vZFKABeNbdzwaeDdaLQX/1zavPrYJNCu7+HKk7lqLb1rp7d7D6IqlnI4pGujoH/g74OlBUA0T91PevgMXu/n5Q5p2cBxaTfurrwEnB8jjg7ZwGFSN3/427bw6W9wGtpGY5mAP8U1Dsn4Brkokwu/qrb759bhVsUsjAXwL/lnQQcTOzOcAud38l6Vhy5BzgU2b2kpn91MwuTjqgmN0GNJrZW6RaRcXW+gXAzCqBC4GXgAp3/02w67dA0c2y16e+UYl/bhVlUjCzvyXVVPtR0rHEycxGA/+DVLdCqTgWOJVU87seeNzMLNmQYvVXwF+7+xnAX5N6tqeomNkY4AngNnf/fXRf8DBrUbWA+6tvvnxuFV1SMLN5wGeBG0rg6eizgMnAK2bWRqrZudnM/ijRqOLVDvzEU14GDpOaO6ZYfRH4SbD8Y1KzDxcNMysj9QH5I3fvqWeHmX042P9hoGi6CPupb159bhVVUghe6vN1YLa7/yHpeOLm7q+6+0R3r3T3SlIfmBe5+28TDi1OTwEzAczsHOA4invytLeBPw2W/wzYnmAsWRW08H4ItLr7fZFd0elvvgj8n1zHFof+6ptvn1sF+/Camf0LUEPqW2IHcDep/tbjOTKp3ovu/pVEAoxBujq7+w8j+9uA6r6zzBaqfv6N/xlYDkwDDgJfc/d/TyrGbOqnvm+QenvhsUAX8FV335RUjNlkZp8EngdeJdXig1R36EvA48AfA78CrnP3dDdYFJQB6ruUPPrcKtikICIi2VdU3UciIjIySgoiIhJSUhARkZCSgoiIhJQUREQkpKQgRcHMOkd4/EozO3OIx3zFzP7bSK47VGb2X8zsBTN738y+Ftl+nJk9F7zrXGTYlBSk5JnZucAod985hGOOdfeH3P2RLMZRaWYtgxR7D/jv9Jkd1t0PkppRdG624pHSpKQgRcVSGs3sNTN71czmBtuPMbPvB/PWP2Nma8zs2uCwG4g8NWtmnWb2d8Gc98+a2YeC7S1mdr+ZbQRuNbNv9XxbN7OPmNk6M3vFzDab2VnB9noz2xDMlX/PSOvn7u+4+wbgUJrdTwV1ERk2JQUpNv+V1NPOFwCzSM0w+uFgeyUwBbgJ+HjkmEuB6FPCJwIbg/c1/JTUk8U9jnP3anf/Xp/r/gh4wN0vAD4B/MbMrgDOJjVf0TRgupldlpVapvcaUOyzxkrM1P8oxeaTwL+4+wekJlb7KakPyk8CP3b3w8BvzWx95JgPA7+LrB8GHguWH+XIhHREtofMbCypefGfBHD3rmD7FaRefvTzoOgYUkniuT7HP0lqYsPjgD82sy3BriXu/o+ZVtzdPzCzg2Y2NpivX2TIlBRE4ABQPsD+6Fww+4dwXgP+p7v/YKBC7v45COfYf9jda4Zwjb6OJzVHksiwqPtIis3zwFwzGxWMBVwGvAz8P+DPg7GFClITz/VoBT4SWT8G6BlvqAP+Y6ALBt/K283sGgAzOz5418XTwF8G8+djZqeb2cSRVrA/ZjYeeNfd0403iGRELQUpNk+SGi94hdQ3/K+7+2/N7Ang08A24C1gM7A3OGY1qSSxLljfD8wwsztIzeWfyR09NwE/MLN7SQ0Cf97d15pZFfBC8B6gTuBGRvB+gOBdGRtJvaLzsJndBkwJXtYyM6iLyLBpllQpGWY2xt07g2/ULwOXBgnjBGB9sP6BmXW6+5hkox06M/sJ0ODubyYdixQutRSklPyrmZ1MakD32z0vI3L3A2Z2N6mXxv86yQCHy8yOA55SQpCRUktBRERCGmgWEZGQkoKIiISUFEREJKSkICIiISUFEREJKSmIiEjo/wOEm2DR7iW32QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(cv['log_parea_total_price'], bins=100, label='train true', normed=True)\n",
    "plt.hist(np.log1p(test['total_price'] / test['building_area']), bins=100, label='test',\n",
    "         normed=True, alpha=0.7)\n",
    "plt.xlabel('log(price/area + 1)'); plt.ylabel('ratio')\n",
    "plt.legend(); plt.grid(); plt.show()\n",
    "\n",
    "plt.hist(cv['log_total_price'], bins=100, label='train true', normed=True)\n",
    "plt.hist(np.log1p(test['total_price']), bins=100, label='test', normed=True, alpha=0.7)\n",
    "plt.xlabel('log(price + 1)'); plt.ylabel('ratio')\n",
    "plt.legend(); plt.grid(); plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "plt.hist(np.log1p(cv['building_area']), bins=100, label='train', normed=True)\n",
    "plt.hist(np.log1p(test['building_area']), bins=100, label='test', normed=True, alpha=0.7)\n",
    "plt.xlabel('log(building_area + 1)'); plt.ylabel('ratio'); plt.yscale('log')\n",
    "plt.legend(); plt.grid(); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.385361e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.924782e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.095087e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>5.985690e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.067865e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>UEeCDaAJzPwdKKKA</td>\n",
       "      <td>1.231166e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>i0fgbPaQsDWs7Q87</td>\n",
       "      <td>4.714483e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>YunNwAhcqkf6YclI</td>\n",
       "      <td>1.098235e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>A2NotxtRY9MYoWMl</td>\n",
       "      <td>2.984735e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>kKvgBXiA50gRmQhP</td>\n",
       "      <td>3.117959e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           building_id   total_price\n",
       "0     X5gsdTWGS3W7JJQB  1.385361e+07\n",
       "1     BTshNOJyKHnT2YIT  3.924782e+06\n",
       "2     dhdymr0lV8N5kZOT  1.095087e+07\n",
       "3     VEwyGGMcD56w5BOc  5.985690e+06\n",
       "4     wmUeMoJZfsqaSX9b  1.067865e+06\n",
       "9995  UEeCDaAJzPwdKKKA  1.231166e+06\n",
       "9996  i0fgbPaQsDWs7Q87  4.714483e+07\n",
       "9997  YunNwAhcqkf6YclI  1.098235e+06\n",
       "9998  A2NotxtRY9MYoWMl  2.984735e+06\n",
       "9999  kKvgBXiA50gRmQhP  3.117959e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape = (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "test[['building_id', 'total_price']].check()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output file to: output/stack_meanlog_18_1-10,12-22,24-29.csv\n"
     ]
    }
   ],
   "source": [
    "output_filename = 'output/stack_meanlog_{}_{}.csv'.format(stack_idx, models)\n",
    "print('Output file to:', output_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[['building_id', 'total_price']].to_csv(output_filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.385361e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.924782e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.095087e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>5.985690e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.067865e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>UEeCDaAJzPwdKKKA</td>\n",
       "      <td>1.231166e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>i0fgbPaQsDWs7Q87</td>\n",
       "      <td>4.714483e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>YunNwAhcqkf6YclI</td>\n",
       "      <td>1.098235e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>A2NotxtRY9MYoWMl</td>\n",
       "      <td>2.984735e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>kKvgBXiA50gRmQhP</td>\n",
       "      <td>3.117959e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           building_id   total_price\n",
       "0     X5gsdTWGS3W7JJQB  1.385361e+07\n",
       "1     BTshNOJyKHnT2YIT  3.924782e+06\n",
       "2     dhdymr0lV8N5kZOT  1.095087e+07\n",
       "3     VEwyGGMcD56w5BOc  5.985690e+06\n",
       "4     wmUeMoJZfsqaSX9b  1.067865e+06\n",
       "9995  UEeCDaAJzPwdKKKA  1.231166e+06\n",
       "9996  i0fgbPaQsDWs7Q87  4.714483e+07\n",
       "9997  YunNwAhcqkf6YclI  1.098235e+06\n",
       "9998  A2NotxtRY9MYoWMl  2.984735e+06\n",
       "9999  kKvgBXiA50gRmQhP  3.117959e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape = (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "pd.read_csv(output_filename).check()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute submission for linear mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict for liear mean with [1, 3, 6, 8, 12, 13, 15, 16, 17, 18, 19, 21, 22, 24, 25, 26, 27, 28, 29]\n"
     ]
    }
   ],
   "source": [
    "print('Predict for liear mean with', best_models_subset_none)\n",
    "test['total_price'] = predict_mean_subset(test, best_models_subset_none, is_per_area=False, is_log=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/matplotlib/axes/_axes.py:6571: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHUpJREFUeJzt3X90VeWd7/H3F4xGDIJATat4J5RqL/gLJaLV6oQWBWsX2FtbLK1XOgU6U/Xq1MmU3lan2ukaam4dxfqjXMowHRvRYvWyLtwRY406U38BxR8QFXSlNaBBsSJRogS+94+9szkJ+XFOcvbZ58fntVYWZ+/97L2/T3I43/M8z97PNndHREQEYEjSAYiISP5QUhARkYiSgoiIRJQUREQkoqQgIiIRJQUREYkoKYiISERJQUREIkoKIiISOSTpADI1ZswYr6qqyni/999/nyOOOCL7AeU51bu0qN6lJZN6r1+//m13/1h/5QouKVRVVbFu3bqM92tsbKSmpib7AeU51bu0qN6lJZN6m9kf0ymn7iMREYkoKYiISERJQUREIgU3piAixWnv3r20tLTQ3t6e8b4jRoygqakphqjyW0/1Li8vZ+zYsZSVlQ3omEoKIpIXWlpaGD58OFVVVZhZRvvu3r2b4cOHxxRZ/upeb3dn586dtLS0MG7cuAEdU91HIpIX2tvbGT16dMYJQQ4wM0aPHj2g1lYnJQURyRtKCIM32N+hkoKIiEQ0piAiealq4eqsHq950UV9bn/33Xepr6/nO9/5TsbH/sIXvkB9fT0jR45Mq/yDDz7ICSecwMSJEzM+V9zUUpDCVz87+BEZhHfffZc77rijx20dHR197rtmzZq0EwIESWHz5s0DOlfclBRERICFCxfy6quvMmnSJGpra2lsbOTcc89l5syZ0Tf6iy++mMmTJ3PiiSeyZMmSaN+qqirefvttmpubmTBhAvPnz+fEE0/kggsuYM+ePV3O8/vf/55Vq1ZRW1vLpEmTePXVV6mpqeGaa66hurqaW2+9lblz57Jy5cpon4qKiuh1XV0dZ5xxBqeccgo/+clPsv57UFIQEQEWLVrE+PHj2bhxI3V1dQBs2LCBW2+9lVdeeQWAZcuWsX79etatW8fixYvZuXPnQcfZsmULV1xxBZs2bWLkyJHcf//9XbafffbZzJw5k7q6OjZu3Mj48eMB+Oijj1i3bh3XXnttrzGuXbuWLVu28Mwzz7Bx40Y2btzI448/nq1fAaAxBRGRXk2ZMqXL9f6LFy/mgQceAOD1119ny5YtjB49uss+48aNY9KkSQBMnjyZ5ubmtM41e3b/XaBr165l7dq1nHbaaQC89957bNmyhfPOOy+tc6RDSUFEpBep01I3NjbS0NDAk08+ybBhw6ipqenxfoDDDjssej106NCDuo/SOdchhxzC/v37Adi/fz8fffQRENyc9v3vf59vf/vbQDw37an7SEQEGD58OLt37+51+65duzjqqKMYNmwYL730Ek899VRs56qqqmL9+vUArFq1ir179wIwffp0li1bRltbGwDbt29nx44dA46jJ2opiEhe6u8S0lTZ+MY8evRozjnnHE466SQuvPBCLrqo6/lnzJjBXXfdxYQJE/j0pz/NWWedNeBzXXrppcyfP5/Fixd3GVDuNH/+fGbNmsWpp57KjBkzolbEBRdcQFNTE5/5zGcAOPzww7nnnns4+uijBxxLd+buWTtYLlRXV7sespO+kqh35+Woc+6NVpVEvXtQyPVuampiwoQJA9pXcx911dPv0szWu3t1f8dU95GIiESUFEREJKKkICIiESUFERGJKClIcXrnNc2HJDIASgoiIhLRfQoikp8yaOkd3tEBh/TzcZZyyXJPBjN19i233MKCBQsYNmxYxvvmG7UUREToe+rs/txyyy188MEHWY4oGWopSNGpWria207poKGplWlJByMFI3Xq7PPPP5+jjz6a++67jw8//JAvfelL3HDDDbz//vt89atfpaWlhX379nHdddfR2trK9u3bmTp1KmPGjOHRRx9NuiqDoqQgIkIwdfaLL77Ixo0bWbt2LStXruSZZ57B3Zk5cyaPP/44b731FscccwyrVwdPhdu1axcjRozg5ptv5tFHH2XMmDEJ12Lw1H0kItJN6hTVp59+Oi+99BJbtmzh5JNP5uGHH+Z73/seTzzxBCNGjEg61KxTS0FEpJvuU1Sn2rBhA2vWrOGHP/whn//857n++usTiDA+sbYUzGyGmb1sZlvNbGEf5b5sZm5m/U7WJNKThqZWqhauzvrD3qV0pE5n3X2K6m3btrFjxw62b9/OsGHD+MY3vkFtbS0bNmw4aN9CF1tLwcyGArcD5wMtwLNmtsrdN3crNxy4Gng6rlhEpAD1cwlpqj0xTJ09Z86caIrqiooK7r77brZu3UptbS1DhgyhrKyMO++8E4AFCxYwY8YMjjnmGA0092EKsNXdXwMwsxXALGBzt3I/Bn4K1MYYi4hIv+rr67ssX3311V2Wx48fz/Tp0w/a76qrruKqq66KNbZcibP76Fjg9ZTllnBdxMxOB45zd7X5JR71szXdhUgGEhtoNrMhwM3A3DTKLgAWAFRWVtLY2Jjx+dra2ga0X6EriXqXT2fPhA6u9Y5o1f7yUeyZMIfG8vAtXuy/g1Ah/71HjBgx4H75ffv2FU2ffiZ6q3d7e/uA3wdxJoVtwHEpy2PDdZ2GAycBjWYG8HFglZnNdPcuj1Zz9yXAEgievDaQJ0sV8hOpBqMk6l1/Jw1Nrfxs74EeyNtOaeXwpnpqJlQGK2rS758uZIX8925qaqKiooLw8yAjevLaAe5OeXk5p5122oCOGWf30bPA8WY2zswOBS4FVnVudPdd7j7G3avcvQp4CjgoIUiC1PUiOVReXs7OnTsptEcE5xN3Z+fOnZSXlw/4GLG1FNy9w8yuBB4ChgLL3H2Tmd0IrHP3VX0fQURKydixY2lpaeGtt97KeN/29vZBfRAWqp7qXV5eztixYwd8zFjHFNx9DbCm27oe7/Rw95o4YxGR/FZWVsa4ceMGtG9jY+OAu0sKWRz11jQXIiISUVIQEZGIkoJkRx4MSC8tq0s6BJGCpwnxJPs6E0QG0xQMROc8R0vLWvsvXD879nhEioFaCiIiElFSEBGRiJKCiIhENKYgmcnReEE6NLAskn1KClJUOhPFHuYkHIlIYVJSkMHJ8aWoqU9WW1rWf/mGpgNXJk3rHmsetHZE8o3GFCR+WUwcS8vq1G0kEiMlBcmtLM28qsQgEg8lBRERiSgpiIhIRElBBq2hqZWGptYug8AiUpiUFEREJKJLUiVjDU2tzMtkMjoRKRhKCjIguvpHpDip+0hERCJqKUjJ6HJ384TKBCMRyV9KCpJVVQtXR+MM0xKORUQyp+4jERGJqKUgsUl9XGbqZHSpVy81L7ookdhEpGdqKYiISERJQfqXpUnsRCT/qftIciL1yh8RyV9qKYiISERJQdLSOeGdvvGLFDclBRERiSgpiIhIRElBREQiuvpIsiaTmVMPlNXNayL5RC0FSZumyxYpfmopSGzSSiKpN8XNuTe+YEQkLWopiIhIRElBREQiSgoiIhKJdUzBzGYAtwJDgaXuvqjb9r8GrgD2AW3AAnffHGdM0rfO6a6B6GE5ceryNLRe4ki1tCzmgERKXGwtBTMbCtwOXAhMBL5mZhO7Fat395PdfRJwE3BzXPFIP0psJtTOaTt6Sz4ipSrO7qMpwFZ3f83dPwJWALNSC7j7eymLRwAeYzyShqVldbr0VKSExdl9dCzwespyC3Bm90JmdgXwXeBQ4HMxxiMiIv0w93i+nJvZJcAMd58XLl8GnOnuV/ZSfg4w3d0v72HbAmABQGVl5eQVK1ZkHE9bWxsVFRUZ71fo0q73O68BsLu9A4Bmr6TKcjsj6vBjTohev7BtV49l0o1pf/kohrS/02eZZq8E4ORjR6QZYf7T+7y0ZFLvqVOnrnf36v7KxdlS2AYcl7I8NlzXmxXAnT1tcPclwBKA6upqr6mpyTiYxsZGBrJfoUu73vXBr75z4Pdne2tZWlYfY2QHqyH4kGbOvcztdaA5vZj2TJjD4U19l/3Z3loAmr9ek3aM+U7v89ISR73jTArPAseb2TiCZHApMCe1gJkd7+5bwsWLgC2I5FjqYHPzIs3FJKUttqTg7h1mdiXwEMElqcvcfZOZ3Qisc/dVwJVmNg3YC/wZOKjrSEpb56D3vPBbvYjEK9b7FNx9DbCm27rrU15fHef5RUQkM7qjWUREIpolVXqUxL0KnYPc83oYZNa9EyK5oZaCiIhElBRERCSipCAiIhElBRERiSgplLISmhVVRNKjpCAiIhElBRERiSgpiIhIRElB8pZuWBPJPSUFERGJKCmIiEhESUFKnrqpRA5Ie0I8MzsVODdcfMLdn4snJCl1+pAWSU5aScHMrgbmA78NV91tZkvc/bbYIpPc0A1sIpIi3ZbCt4Az3f19ADP7KfAkoKQgBaunFokezSmlLt0xBQP2pSzvC9eJiEgRSbel8C/A02b2QLh8MfDLeEISEZGkpJUU3P1mM2sEPhuu+qa7/yG2qCSnOp94Vso6u5Lm7a1NOBKRZPWZFMzsSHd/z8xGAc3hT+e2Ue7+TrzhiYhILvXXUqgHvgisBzxlvYXLn4wpLpFEqMUgpa7PpODuXwz/HZebcEREJElpXX1kZo+ks05ERApbf2MK5cAwYIyZHcWBy1CPBI6NOTYREcmx/sYUvg1cAxxDMK7QmRTeA34eY1wiIpKAPruP3P3WcDzh79z9k+4+Lvw51d2VFKRoLS2r0xQgUpLSvU/hNjM7CZgIlKes/1VcgYnkhfrZMOfepKMQyZl0J8T7B6CGICmsAS4E/gNQUhARKSLpzn10CfB54E13/yZwKjAitqhERCQR6SaFdnffD3SY2ZHADuC4+MISSV5DUysNTa1dZk4VKXb9dh+ZmQHPm9lI4H8TXIXURjB1toiIFJF+k4K7u5lNcfd3gbvM7N+BI939+fjDExGRXEq3+2iDmZ0B4O7NSggiIsUp3ecpnAl83cz+CLxPOCGeu58SW2QSq6qFq1lapimzRaSrdJPC9FijEBGRvJBW95G7/7Gnn7iDE8kHurtZSkm6YwoDYmYzzOxlM9tqZgt72P5dM9tsZs+b2SNm9hdxxiMiIn2LLSmY2VDgdoK7nycCXzOzid2K/QGoDscmVgI3xRWPiIj0L86WwhRgq7u/5u4fASuAWakF3P1Rd/8gXHwKGBtjPCIi0g9z9/5LDeTAZpcAM9x9Xrh8GXCmu1/ZS/mfE0yj8Y89bFsALACorKycvGLFiozjaWtro6KiIuP9Cl1v9X5h2y6qrHivPtpfPooh7dl7hPjw8kNgVP4/fVbv89KSSb2nTp263t2r+yuX7tVHsTKzbwDVwF/2tN3dlwBLAKqrq72mpibjczQ2NjKQ/Qpdb/Weu3A1S8vqcx9QjuyZMIfDm7JXv5oJlVCT/7Ol6n1eWuKod5zdR9voOj/S2HBdF2Y2DfgBMNPdP4wxHpEB65wDqcs8SLoiSYpQnEnhWeB4MxtnZocClwKrUguY2WnALwgSwo4YYxERkTTElhTcvQO4EngIaALuc/dNZnajmc0Mi9UBFcBvzGyjma3q5XAiIpIDsY4puPsagofypK67PuX1tDjPL5JNS8vqwlcXJRqHSJxivXlNREQKi5KCiIhE8uKSVMkNPUFMRPqjloKIiESUFEREJKKkICIiEY0plILwzls9aU1E+qOkIJKhzgH7pWWt6EYbKTbqPhIRkYiSgoiIRJQUREQkoqQgIiIRDTQXuRe27aLjJV11FJvOZyrMyf8H8IikQy0FERGJKCmIiEhESUFERCJKCiIiElFSEBGRiJKCiIhEdEmqSJZ0f4hR8yI9y1kKj1oKItlQP5ulZXVJRyEyaEoKIlm0tKxOyUEKmrqPRDKU7od+aneSupKkUKilICIiEbUURAahoUnzSklxUUtBREQiSgoiMdGgsxQidR8VodQBzttOUfeGiKRPLQUREYkoKYiISERJQUREIkoKIjHQALMUKiUFERGJ6OqjItX5TXUPcxKOREQKiVoKIrlSPzv4EcljaimIxGxpWR0N1x0YY5iWYCwi/Ym1pWBmM8zsZTPbamYLe9h+npltMLMOM7skzliKnr6FFhb9vSRPxZYUzGwocDtwITAR+JqZTexW7E/AXKA+rjhERCR9cXYfTQG2uvtrAGa2ApgFbO4s4O7N4bb9McYhIiJpirP76Fjg9ZTllnCdiIjkKXP3eA4cjBHMcPd54fJlwJnufmUPZZcD/9fdV/ZyrAXAAoDKysrJK1asyDietrY2KioqMt6vYLzzWvRyd3tH9Hp/+SiGtL+TRESJyud6Dy9PaaCP+mRWj1307/NeqN79mzp16np3r+6vXJzdR9uA41KWx4brMubuS4AlANXV1V5TU5PxMRobGxnIfgWj/s7oZeqDX/ZMmMPhTaU3ZJPP9e5IeT3tx41ZPXbRv897oXpnT5xJ4VngeDMbR5AMLgXdSRUXPQFMRLIhtjEFd+8ArgQeApqA+9x9k5ndaGYzAczsDDNrAb4C/MLMNsUVj4iI9C/Wm9fcfQ2wptu661NeP0vQrSQiInlA01yIiEhESUFERCJKCiIiElFSEBGRiGZJLWBVC1cDesqXiGSPWgoi+UAzpkqeUEtBJEEHWnutes6C5AW1FEREJKKkICIiESUFERGJKCmIiEhEA80FTJeiFpfOQWeA5kUXJRiJlDIlhQKT+sGxtCzBQESkKKn7SCRPLC2rU+tPEqekICIiESUFERGJaExBJEE9dRctLauD+l/BnHsTiEhKnVoKIvmqfrbmRJKcU0tBJA81NLVGrzUnkuSSWgoiIhJRUigk9bN1yaKIxEpJQUREIhpTKACpc+4H/6q1ICLxUEtBREQiaimI5DlNlCe5pJZCgVCXUenS315ySUlBREQi6j7KU6ldBiIHWgvqPpJ4qaWQpzSNsogkQS2FPKfEIF10zoWkyfIkJmopiIhIRC2FfKNZMaUPnRPlzVu4WpenSiyUFPJI1cLV0V3LIiJJUPdRHtH4gaQreBCPWpWSfUoKIiISUfdRwoIuI7UQJHMNTa3MC+9n0fiCZIuSQgJ0Y5rEon42lE8HapKORApYrN1HZjbDzF42s61mtrCH7YeZ2b3h9qfNrCrOePKNblCTbKlauJqGplZ2t3ckHYoUuNhaCmY2FLgdOB9oAZ41s1Xuvjml2LeAP7v7p8zsUuCnQNGMnqV2Dc3bWxutVyKQbOr+fmq4rgbo+p4DdTFJeuLsPpoCbHX31wDMbAUwC0hNCrOAH4WvVwI/NzNzd48xrkQpIUg29fV+St02b2/tQeNXnUkjSha6W1qINykcC7yestwCnNlbGXfvMLNdwGjg7RjjGrDBjAUoGUiS+nr/dbYsItfV5KyV0duzIvQMieQUxECzmS0AFoSLbWb28gAOM4YEks35uT7hQR5LpN7JU73TKAv09h59rMuS/XQwMaWnt3Okee4S/XtnVO+/SKdQnElhG3BcyvLYcF1PZVrM7BBgBLCz+4HcfQmwZDDBmNk6d68ezDEKkepdWlTv0hJHveO8+uhZ4HgzG2dmhwKXAqu6lVkFXB6+vgT4XTGPJ4iI5LvYWgrhGMGVwEPAUGCZu28ysxuBde6+Cvgl8G9mthV4hyBxiIhIQmIdU3D3NcCabuuuT3ndDnwlzhhSDKr7qYCp3qVF9S4tWa+3qbdGREQ6aUI8ERGJFGVSMLNlZrbDzF5MWTfKzB42sy3hv0clGWMceql3nZm9ZGbPm9kDZjYyyRjj0FO9U7Zda2ZuZmOSiC1OvdXbzK4K/+abzOympOKLSy/v80lm9pSZbTSzdWY2JckY42Bmx5nZo2a2OfzbXh2uz+pnW1EmBWA5MKPbuoXAI+5+PPBIuFxslnNwvR8GTnL3U4BXgO/nOqgcWM7B9cbMjgMuAP6U64ByZDnd6m1mUwlmCjjV3U8E/lcCccVtOQf/vW8CbnD3ScD14XKx6QCudfeJwFnAFWY2kSx/thVlUnD3xwmuZko1C/jX8PW/AhfnNKgc6Kne7r7W3TtnSXuK4H6RotLL3xvgn4G/B4py4KyXev8NsMjdPwzL7Mh5YDHrpd4OHBm+HgFsz2lQOeDub7j7hvD1bqCJYFaIrH62FWVS6EWlu78Rvn4TqEwymIT8FfD/kg4iF8xsFrDN3Z9LOpYcOwE4N5x1+DEzOyPpgHLkGqDOzF4naB0VY4s4Es4ofRrwNFn+bCulpBAJb5Arym+PvTGzHxA0P3+ddCxxM7NhwP8k6EYoNYcAowi6F2qB+8zMkg0pJ/4G+Ft3Pw74W4J7oIqSmVUA9wPXuPt7qduy8dlWSkmh1cw+ARD+W3TN6t6Y2Vzgi8DXS+SO8fHAOOA5M2sm6DLbYGYfTzSq3GgBfuuBZ4D9BPPjFLvLgd+Gr39DMEtz0TGzMoKE8Gt376xvVj/bSikppE6pcTnwfxKMJWfMbAZBv/pMd/8g6Xhywd1fcPej3b3K3asIPihPd/c3Ew4tFx4EpgKY2QnAoZTGRHHbgb8MX38O2JJgLLEIW3y/BJrc/eaUTdn9bHP3ovsB7gHeAPYSfCB8i2BK7kcI3iwNwKik48xRvbcSTE++Mfy5K+k4c1HvbtubgTFJx5mjv/ehwN3Ai8AG4HNJx5mjen8WWA88R9DPPjnpOGOo92cJuoaeT/n//IVsf7bpjmYREYmUUveRiIj0Q0lBREQiSgoiIhJRUhARkYiSgoiIRJQUJO+YWdsg919pZp/McJ+/NrP/Pohz3mVm5wx0/ziZ2XlmtsHMOszskpT1HzOzf08yNsk/SgpSVMzsRGCou7+WwT6HuPtd7v6rQZz6LIIJB9M63yDO0/1YNWa2vJ9ifwLmAvWpK939LeCNfE1mkgwlBclbFqgzsxfN7AUzmx2uH2Jmd4TPDHjYzNakfAP+Oil3dJpZm5n9czj//CNm9rFwfaOZ3WJm64CrzexHZvZ34bZPmVmDmT0XfsMeH66vNbNnw2dT3JByjgnAK+6+z8zmh2WeM7P7w3mYMLPlYWviaeAmMzsifC7AM2b2h3ACP8ysysyeCM+7wczOHuzv0d2b3f15gikvunsw/J2JAEoKkt/+GzAJOBWYRjAL5ifC9VXAROAy4DMp+5xDcGdrpyOAdR48W+Ax4B9Sth3q7tXu/rNu5/01cLu7nwqcTfBt+gLgeII5dSYBk83svLD8hUBnN8xv3f2McN8mgrttO40Fznb37wI/AH7n7lMIpqWoM7MjCOatOd/dTwdmA4vT+1UN2Drg3JjPIQUka81YkRh8FrjH3fcRTPr1GHBGuP437r4feNPMHk3Z5xPAWynL+4F7w9d3c2DSNFLWR8xsOHCsuz8A4O7t4foLCB7Y84ewaAVBkngcmA58M1x/kpn9IzAyLPNQyuF/E9aF8FgzO1snQDnwXwjm8Pm5mU0C9hFMhX2QsMVxWHiOUWa2Mdz0PXd/qKd9erEDOCaD8lLklBSk2Owh+IDtTeq8Lu9ncFwD/sndf9FlZdA9NNLdOx/qshy42N2fC2enrenlfAZ82d1f7na8HwGtBK2jIUB7j5VwPzMsXwPMdfe5GdQlVTnB70wEUPeR5LcngNlmNjQcCzgPeAb4T+DL4dhCJV0/eJuAT6UsDwE6xxvmAP/R1wk9eKJVi5ldDGBmh4Uf/A8BfxXOZY+ZHWtmRxN0/aS2VIYTdDeV0Xdf/UPAVZ3POjCz08L1I4A3wlbQZcDQvuLNghMIJs8TAZQUJL89QDAj5HPA74C/92D66/sJZsfcTNAltAHYFe6zmoO/nU+x4CHvnwNuTOO8lwH/w8yeB34PfNzd1xJcvfOkmb0ArCRIAKnjCQDXEczS+Z/AS32c48dAGfC8mW0KlwHuAC43s+eA/0pmrZkemdkZZtYCfAX4RXi+TlMJfmciAJolVQqTmVW4e5uZjSZoPZzj7m+a2eEE39zPCa8GanP3ihjj2ACc6e574zpHnMzscWCWu/856VgkPygpSEEys0aCwdxDgZvcfXnKtukEDyL5U9xJoZCFXXLnuPuDScci+UNJQUREIhpTEBGRiJKCiIhElBRERCSipCAiIhElBRERiSgpiIhI5P8D04lVZNpjj08AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHLRJREFUeJzt3X94VdWd7/H3V4wiBlGgZKo4E7TaAUVRIra1OmHqj1j7gJ2xYqNOmSnQTsWrM5Y2veOPav8Y2tw6QsdquSnjODaDFquXZ2CuiCXVufUHhOIPiArlSWugxYoDJZQoyPf+cXY2O+Ek5yQ5++zz4/N6Hh/P3nudvb8LDud71lp7rW3ujoiICMBRSQcgIiKFQ0lBRERCSgoiIhJSUhARkZCSgoiIhJQUREQkpKQgIiIhJQUREQkpKYiISOjopAMYqLFjx3p1dXXaY/v27eP444/Pb0AJK7c6q76lr9zqnK/6tra2vuPuH8pUruiSQnV1NevXr097rKWlhdra2vwGlLByq7PqW/rKrc75qq+Z/Sqbcuo+EhGRkJKCiIiElBRERCRUdGMKIlKaDhw4QEdHB6NGjaKtrS3pcPIm1/UdPnw448ePp6KiYlDvV1IQkYLQ0dHByJEjGTNmDCeccELS4eTN3r17GTlyZE7O5e7s2rWLjo4OJkyYMKhzqPtIRApCV1cXY8aMwcySDqVomRljxoyhq6tr0OdQUhCRgqGEMHRD/TNUUhARkZDGFESkIFU3rMzp+doXXtXv8d27d9Pc3MxXvvKVAZ/705/+NM3NzZx44olZlX/yySc588wzmTRp0oCvFTe1FCQR1Q0rw/9ECsHu3bv5/ve/n/bYwYMH+33vqlWrsk4IkEoKmzdvHtS14qakICICNDQ08Mtf/pIpU6awYMECWlpauPjii5kxY0b4i/7qq69m6tSpnHXWWSxZsiR8b3V1Ne+88w7t7e1MnDiRuXPnctZZZ3H55Zezf//+Htf5+c9/zooVK1iwYAFTpkxh27Zt1NbWcuutt1JTU8OiRYuYPXs2y5cvD99TWVkZvm5sbOSCCy7gnHPO4a677sr5n4OSgogIsHDhQk4//XQ2btxIY2MjABs2bGDRokW8+eabACxdupTW1lbWr1/P4sWL2bVr1xHn2bJlCzfddBObNm3ixBNP5PHHH+9x/BOf+AQzZsygsbGRjRs3ctpppwHw/vvvs379em677bY+Y1y9ejVbtmzhpZdeYuPGjbS2tvLss8/m6o8A0JiCiEifpk2b1uN+/8WLF/PEE08A8NZbb7FlyxbGjBnT4z0TJkxgypQpAEydOpX29vasrjVr1qyMZVavXs3q1as577zzAOjs7GTLli1ccsklWV0jG0oKIiJ9iC5p3dLSwpo1a3j++ecZMWIEtbW1aecDHHvsseHrYcOGHdF9lM21jj76aA4dOgTAoUOHeP/994HU5LRvfOMbfOlLXxpUfbKh7iMREWDkyJHs3bu3z+N79uzhpJNOYsSIEbz++uu88MILsV2rurqa1tZWAFasWMGBAwcAuOKKK1i6dCmdnZ0AbN++nbfffnvQcaSjloKIFKRMt5Dm2pgxY7jooos4++yzufLKK7nqqp7Xr6ur48EHH2TixIl89KMf5WMf+9igr3Xdddcxd+5cFi9ezEMPPXTE8blz5zJz5kzOPfdc6urqwlbE5ZdfTltbGx//+MeB1AD0I488wrhx4wYdS2/m7jk7WT7U1NS4HrJzWLHWOXor6kD+8RdrfQernOrb1tbGxIkTc7oWUDGIo77df5ZRZtbq7jWZ3qvuIxERCSkpiIhISElBRERCSgoiIhJSUhARkZCSgoiIhDRPQUQKU3PmZR8GpP7Rfg8PZens++67j3nz5jFixIjBRlcw1FIQEaH/pbMzue+++/jDH/6Q44iSoZaCiAg9l86+7LLLGDduHI899hjvvfcen/3sZ7n77rvZt28f1157LR0dHXzwwQfccccd7Ny5kx07djB9+nTGjh3L2rVrk67KkCgpiIiQWjr7tddeY+PGjaxevZrly5fz0ksv4e7MmDGDZ599lt/97necfPLJrFyZmpG/Z88eRo0axb333svatWsZO3ZswrUYOnUfiYj0El2i+vzzz+f1119ny5YtTJ48maeffpqvf/3rPPfcc4waNSrpUHNOLQURkV76W6J6w4YNrFq1ittvv51PfepT3HnnnQlEGB+1FERE6LmcdV9LVO/YsYMRI0Zwww03sGDBAjZs2HDEe4udWgoiUpgy3EKaa72Xzq6vrz9iieqtW7eyYMECjjrqKCoqKnjggQcAmDdvHnV1dZx88skaaBYRKRXNzc09tm+55ZYe26effjpXXHHFEe+7+eabufnmm2ONLV/UfSQiIiElBRERCcWaFMyszszeMLOtZtbQT7m/NDM3s4xPBRKR0lVsT4IsREP9M4wtKZjZMOB+4EpgEvB5M5uUptxI4BbgxbhiEZHCN3z4cHbt2qXEMATuzq5duxg+fPigzxHnQPM0YKu7bwMws2XATGBzr3LfAr4NLIgxFil00cXP8nzXiRSG8ePH09HRwe7du4f0pVZsurq6clrf4cOHM378+EG/P86kcArwVmS7A7gwWsDMzgdOdfeVZqakIFLGKioqmDBhAi0tLZx33nlJh5M3hVZfi6upZmbXAHXuPifYvhG40N3nB9tHAT8FZrt7u5m1AF919/VpzjUPmAdQVVU1ddmyZWmv2dnZSWVlZRzVKVjFWudXt+8JX08+ZRS8u+3wwdGn9fm+Yq3vYJVbfaH86pyv+k6fPr3V3TOO28bZUtgOnBrZHh/s6zYSOBtoMTOAPwJWmNmM3onB3ZcASwBqamq8trY27QVbWlro61ipKtY6z25YGb5uv74Wmh84fLC27+6jYq3vYJVbfaH86lxo9Y3z7qN1wBlmNsHMjgGuA1Z0H3T3Pe4+1t2r3b0aeAE4IiGIiEj+xJYU3P0gMB94CmgDHnP3TWZ2j5nNiOu6IiIyeLEuc+Huq4BVvfalXVLQ3WvjjEUKSPMsmip2MueA7i0QKTRa+0jyI9fP2xWRWGiZCxERCSkpiIhISElBRERCSgoiIhLSQLPkRHV0MtrCqxKMRESGQklBYtWdLJoqdnLpxKqEoxGRTNR9JCIiIbUUJD7BJDURKR5KCpK46oaVPZLHnKDLSWMTIvmnpCA5Fx1H6E9TRWM+whGRAdCYgoiIhJQUREQkpKQgIiIhJQUREQkpKYiISEh3H0lO6Y4ikeKmpCA5oWQgUhrUfSQiIiElBRERCan7SAZES2SLlDa1FEREJKSkICIiIXUfSUbRLqOkY7ht8kFqkw1FpKSppSAiIiElBRERCSkpiIhISGMKUnDC2dHND0P9o8kGI1JmlBQko+gSFnMOLBj0eda06XnNIoVO3UciIhJSS0EGr3lW0hGISI6ppSAiIiElBRERCSkpiIhISElBRERCGmiWAen5hLWqxOIQkXjEmhTMrA5YBAwDmtx9Ya/jXwZuAj4AOoF57r45zpikb3pWgojE1n1kZsOA+4ErgUnA581sUq9ize4+2d2nAN8B7o0rHhERySzOlsI0YKu7bwMws2XATCBsCbj77yPljwc8xngkC4e7h9RSEClH5h7P97CZXQPUufucYPtG4EJ3n9+r3E3A3wPHAH/u7lvSnGseMA+gqqpq6rJly9Jes7Ozk8rKypzWo9Dlss6vbt9DtaWWohh58pnh/r073kxbfuTww78p9nYdzEkMR5x/9GlhbABVx8G40aNyfq1Cpc906ctXfadPn97q7jWZyiU+0Ozu9wP3m1k9cDvwhTRllgBLAGpqary2tjbtuVpaWujrWKnKZZ1nN6ykqaIZgNr6lnD/mju+mbZ8NA0cl5MIeqqdWAW1j4axQeohO9dG6lvq4yD6TJe+QqtvnElhO3BqZHt8sK8vy4AHYoxHBqJQlrAI4miqSLVg9lt9QTwJTqRUxTlPYR1whplNMLNjgOuAFdECZnZGZPMq4IiuIxERyZ/YWgruftDM5gNPkboldam7bzKze4D17r4CmG9mlwIHgP8mTdeRiIjkT6xjCu6+CljVa9+dkde3xHl9EREZmMQHmkUGques6qjSG2gWyTetfSQiIiElBRERCSkpiIhISElBRERCSgoiIhJSUhARkZCSgoiIhJQUREQkpKQgIiKhrGc0m9m5wMXB5nPu/nI8IYmISFKySgpmdgswF/hJsOsRM1vi7t+LLTIpe2vadiYdgkjZybal8EVST03bB2Bm3waeB5QURERKSLZjCgZ8ENn+INgnIiIlJNuWwr8AL5rZE8H21cAP4wlJ8q7X081EpHxllRTc/V4zawE+Gez6a3f/RWxRiYhIIvpNCmZ2grv/3sxGA+3Bf93HRrv7u/GGJyIi+ZSppdAMfAZoBTyy34Lt02KKS2Tggm4wAOofTS4OkSLWb1Jw988E/5+Qn3BERCRJWd19ZGbPZLNPRESKW6YxheHACGCsmZ3E4dtQTwBOiTk2yTV1r4hIBpnGFL4E3AqcTGpcoTsp/B745xjjkjypblipW1FFJJRpTGERsMjMbtaSFiIipS/beQrfM7OzgUnA8Mj+h+MKTGKmCWsikka2C+LdBdSSSgqrgCuB/wKUFKRgRBfQuzTBOESKWbbLXFwDnAv8wt3/2syqgEfiC0vipNVHRaQv2S6I1+Xuh4CDZnYC8DZwanxhiYhIEjK2FMzMgFfM7ETgf5O6C6mT1NLZIgWpumFl+Lp94VUJRiJSXDImBXd3M5vm7ruBB83s/wInuPsr8YcnIiL5lG330QYzuwDA3duVEERESlO2A80XAteb2a+AfQQL4rn7ObFFJkOj2csiMgjZJoUrYo1CREQKQraT134VdyAyMN0DqbdNPkhtpsLRVoOISD+yHVMQKSpNFY00VTQmHYZI0cm2+0iKRHcLoqliJ5dOrEo4msKjW1VF+hdrS8HM6szsDTPbamYNaY7/vZltNrNXzOwZM/uTOOOR8tNU0ZjqPlMXmkhWYksKZjYMuJ/UOkmTgM+b2aRexX4B1AR3MS0HvhNXPCIiklmc3UfTgK3uvg3AzJYBM4HN3QXcfW2k/AvADTHGU9a03pGIZCPO7qNTgLci2x30/7S2LwL/GWM8IiKSgbl7PCc2uwaoc/c5wfaNwIXuPj9N2RuA+cCfuft7aY7PA+YBVFVVTV22bFnaa3Z2dlJZWZm7ShSwV7fvAaDqOBg3etQR+6ttaC2DkcMPNyL3dh0c0rly6dDw0RzV9e6Qz9PuVUw+ZVTmggkrp890t3Krc77qO3369FZ3r8lULs7uo+30XEl1fLCvBzO7FPgH+kgIAO6+BFgCUFNT47W1tWkv2NLSQl/HSs3s4C6a752zndodzeH+g6/nppuoNnLnUiF1Pe2fWM9xbc2ZC2bw3QMLaL++dugBxaycPtPdyq3OhVbfOJPCOuAMM5tAKhlcB9RHC5jZecAPSLUo3o4xFhmgQkoEIpI/sY0puPtBUl1CTwFtwGPuvsnM7jGzGUGxRqAS+LGZbTSzFXHFIyIimcU6ec3dV5F6fGd0352R13pqoohIAdEyFyIiElJSEBGRkJKCiIiElBRERCSkpCAiIiElBRERCSkpiIhISElBRERCevJaMWqeRVNFahmK/T1XDhERGRK1FKQs6YlsIukpKYiISEhJQUREQhpTEEkn2q1U/2hycYjkmVoKIiISUktBBKgOnmQH0L7wqgQjEUmWWgoiIhJSS0HKVvcjR+dEWgki5U5JQaSX6oaV4eTASydWJRyNSH4pKUjZa6poDF/PObAgwUhEkqekUEw0+zZ20QQhUo400CwiIiElBRERCan7SCRLmssg5UAtBRERCSkpiIhISN1HhU53HIlIHqmlICIiIbUUCoQGMUWkEKilICIiIbUUClz3om3Q9zo80TIiIkOhpCCSSTDY31SxU2sjSclTUigiahEkL1wbqflhPaZTSpLGFEREJKSWgshgReeQqNUgJUJJoRBpwpqIJETdRyJDtKZtJ9UNK3vMNREpVrEmBTOrM7M3zGyrmTWkOX6JmW0ws4Nmdk2csYiISGaxdR+Z2TDgfuAyoANYZ2Yr3H1zpNivgdnAV+OKQyQOuhNMSlWcYwrTgK3uvg3AzJYBM4EwKbh7e3DsUIxxiAyavvyl3Ji7x3PiVHdQnbvPCbZvBC509/lpyj4E/Ie7L+/jXPOAeQBVVVVTly1blvaanZ2dVFZW5qYCefbq9j3h68nH7Qpf7+062O/7Dg0fzVFd78YWV6Ep1Pq2e2q2+eRTRuX0vMX8mR6scqtzvuo7ffr0VnevyVSuKO4+cvclwBKAmpoar62tTVuupaWFvo4VutmRQcqmiofD18dleN/+ifUc19YcU1SFp1Dr+91gpnP79bU5PW8xf6YHq9zqXGj1jXOgeTtwamR7fLBPREQKVJwthXXAGWY2gVQyuA6oj/F6IokJl79Ay55LcYutpeDuB4H5wFNAG/CYu28ys3vMbAaAmV1gZh3A54AfmNmmuOIREZHMYh1TcPdVwKpe++6MvF5Hqlup7B3+pSkikpyiGGgWKQV6up4UAyUFkRzSF78UO619JCIiIbUU8ky/JEtbdGyo+ojVvkQKn1oKIiISUktBJCbRVoOe7SzFQkkhSXqYjogUGHUfieRZU0WjfhBIwVJLIV+CL4Gmip3qSihDmpwoxUItBRERCSkpiIhISN1HCTjclVCVaBwiIr0pKcSse7JaU4Ue6ygihU/dRyIiElJSEEnAmradVDes7LHsiUghUFIQEZGQxhREEqZFEqWQKCmIJERrI0khUveRiIiE1FIQKQDdrYY1dzSGrQZ1JUkSlBQStKZNcxdEpLCo+0ikkL27LbWYolZVlTxRSyFHdAeJiJQCtRREClR1w0r2dh1UN6PklVoKcWqepTWPJCfWtO1kTtAaVUtU4qSkIFJgovMX9lOfYCRSjpQUYqCVUUWkWCkpDFCmAWU9dlHicvizpe4jiY+SgkiJWHNHbfj60m+1JBaHFDclhSxoeWMpKJE5C9Wv/FX4uqkiiWCk1CgpiBSZ6C2q6q6UXFNSGKCe/wjVtyuFSZMpZbCUFIYg+g9Pv9ikaEWX0Kh/NLk4pCAoKWShry98JQIpdE0Vjay5I/U5nXNggVoNkpGSgkgJSveDpamiEZofTm3UP9pjPs2lE6vyGZ4UsFiTgpnVAYuAYUCTuy/sdfxY4GFgKrALmOXu7XHGlFHQlNZ6M1KKuj/Xc3rdURd+3iO3tQKpZDGQLiV1RRW92JKCmQ0D7gcuAzqAdWa2wt03R4p9Efhvd/+ImV0HfBtIbI3g6oaVmoUsZSHbrs++1lzqfZt29/nU4ih+cbYUpgFb3X0bgJktA2YC0aQwE/hm8Ho58M9mZu7uMcbVg+YgiPQv+lS4w/vSP1c62sKe08eNGN3v670v7XhHLlseasVkJc6kcArwVmS7A7iwrzLuftDM9gBjgHdiiShN15Am/IgMTqbWxkBu0IgOiO+fWM+aO74ZHou2PvpqoQA9H2PanQAiX/49uoSjs7/TtW6ySRolmmQsrh/lZnYNUOfuc4LtG4EL3X1+pMxrQZmOYPuXQZl3ep1rHjAv2Pwo8EYflx1LXAmlcJVbnVXf0ldudc5Xff/E3T+UqVCcLYXtwKmR7fHBvnRlOszsaGAUqQHnHtx9CbAk0wXNbL271ww64iJUbnVWfUtfudW50Oob55PX1gFnmNkEMzsGuA5Y0avMCuALwetrgJ/mczxBRER6iq2lEIwRzAeeInVL6lJ332Rm9wDr3X0F8EPg38xsK/AuqcQhIiIJiXWegruvAlb12ndn5HUX8LkcXjJjF1MJKrc6q76lr9zqXFD1jW2gWUREik+cYwoiIlJkijYpmNlSM3s7uK21e1+jmb1uZq+Y2RNmdmKSMeZaujpHjt1mZm5mY5OILQ591dfMbg7+njeZ2XeSii/X+vhMTzGzF8xso5mtN7NpScaYS2Z2qpmtNbPNwd/lLcH+0Wb2tJltCf5/UtKx5kI/9S2o762iTQrAQ0Bdr31PA2e7+znAm8A38h1UzB7iyDpjZqcClwO/zndAMXuIXvU1s+mkZsKf6+5nAf8rgbji8hBH/v1+B7jb3acAdwbbpeIgcJu7TwI+BtxkZpOABuAZdz8DeCbYLgV91begvreKNim4+7Ok7liK7lvt7geDzRdIzY0oGenqHPgn4GtASQ0Q9VHfvwUWuvt7QZm38x5YTPqorwMnBK9HATvyGlSM3P037r4heL0XaCO1ysFM4F+DYv8KXJ1MhLnVV30L7XuraJNCFv4G+M+kg4ibmc0Etrv7y0nHkidnAheb2Ytm9jMzuyDpgGJ2K9BoZm+RahWVWusXADOrBs4DXgSq3P03waHfAiW3yl6v+kYl/r1VkknBzP6BVFPtR0nHEiczGwH8T1LdCuXiaGA0qeb3AuAxM7NkQ4rV3wJ/5+6nAn9Ham5PSTGzSuBx4FZ3/330WDCZtaRawH3Vt1C+t0ouKZjZbOAzwPVlMDv6dGAC8LKZtZNqdm4wsz9KNKp4dQA/8ZSXgEOk1o4pVV8AfhK8/jGp1YdLhplVkPqC/JG7d9dzp5l9ODj+YaBkugj7qG9BfW+VVFIIHurzNWCGu/8h6Xji5u6vuvs4d69292pSX5jnu/tvEw4tTk8C0wHM7EzgGEp78bQdwJ8Fr/8c2JJgLDkVtPB+CLS5+72RQ9Hlb74A/J98xxaHvupbaN9bRTt5zcz+Hagl9StxJ3AXqf7WYzm8qN4L7v7lRAKMQbo6u/sPI8fbgZreq8wWqz7+jv8NWApMAd4HvuruP00qxlzqo75vkHp64dFAF/AVd29NKsZcMrNPAs8Br5Jq8UGqO/RF4DHgj4FfAde6e7obLIpKP/VdTAF9bxVtUhARkdwrqe4jEREZGiUFEREJKSmIiEhISUFEREJKCiIiElJSkJJgZp1DfP9yMzttgO/5spn91VCuO1Bm9qdm9ryZvWdmX43sP8bMng2edS4yaEoKUvbM7CxgmLtvG8B7jnb3B9394RzGUW1mLRmKvQv8D3qtDuvu75NaUXRWruKR8qSkICXFUhrN7DUze9XMZgX7jzKz7wfr1j9tZqvM7JrgbdcTmTVrZp1m9k/BmvfPmNmHgv0tZnafma0HbjGzb3b/Wjezj5jZGjN72cw2mNnpwf4FZrYuWCv/7qHWz93fdvd1wIE0h58M6iIyaEoKUmr+gtRs53OBS0mtMPrhYH81MAm4Efh45D0XAdFZwscD64PnNfyM1Mzibse4e427f7fXdX8E3O/u5wKfAH5jZpcDZ5Bar2gKMNXMLslJLdN7DSj1VWMlZup/lFLzSeDf3f0DUgur/YzUF+UngR+7+yHgt2a2NvKeDwO/i2wfAh4NXj/C4QXpiOwPmdlIUuviPwHg7l3B/stJPfzoF0HRSlJJ4tle73+C1MKGxwB/bGYbg0OL3P1fsq24u39gZu+b2chgvX6RAVNSEIH9wPB+jkfXgtk3gPMa8I/u/oP+Crn7ZyFcY/8hd68dwDV6O5bUGkkig6LuIyk1zwGzzGxYMBZwCfAS8P+AvwzGFqpILTzXrQ34SGT7KKB7vKEe+K/+Lhj8Ku8ws6sBzOzY4FkXTwF/E6yfj5mdYmbjhlrBvpjZGOAdd0833iCSFbUUpNQ8QWq84GVSv/C/5u6/NbPHgU8Bm4G3gA3AnuA9K0kliTXB9j5gmpndTmot/2zu6LkR+IGZ3UNqEPhz7r7azCYCzwfPAeoEbmAIzwcInpWxntQjOg+Z2a3ApOBhLdODuogMmlZJlbJhZpXu3hn8on4JuChIGMcBa4PtD8ys090rk4124MzsJ0CDu7+ZdCxSvNRSkHLyH2Z2IqkB3W91P4zI3feb2V2kHhr/6yQDHCwzOwZ4UglBhkotBRERCWmgWUREQkoKIiISUlIQEZGQkoKIiISUFEREJKSkICIiof8PULRfTKn0vh0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(cv['log_parea_total_price'], bins=100, label='train true', normed=True)\n",
    "plt.hist(np.log1p(test['total_price'] / test['building_area']), bins=100, label='test',\n",
    "         normed=True, alpha=0.7)\n",
    "plt.xlabel('log(price/area + 1)'); plt.ylabel('ratio')\n",
    "plt.legend(); plt.grid(); plt.show()\n",
    "\n",
    "plt.hist(cv['log_total_price'], bins=100, label='train true', normed=True)\n",
    "plt.hist(np.log1p(test['total_price']), bins=100, label='test', normed=True, alpha=0.7)\n",
    "plt.xlabel('log(price + 1)'); plt.ylabel('ratio')\n",
    "plt.legend(); plt.grid(); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.390394e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.924789e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.098352e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>5.987318e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.066824e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>UEeCDaAJzPwdKKKA</td>\n",
       "      <td>1.230010e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>i0fgbPaQsDWs7Q87</td>\n",
       "      <td>4.739660e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>YunNwAhcqkf6YclI</td>\n",
       "      <td>1.097613e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>A2NotxtRY9MYoWMl</td>\n",
       "      <td>2.985698e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>kKvgBXiA50gRmQhP</td>\n",
       "      <td>3.127253e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           building_id   total_price\n",
       "0     X5gsdTWGS3W7JJQB  1.390394e+07\n",
       "1     BTshNOJyKHnT2YIT  3.924789e+06\n",
       "2     dhdymr0lV8N5kZOT  1.098352e+07\n",
       "3     VEwyGGMcD56w5BOc  5.987318e+06\n",
       "4     wmUeMoJZfsqaSX9b  1.066824e+06\n",
       "9995  UEeCDaAJzPwdKKKA  1.230010e+06\n",
       "9996  i0fgbPaQsDWs7Q87  4.739660e+07\n",
       "9997  YunNwAhcqkf6YclI  1.097613e+06\n",
       "9998  A2NotxtRY9MYoWMl  2.985698e+06\n",
       "9999  kKvgBXiA50gRmQhP  3.127253e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape = (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "test[['building_id', 'total_price']].check()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output file to: output/stack_mean_18_1-10,12-22,24-29.csv\n"
     ]
    }
   ],
   "source": [
    "output_filename = 'output/stack_mean_{}_{}.csv'.format(stack_idx, models)\n",
    "print('Output file to:', output_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[['building_id', 'total_price']].to_csv(output_filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_id</th>\n",
       "      <th>total_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X5gsdTWGS3W7JJQB</td>\n",
       "      <td>1.390394e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BTshNOJyKHnT2YIT</td>\n",
       "      <td>3.924789e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dhdymr0lV8N5kZOT</td>\n",
       "      <td>1.098352e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VEwyGGMcD56w5BOc</td>\n",
       "      <td>5.987318e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>wmUeMoJZfsqaSX9b</td>\n",
       "      <td>1.066824e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>UEeCDaAJzPwdKKKA</td>\n",
       "      <td>1.230010e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>i0fgbPaQsDWs7Q87</td>\n",
       "      <td>4.739660e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>YunNwAhcqkf6YclI</td>\n",
       "      <td>1.097613e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>A2NotxtRY9MYoWMl</td>\n",
       "      <td>2.985698e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>kKvgBXiA50gRmQhP</td>\n",
       "      <td>3.127253e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           building_id   total_price\n",
       "0     X5gsdTWGS3W7JJQB  1.390394e+07\n",
       "1     BTshNOJyKHnT2YIT  3.924789e+06\n",
       "2     dhdymr0lV8N5kZOT  1.098352e+07\n",
       "3     VEwyGGMcD56w5BOc  5.987318e+06\n",
       "4     wmUeMoJZfsqaSX9b  1.066824e+06\n",
       "9995  UEeCDaAJzPwdKKKA  1.230010e+06\n",
       "9996  i0fgbPaQsDWs7Q87  4.739660e+07\n",
       "9997  YunNwAhcqkf6YclI  1.097613e+06\n",
       "9998  A2NotxtRY9MYoWMl  2.985698e+06\n",
       "9999  kKvgBXiA50gRmQhP  3.127253e+06"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape = (10000, 2)\n"
     ]
    }
   ],
   "source": [
    "pd.read_csv(output_filename).check()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "247px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}