main-notebook-cite.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4e4b61ff",
   "metadata": {
    "papermill": {
     "duration": 0.007745,
     "end_time": "2023-03-26T12:49:51.889234",
     "exception": false,
     "start_time": "2023-03-26T12:49:51.881489",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# **Main CITE Notebook**\n",
    "\n",
    "This is the main notebook for CITE part of the project, where the task is to predict normalized surface protein levels given information about library-size normalized and log1p transformed counts (gene expression levels) for the same cells.\n",
    "\n",
    "In this Jupyter notebook, data from several sources is joined together and is used further to either cross-validate or create predictions for the test dataset. The sources are:\n",
    "1. pre-calculated Truncated SVD values from all the RNA expression levels data (see Prepare_SVD_for_CITE notebook)\n",
    "2. source data for RNA gene expression levels - for pre-selected important genes this data is going to be put into the model as is\n",
    "3. metadata - cells' donor ID and day the cells were analyzed. Few features are built from metadata information.\n",
    "4. target values for the train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "70b4a14f",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-output": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-26T12:49:51.905914Z",
     "iopub.status.busy": "2023-03-26T12:49:51.904738Z",
     "iopub.status.idle": "2023-03-26T12:49:53.190964Z",
     "shell.execute_reply": "2023-03-26T12:49:53.189610Z"
    },
    "papermill": {
     "duration": 1.297749,
     "end_time": "2023-03-26T12:49:53.193860",
     "exception": false,
     "start_time": "2023-03-26T12:49:51.896111",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Importing the libraries\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "#import lightgbm\n",
    "from catboost import CatBoostRegressor\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5300fbba",
   "metadata": {
    "papermill": {
     "duration": 0.010366,
     "end_time": "2023-03-26T12:49:53.214337",
     "exception": false,
     "start_time": "2023-03-26T12:49:53.203971",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# **Loading the Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f55cfba8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:49:53.235635Z",
     "iopub.status.busy": "2023-03-26T12:49:53.235209Z",
     "iopub.status.idle": "2023-03-26T12:49:53.248160Z",
     "shell.execute_reply": "2023-03-26T12:49:53.247273Z"
    },
    "papermill": {
     "duration": 0.027176,
     "end_time": "2023-03-26T12:49:53.251231",
     "exception": false,
     "start_time": "2023-03-26T12:49:53.224055",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"/kaggle/input/open-problems-multimodal/\"\n",
    "FP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n",
    "\n",
    "FP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\n",
    "FP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\n",
    "FP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\n",
    "FP_CITE_TEST_INPUTS_FIX = os.path.join(DATA_DIR,\"test_cite_inputs_day_2_donor_27678.h5\")\n",
    "\n",
    "FP_MULTIOME_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_multi_inputs.h5\")\n",
    "FP_MULTIOME_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_multi_targets.h5\")\n",
    "FP_MULTIOME_TEST_INPUTS = os.path.join(DATA_DIR,\"test_multi_inputs.h5\")\n",
    "\n",
    "FP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\n",
    "FP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6a0abb29",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:49:53.271805Z",
     "iopub.status.busy": "2023-03-26T12:49:53.271363Z",
     "iopub.status.idle": "2023-03-26T12:49:53.287655Z",
     "shell.execute_reply": "2023-03-26T12:49:53.286723Z"
    },
    "papermill": {
     "duration": 0.030018,
     "end_time": "2023-03-26T12:49:53.290791",
     "exception": false,
     "start_time": "2023-03-26T12:49:53.260773",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load list of important columns\n",
    "important_columns_path = '../input/kagglegenes/important_columns.csv'\n",
    "df_imp_cols =  pd.read_csv(important_columns_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "88cda179",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:49:53.312358Z",
     "iopub.status.busy": "2023-03-26T12:49:53.311956Z",
     "iopub.status.idle": "2023-03-26T12:50:46.066375Z",
     "shell.execute_reply": "2023-03-26T12:50:46.065451Z"
    },
    "papermill": {
     "duration": 52.775996,
     "end_time": "2023-03-26T12:50:46.076842",
     "exception": false,
     "start_time": "2023-03-26T12:49:53.300846",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import finished\n",
      "CPU times: user 23.9 s, sys: 7.62 s, total: 31.5 s\n",
      "Wall time: 52.7 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(70988, 642)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Load train data. In this case, it is impossible to load only important columns, \n",
    "# so I have to load all the data, and then remove columns that are not in important columns list.\n",
    "df = pd.read_hdf(FP_CITE_TRAIN_INPUTS)\n",
    "df = df[df_imp_cols['important_columns']]\n",
    "print('import finished')\n",
    "gc.collect()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "824554c4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:50:46.093140Z",
     "iopub.status.busy": "2023-03-26T12:50:46.092317Z",
     "iopub.status.idle": "2023-03-26T12:50:46.135531Z",
     "shell.execute_reply": "2023-03-26T12:50:46.133998Z"
    },
    "papermill": {
     "duration": 0.054234,
     "end_time": "2023-03-26T12:50:46.138086",
     "exception": false,
     "start_time": "2023-03-26T12:50:46.083852",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187.2 MB\n"
     ]
    }
   ],
   "source": [
    "# Check dataframe size\n",
    "size = df.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "39a5fb8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:50:46.153206Z",
     "iopub.status.busy": "2023-03-26T12:50:46.152624Z",
     "iopub.status.idle": "2023-03-26T12:50:46.157965Z",
     "shell.execute_reply": "2023-03-26T12:50:46.157100Z"
    },
    "papermill": {
     "duration": 0.015287,
     "end_time": "2023-03-26T12:50:46.160301",
     "exception": false,
     "start_time": "2023-03-26T12:50:46.145014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# I made it possible to run this notebook with SUBMIT = False in case if I am going to run cross-validation only\n",
    "SUBMIT = True\n",
    "CROSS_VALIDATE = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f67e3886",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:50:46.174926Z",
     "iopub.status.busy": "2023-03-26T12:50:46.174251Z",
     "iopub.status.idle": "2023-03-26T12:51:19.033187Z",
     "shell.execute_reply": "2023-03-26T12:51:19.032128Z"
    },
    "papermill": {
     "duration": 32.875377,
     "end_time": "2023-03-26T12:51:19.042231",
     "exception": false,
     "start_time": "2023-03-26T12:50:46.166854",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import of test data finished\n",
      "(48663, 642)\n",
      "CPU times: user 16.3 s, sys: 2.47 s, total: 18.7 s\n",
      "Wall time: 32.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load the test data (this step is skipped if notebook runs only for cross-validation)\n",
    "\n",
    "if SUBMIT:\n",
    "    df_test = pd.read_hdf(FP_CITE_TEST_INPUTS)\n",
    "    df_test = df_test[df_imp_cols['important_columns']]\n",
    "    print('import of test data finished')\n",
    "    print(df_test.shape)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b8cf7b32",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:19.056743Z",
     "iopub.status.busy": "2023-03-26T12:51:19.056386Z",
     "iopub.status.idle": "2023-03-26T12:51:19.797401Z",
     "shell.execute_reply": "2023-03-26T12:51:19.796097Z"
    },
    "papermill": {
     "duration": 0.751162,
     "end_time": "2023-03-26T12:51:19.799997",
     "exception": false,
     "start_time": "2023-03-26T12:51:19.048835",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48663, 645)\n",
      "CPU times: user 529 ms, sys: 82 ms, total: 611 ms\n",
      "Wall time: 730 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(70988, 645)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Load metadata and select rows that refer to CITE\n",
    "md_df = pd.read_csv(FP_CELL_METADATA, index_col='cell_id')\n",
    "md_df = md_df.loc[md_df['technology'] == \"citeseq\"]\n",
    "md_df['day'] = md_df['day'].astype('int8')\n",
    "del md_df['technology']\n",
    "# Join the metadata to train.\n",
    "df = df.join(md_df, how = 'left', on = 'cell_id')\n",
    "# Join the metadata to test.\n",
    "if SUBMIT:\n",
    "    df_test = df_test.join(md_df, how = 'left', on = 'cell_id')\n",
    "    print(df_test.shape)\n",
    "del md_df\n",
    "gc.collect()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9193e0ff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:19.815361Z",
     "iopub.status.busy": "2023-03-26T12:51:19.815072Z",
     "iopub.status.idle": "2023-03-26T12:51:47.245273Z",
     "shell.execute_reply": "2023-03-26T12:51:47.243682Z"
    },
    "papermill": {
     "duration": 27.440638,
     "end_time": "2023-03-26T12:51:47.248013",
     "exception": false,
     "start_time": "2023-03-26T12:51:19.807375",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70988, 512)\n",
      "(41187, 512)\n",
      "(70988, 1158)\n",
      "(41187, 1158)\n",
      "CPU times: user 14.2 s, sys: 1.42 s, total: 15.6 s\n",
      "Wall time: 27.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load the prepared TruncatedSVD data and join it to the rest of train/test datasets.\n",
    "# There was a mess with organizers initially publishing wrong data as part of test set.\n",
    "# They've later released a fix (see FP_CITE_TEST_INPUTS_FIX) for information only.\n",
    "# This means participants didn't have to make predictions for the fixed data, but could use it for their models.\n",
    "# I used that data to calculate the TruncatedSVD.\n",
    "# This is why this part of code is not as simple as it could be otherwise.\n",
    "\n",
    "svd_path = '../input/notebookb136a7c42f/svd.csv'\n",
    "svd = pd.read_csv(svd_path)\n",
    "svd = svd.add_prefix('svd_')\n",
    "svd_train = svd.iloc[:70988]\n",
    "print(svd_train.shape)\n",
    "\n",
    "#join the svd data to train\n",
    "df = df.reset_index()\n",
    "df = pd.concat([df, svd_train], axis=1)\n",
    "\n",
    "#join the svd data to test\n",
    "if SUBMIT:\n",
    "    df_test = df_test.iloc[7476:]\n",
    "    df_test = df_test.reset_index()\n",
    "    svd_test = svd.iloc[70988:112175].reset_index(drop = True)\n",
    "    print(svd_test.shape)\n",
    "    df_test = pd.concat([df_test, svd_test], axis=1)\n",
    "    del svd_test\n",
    "\n",
    "del svd, svd_train\n",
    "gc.collect()\n",
    "print(df.shape)\n",
    "if SUBMIT:\n",
    "    print(df_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e78a4466",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:47.264629Z",
     "iopub.status.busy": "2023-03-26T12:51:47.264272Z",
     "iopub.status.idle": "2023-03-26T12:51:48.232831Z",
     "shell.execute_reply": "2023-03-26T12:51:48.231482Z"
    },
    "papermill": {
     "duration": 0.979339,
     "end_time": "2023-03-26T12:51:48.235672",
     "exception": false,
     "start_time": "2023-03-26T12:51:47.256333",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Y shape: (70988, 140)   0.037 GByte\n",
      "CPU times: user 419 ms, sys: 123 ms, total: 542 ms\n",
      "Wall time: 960 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(70988, 1298)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Import the target values and merge them into train dataframe.\n",
    "\n",
    "Y = pd.read_hdf(FP_CITE_TRAIN_TARGETS)\n",
    "Y = Y.add_prefix('y_')\n",
    "print(f\"Y shape: {str(Y.shape):14} {Y.size*4/1024/1024/1024:2.3f} GByte\")\n",
    "df = df.join(Y, how = 'left', on = 'cell_id')\n",
    "\n",
    "del Y\n",
    "gc.collect()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fe5dbad8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.251941Z",
     "iopub.status.busy": "2023-03-26T12:51:48.251300Z",
     "iopub.status.idle": "2023-03-26T12:51:48.265363Z",
     "shell.execute_reply": "2023-03-26T12:51:48.264453Z"
    },
    "papermill": {
     "duration": 0.024328,
     "end_time": "2023-03-26T12:51:48.267325",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.242997",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n# Experimented with some manually crafted features from raw data, but decided not to include them into final submission.\\nraw_features_path = '../input/values-from-raw/features_from_raw.ftr'\\ndf_raw_features = pd.read_feather(raw_features_path)\\ndf = df.merge(df_raw_features, how = 'left', on = 'cell_id')\\ndel df_raw_features\\ngc.collect()\\nprint(df.shape)\\nif SUBMIT:\\n    test_raw_features_path = '../input/values-from-raw/features_from_raw_test.ftr'\\n    df_raw_features_test = pd.read_feather(test_raw_features_path)\\n    df_test = df_test.merge(df_raw_features_test, how = 'left', on = 'cell_id')\\n    del df_raw_features_test\\n    gc.collect()\\n    print(df_test.shape)\\n\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# Experimented with some manually crafted features from raw data, but decided not to include them into final submission.\n",
    "raw_features_path = '../input/values-from-raw/features_from_raw.ftr'\n",
    "df_raw_features = pd.read_feather(raw_features_path)\n",
    "df = df.merge(df_raw_features, how = 'left', on = 'cell_id')\n",
    "del df_raw_features\n",
    "gc.collect()\n",
    "print(df.shape)\n",
    "if SUBMIT:\n",
    "    test_raw_features_path = '../input/values-from-raw/features_from_raw_test.ftr'\n",
    "    df_raw_features_test = pd.read_feather(test_raw_features_path)\n",
    "    df_test = df_test.merge(df_raw_features_test, how = 'left', on = 'cell_id')\n",
    "    del df_raw_features_test\n",
    "    gc.collect()\n",
    "    print(df_test.shape)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e10739dd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.284293Z",
     "iopub.status.busy": "2023-03-26T12:51:48.282675Z",
     "iopub.status.idle": "2023-03-26T12:51:48.291290Z",
     "shell.execute_reply": "2023-03-26T12:51:48.290464Z"
    },
    "papermill": {
     "duration": 0.018834,
     "end_time": "2023-03-26T12:51:48.293408",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.274574",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nsvd_raw_path = '../input/svd-from-raw-data/svd_raw.csv'\\nsvd_raw = pd.read_csv(svd_raw_path)\\nsvd_raw = svd_raw.add_prefix('ext_')\\nsvd_train_raw = svd_raw.iloc[:70988, :32]\\n#svd = pd.DataFrame(svd, dtype='float16')\\nprint(svd_train_raw.shape)\\n\\n#join the svd data to train\\ndf = pd.concat([df, svd_train_raw], axis=1)\\n\\n#join the svd data to test\\nif SUBMIT:\\n    print(df_test.shape)\\n    svd_test_raw = svd_raw.iloc[70988:112175, :32].reset_index(drop=True)\\n    print(svd_test_raw.shape)\\n    df_test = pd.concat([df_test, svd_test_raw], axis=1)\\n    del svd_test_raw\\n\\ndel svd_raw, svd_train_raw\\ngc.collect()\\nprint(df.shape)\\nif SUBMIT:\\n    print(df_test.shape)\\n\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Made experiments with using TruncatedSVD built on raw data, but cross-validation showed it only decreases\n",
    "# model's performance, even if I only take first 32 components.\n",
    "'''\n",
    "svd_raw_path = '../input/svd-from-raw-data/svd_raw.csv'\n",
    "svd_raw = pd.read_csv(svd_raw_path)\n",
    "svd_raw = svd_raw.add_prefix('ext_')\n",
    "svd_train_raw = svd_raw.iloc[:70988, :32]\n",
    "#svd = pd.DataFrame(svd, dtype='float16')\n",
    "print(svd_train_raw.shape)\n",
    "\n",
    "#join the svd data to train\n",
    "df = pd.concat([df, svd_train_raw], axis=1)\n",
    "\n",
    "#join the svd data to test\n",
    "if SUBMIT:\n",
    "    print(df_test.shape)\n",
    "    svd_test_raw = svd_raw.iloc[70988:112175, :32].reset_index(drop=True)\n",
    "    print(svd_test_raw.shape)\n",
    "    df_test = pd.concat([df_test, svd_test_raw], axis=1)\n",
    "    del svd_test_raw\n",
    "\n",
    "del svd_raw, svd_train_raw\n",
    "gc.collect()\n",
    "print(df.shape)\n",
    "if SUBMIT:\n",
    "    print(df_test.shape)\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abc3a4f7",
   "metadata": {
    "papermill": {
     "duration": 0.006993,
     "end_time": "2023-03-26T12:51:48.307774",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.300781",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# **Cross-validation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "819d744f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.323898Z",
     "iopub.status.busy": "2023-03-26T12:51:48.323538Z",
     "iopub.status.idle": "2023-03-26T12:51:48.328727Z",
     "shell.execute_reply": "2023-03-26T12:51:48.327654Z"
    },
    "papermill": {
     "duration": 0.01652,
     "end_time": "2023-03-26T12:51:48.331722",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.315202",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lightgbm_params = {\n",
    "     'learning_rate': 0.05, \n",
    "     'max_depth': 7, \n",
    "     'min_child_samples': 100,\n",
    "     'subsample': 0.6, \n",
    "     \"seed\": 1,\n",
    "     \"device\": \"gpu\",\n",
    "     \"gpu_platform_id\": 0,\n",
    "     \"gpu_device_id\": 0,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d6a47907",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.347129Z",
     "iopub.status.busy": "2023-03-26T12:51:48.346849Z",
     "iopub.status.idle": "2023-03-26T12:51:48.353778Z",
     "shell.execute_reply": "2023-03-26T12:51:48.352877Z"
    },
    "papermill": {
     "duration": 0.017023,
     "end_time": "2023-03-26T12:51:48.355814",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.338791",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Function to calculate the competition's custom metric.\n",
    "\n",
    "def correlation_score(y_true, y_pred):\n",
    "    if type(y_true) == pd.DataFrame: y_true = y_true.values\n",
    "    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n",
    "    corrsum = 0\n",
    "    for i in range(len(y_true)):\n",
    "        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n",
    "    return corrsum / len(y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "972d3fe3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.372458Z",
     "iopub.status.busy": "2023-03-26T12:51:48.371653Z",
     "iopub.status.idle": "2023-03-26T12:51:48.378089Z",
     "shell.execute_reply": "2023-03-26T12:51:48.377170Z"
    },
    "papermill": {
     "duration": 0.017453,
     "end_time": "2023-03-26T12:51:48.380342",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.362889",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if CROSS_VALIDATE:\n",
    "    df['svd_var1'] = 0\n",
    "    #df['svd_var2'] = 0\n",
    "    df['svd_var3'] = 0\n",
    "    df['svd_var4'] = 0\n",
    "    df['svd_sex'] = 0\n",
    "    df.loc[df['donor'] == 13176, 'svd_sex'] = 1\n",
    "    \n",
    "    x_cols = [col for col in list(df.columns) if (col.startswith('ENSG')) | (col.startswith('svd'))]\n",
    "    y_cols = [col for col in list(df.columns) if (col.startswith('y_'))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "937be065",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.396020Z",
     "iopub.status.busy": "2023-03-26T12:51:48.395711Z",
     "iopub.status.idle": "2023-03-26T12:51:48.402787Z",
     "shell.execute_reply": "2023-03-26T12:51:48.401858Z"
    },
    "papermill": {
     "duration": 0.017612,
     "end_time": "2023-03-26T12:51:48.404995",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.387383",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# At some point, close to competition end decided to use different catboost parameters depending on target.\n",
    "cat_params_slow = {\n",
    "    \"random_state\": 23,\n",
    "    \"learning_rate\" : 0.02,\n",
    "    \"eval_metric\" : 'RMSE', \n",
    "    \"max_depth\" : 7,\n",
    "    \"verbose\" : 100,\n",
    "    \"n_estimators\" : 300,\n",
    "    \"task_type\" : 'GPU'\n",
    "    }\n",
    "cat_params_middle = {\n",
    "    \"random_state\": 23,\n",
    "    \"learning_rate\" : 0.03,\n",
    "    \"eval_metric\" : 'RMSE', \n",
    "    \"max_depth\" : 7,\n",
    "    \"verbose\" : 100,\n",
    "    \"n_estimators\" : 400,\n",
    "    \"task_type\" : 'GPU'\n",
    "    }\n",
    "cat_params_fast = {\n",
    "    \"random_state\": 23,\n",
    "    \"learning_rate\" : 0.05,\n",
    "    \"eval_metric\" : 'RMSE', \n",
    "    \"max_depth\" : 7,\n",
    "    \"verbose\" : 100,\n",
    "    \"n_estimators\" : 500,\n",
    "    \"task_type\" : 'GPU'\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3e09eaef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.420574Z",
     "iopub.status.busy": "2023-03-26T12:51:48.420291Z",
     "iopub.status.idle": "2023-03-26T12:51:48.435811Z",
     "shell.execute_reply": "2023-03-26T12:51:48.434936Z"
    },
    "papermill": {
     "duration": 0.025792,
     "end_time": "2023-03-26T12:51:48.438036",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.412244",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define the targets' grouping depending on how well they can be predicted.\n",
    "df_model_groups = pd.read_csv('../input/model-groups/model_groups.csv')\n",
    "df_model_groups['mean'] = df_model_groups.mean(axis=1, numeric_only=True)\n",
    "df_model_groups['group'] = 2\n",
    "df_model_groups.loc[df_model_groups['mean'] < 0.25, 'group'] = 1\n",
    "df_model_groups.loc[df_model_groups['mean'] > 0.5 , 'group'] = 3\n",
    "\n",
    "y_slow = list(df_model_groups.loc[df_model_groups['group'] == 1, 'columns'])\n",
    "y_fast = list(df_model_groups.loc[df_model_groups['group'] == 3, 'columns'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e2edab60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.454742Z",
     "iopub.status.busy": "2023-03-26T12:51:48.454382Z",
     "iopub.status.idle": "2023-03-26T12:51:48.462666Z",
     "shell.execute_reply": "2023-03-26T12:51:48.461581Z"
    },
    "papermill": {
     "duration": 0.018566,
     "end_time": "2023-03-26T12:51:48.464700",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.446134",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Used this to run cross-validation only on 5 targets of 140 - for test.\n",
    "#y_cols = y_cols[:5]\n",
    "\n",
    "# Function to perform cv split by day or by donor.\n",
    "# Closer to the end of competition, cross-validated only on days.\n",
    "# Also added cross-validation with evaluation set to check how well models converge.\n",
    "def custom_cv_split(option, d_frame, USE_EVAL_SET=False):\n",
    "    d_frame['svd_var1'] = d_frame['day']\n",
    "    if option > 20:\n",
    "        cv_col = 'donor'\n",
    "    else:\n",
    "        d_frame.loc[d_frame['donor'] == 31800, 'svd_var3'] = 1\n",
    "        d_frame.loc[d_frame['donor'] == 32606, 'svd_var4'] = 1\n",
    "        cv_col = 'day'\n",
    "    Xc_tr = d_frame.loc[d_frame[cv_col] != option, x_cols]\n",
    "    Xc_va = d_frame.loc[d_frame[cv_col] == option, x_cols]\n",
    "    yc_va = d_frame.loc[d_frame[cv_col] == option, y_cols]\n",
    "    if USE_EVAL_SET:\n",
    "        Xc_tr_cv = Xc_tr.sample(frac=0.1, random_state=25)\n",
    "        Xc_tr = Xc_tr.drop(Xc_tr_cv.index)\n",
    "        yc_tr_cv = d_frame[y_cols].iloc[Xc_tr_cv.index]\n",
    "        yc_tr = d_frame[y_cols].iloc[Xc_tr.index]\n",
    "        return Xc_tr, yc_tr, Xc_va, yc_va, Xc_tr_cv, yc_tr_cv\n",
    "    else:\n",
    "        yc_tr = d_frame[y_cols].iloc[Xc_tr.index]\n",
    "        return Xc_tr, yc_tr, Xc_va, yc_va"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f8f49da5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.480782Z",
     "iopub.status.busy": "2023-03-26T12:51:48.479996Z",
     "iopub.status.idle": "2023-03-26T12:51:48.490329Z",
     "shell.execute_reply": "2023-03-26T12:51:48.489003Z"
    },
    "papermill": {
     "duration": 0.021861,
     "end_time": "2023-03-26T12:51:48.493684",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.471823",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 0 ns, total: 4 µs\n",
      "Wall time: 7.39 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Cross-validation with LGBMRegressor or CatboostRegressor in a loop.\n",
    "\n",
    "if CROSS_VALIDATE:\n",
    "    df_mse_corr = pd.DataFrame({'columns': y_cols})\n",
    "    df_importances = pd.DataFrame({'columns':x_cols})\n",
    "    #cv_options = [2,3,4, 32606, 31800]\n",
    "    cv_options = [2,3,4]\n",
    "    score_list = []\n",
    "    for cv_option in cv_options:\n",
    "        X_tr, y_tr, X_va, y_va = custom_cv_split(cv_option, df)\n",
    "        mse_list = []\n",
    "        corr_list = []\n",
    "        for i in range(len(y_cols)):\n",
    "            print(str(cv_option) + '_' + str(i) + '_' + y_cols[i])\n",
    "            column_name = str(cv_option) + '_' + y_cols[i]\n",
    "            y_column_name = 'pred_' + y_cols[i]\n",
    "            #model = lightgbm.LGBMRegressor(**lightgbm_params)\n",
    "            if y_cols[i] in y_slow:\n",
    "                model = CatBoostRegressor(**cat_params_slow)\n",
    "            elif y_cols[i] in y_fast:\n",
    "                model = CatBoostRegressor(**cat_params_fast)\n",
    "            else:\n",
    "                model = CatBoostRegressor(**cat_params_middle)\n",
    "            model.fit(X_tr, y_tr.iloc[:,i].copy())\n",
    "            y_va[y_column_name] = model.predict(X_va)\n",
    "            df_importances[column_name] = model.feature_importances_\n",
    "            mse_value = mean_squared_error(y_va.iloc[:,i], y_va.iloc[:,i+len(y_cols)], squared=False)\n",
    "            corr_value = np.corrcoef(y_va.iloc[:,i], y_va.iloc[:,i+len(y_cols)])[1, 0]\n",
    "            mse_list.append(mse_value)\n",
    "            corr_list.append(corr_value)\n",
    "            print(str(mse_value) + '_' + str(corr_value))\n",
    "        gc.collect()\n",
    "        \n",
    "        # Save the metric values for all 140 models\n",
    "        cv_option_corr = str(cv_option) + '_corr'\n",
    "        cv_option_mse = str(cv_option) + '_mse'\n",
    "        df_mse_corr[cv_option_corr] = corr_list\n",
    "        df_mse_corr[cv_option_mse] = mse_list   \n",
    "\n",
    "        # Validate the model (mse and correlation over all 140 columns)\n",
    "        y_preds = ['pred_' + y for y in y_cols]\n",
    "        mse = mean_squared_error(y_va[y_cols], y_va[y_preds])\n",
    "        corrscore = correlation_score(y_va[y_cols], y_va[y_preds])         \n",
    "\n",
    "        print(f\"Fold {cv_option}: mse = {mse:.5f}, corr =  {corrscore:.5f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3c319713",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.509931Z",
     "iopub.status.busy": "2023-03-26T12:51:48.509577Z",
     "iopub.status.idle": "2023-03-26T12:51:48.515502Z",
     "shell.execute_reply": "2023-03-26T12:51:48.514391Z"
    },
    "papermill": {
     "duration": 0.016417,
     "end_time": "2023-03-26T12:51:48.517475",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.501058",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Export the cross-validation predictions for a second-level model.\n",
    "# Wasn't used in final submission.\n",
    "\n",
    "if CROSS_VALIDATE & SUBMIT:\n",
    "    for cols in df_cv_pred:\n",
    "        if col in y_cols:\n",
    "            del df_cv_pred[col]\n",
    "    gc.collect()\n",
    "    df_cv_pred = df_cv_pred.reset_index()\n",
    "    export_file_results = 'results_cite_cv.ftr'\n",
    "    df_cv_pred.to_feather(export_file_results)\n",
    "    del df_cv_pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6edc80ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.534671Z",
     "iopub.status.busy": "2023-03-26T12:51:48.533066Z",
     "iopub.status.idle": "2023-03-26T12:51:48.641828Z",
     "shell.execute_reply": "2023-03-26T12:51:48.640758Z"
    },
    "papermill": {
     "duration": 0.119304,
     "end_time": "2023-03-26T12:51:48.644237",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.524933",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if CROSS_VALIDATE:\n",
    "    for col in ['svd_var1', 'svd_var2', 'svd_var3', 'svd_var4', 'svd_sex']:\n",
    "        if col in df.columns:\n",
    "            del df[col]\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c29f93b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.660952Z",
     "iopub.status.busy": "2023-03-26T12:51:48.660656Z",
     "iopub.status.idle": "2023-03-26T12:51:48.763959Z",
     "shell.execute_reply": "2023-03-26T12:51:48.762838Z"
    },
    "papermill": {
     "duration": 0.114272,
     "end_time": "2023-03-26T12:51:48.766298",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.652026",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the feature importance data.\n",
    "\n",
    "if CROSS_VALIDATE & SUBMIT:\n",
    "    export_file_importances = 'imps_cite.ftr'\n",
    "    df_importances.to_feather(export_file_importances)\n",
    "    del df_importances\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f73a284",
   "metadata": {
    "papermill": {
     "duration": 0.007327,
     "end_time": "2023-03-26T12:51:48.781738",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.774411",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# **Prediction**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8373bc12",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.798599Z",
     "iopub.status.busy": "2023-03-26T12:51:48.798267Z",
     "iopub.status.idle": "2023-03-26T12:51:48.805088Z",
     "shell.execute_reply": "2023-03-26T12:51:48.803358Z"
    },
    "papermill": {
     "duration": 0.017324,
     "end_time": "2023-03-26T12:51:48.807166",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.789842",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create features from metadata.\n",
    "# Note: I couldn't use get_dummies function here, as one of days and one of donors only appear in test dataset.\n",
    "\n",
    "def add_metadata_features(d_frame):\n",
    "    d_frame['svd_x_donor_13176'] = 0\n",
    "    d_frame['svd_x_donor_31800'] = 0\n",
    "    d_frame['svd_x_donor_32606'] = 0\n",
    "    d_frame.loc[d_frame['donor'] == 13176, 'svd_x_donor_13176'] = 1\n",
    "    d_frame.loc[d_frame['donor'] == 31800, 'svd_x_donor_31800'] = 1\n",
    "    d_frame.loc[d_frame['donor'] == 32606, 'svd_x_donor_32606'] = 1\n",
    "    d_frame['svd_x_day'] = d_frame['day']\n",
    "    return d_frame\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "983cbb98",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.823408Z",
     "iopub.status.busy": "2023-03-26T12:51:48.823128Z",
     "iopub.status.idle": "2023-03-26T12:51:48.830128Z",
     "shell.execute_reply": "2023-03-26T12:51:48.829120Z"
    },
    "papermill": {
     "duration": 0.017887,
     "end_time": "2023-03-26T12:51:48.832723",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.814836",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nif SUBMIT:\\n    df = add_metadata_features(df)\\n    df_test = add_metadata_features(df_test)\\n    x_cols = [col for col in list(df.columns) if (col.startswith('ENSG')) | (col.startswith('svd'))]\\n    y_cols = [col for col in list(df.columns) if (col.startswith('y_'))]\\n    best_iteration = []\\n    best_cv_result = []\\n    #y_cols = y_cols[:5]\\n    df_cv = df.sample(frac=0.1, random_state=25)\\n    df= df.drop(df_cv.index)\\n    X = df[x_cols].values\\n    Y = df[y_cols].values\\n    X_cv = df_cv[x_cols].values\\n    Y_cv = df_cv[y_cols].values\\n    for i in range(len(y_cols)):\\n        print('Training_column: ' + str(i))\\n        #model = lightgbm.LGBMRegressor(n_estimators=n_estimators, **lightgbm_params)\\n        #model = CatBoostRegressor(n_estimators=n_estimators, **cat_params)\\n        if y_cols[i] in y_slow:\\n            model = CatBoostRegressor(**cat_params_slow)\\n        elif y_cols[i] in y_fast:\\n            model = CatBoostRegressor(**cat_params_fast)\\n        else:\\n            model = CatBoostRegressor(**cat_params_middle)\\n        model.fit(X, Y[:,i].copy(), eval_set=[(X_cv, Y_cv[:,i].copy())])\\n        best_iteration.append(model.best_iteration_)\\n        best_cv_result.append(correlation_value)\\n        correlation_value = np.corrcoef(model.predict(X_cv),  Y_cv[:,i])[1, 0]\\n        print(str(correlation_value))\\n\""
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Predict. Ran a few time with cross-validation on 10% of data to check if it makes sense to add more\n",
    "# iterations. \n",
    "'''\n",
    "if SUBMIT:\n",
    "    df = add_metadata_features(df)\n",
    "    df_test = add_metadata_features(df_test)\n",
    "    x_cols = [col for col in list(df.columns) if (col.startswith('ENSG')) | (col.startswith('svd'))]\n",
    "    y_cols = [col for col in list(df.columns) if (col.startswith('y_'))]\n",
    "    best_iteration = []\n",
    "    best_cv_result = []\n",
    "    #y_cols = y_cols[:5]\n",
    "    df_cv = df.sample(frac=0.1, random_state=25)\n",
    "    df= df.drop(df_cv.index)\n",
    "    X = df[x_cols].values\n",
    "    Y = df[y_cols].values\n",
    "    X_cv = df_cv[x_cols].values\n",
    "    Y_cv = df_cv[y_cols].values\n",
    "    for i in range(len(y_cols)):\n",
    "        print('Training_column: ' + str(i))\n",
    "        #model = lightgbm.LGBMRegressor(n_estimators=n_estimators, **lightgbm_params)\n",
    "        #model = CatBoostRegressor(n_estimators=n_estimators, **cat_params)\n",
    "        if y_cols[i] in y_slow:\n",
    "            model = CatBoostRegressor(**cat_params_slow)\n",
    "        elif y_cols[i] in y_fast:\n",
    "            model = CatBoostRegressor(**cat_params_fast)\n",
    "        else:\n",
    "            model = CatBoostRegressor(**cat_params_middle)\n",
    "        model.fit(X, Y[:,i].copy(), eval_set=[(X_cv, Y_cv[:,i].copy())])\n",
    "        best_iteration.append(model.best_iteration_)\n",
    "        best_cv_result.append(correlation_value)\n",
    "        correlation_value = np.corrcoef(model.predict(X_cv),  Y_cv[:,i])[1, 0]\n",
    "        print(str(correlation_value))\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ef3e8a05",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T12:51:48.849882Z",
     "iopub.status.busy": "2023-03-26T12:51:48.848933Z",
     "iopub.status.idle": "2023-03-26T13:34:44.628968Z",
     "shell.execute_reply": "2023-03-26T13:34:44.628096Z"
    },
    "papermill": {
     "duration": 2575.791229,
     "end_time": "2023-03-26T13:34:44.631581",
     "exception": false,
     "start_time": "2023-03-26T12:51:48.840352",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training_column: 0\n",
      "0:\tlearn: 1.3880658\ttotal: 37.9ms\tremaining: 15.1s\n",
      "100:\tlearn: 1.2274369\ttotal: 2.89s\tremaining: 8.56s\n",
      "200:\tlearn: 1.2000120\ttotal: 5.27s\tremaining: 5.22s\n",
      "300:\tlearn: 1.1826643\ttotal: 7.69s\tremaining: 2.53s\n",
      "399:\tlearn: 1.1691044\ttotal: 10.1s\tremaining: 0us\n",
      "Training_column: 1\n",
      "0:\tlearn: 0.8930985\ttotal: 28.2ms\tremaining: 8.44s\n",
      "100:\tlearn: 0.8547100\ttotal: 2.59s\tremaining: 5.1s\n",
      "200:\tlearn: 0.8490743\ttotal: 5.46s\tremaining: 2.69s\n",
      "299:\tlearn: 0.8451139\ttotal: 7.99s\tremaining: 0us\n",
      "Training_column: 2\n",
      "0:\tlearn: 0.9192535\ttotal: 29.1ms\tremaining: 11.6s\n",
      "100:\tlearn: 0.8435130\ttotal: 2.95s\tremaining: 8.74s\n",
      "200:\tlearn: 0.8361452\ttotal: 5.53s\tremaining: 5.47s\n",
      "300:\tlearn: 0.8302002\ttotal: 8.13s\tremaining: 2.67s\n",
      "399:\tlearn: 0.8246224\ttotal: 10.7s\tremaining: 0us\n",
      "Training_column: 3\n",
      "0:\tlearn: 2.4953868\ttotal: 27.9ms\tremaining: 13.9s\n",
      "100:\tlearn: 1.7882656\ttotal: 2.69s\tremaining: 10.6s\n",
      "200:\tlearn: 1.7434499\ttotal: 5.61s\tremaining: 8.34s\n",
      "300:\tlearn: 1.7154959\ttotal: 8.19s\tremaining: 5.42s\n",
      "400:\tlearn: 1.6922450\ttotal: 10.8s\tremaining: 2.67s\n",
      "499:\tlearn: 1.6704558\ttotal: 13.4s\tremaining: 0us\n",
      "Training_column: 4\n",
      "0:\tlearn: 1.9989614\ttotal: 29.9ms\tremaining: 14.9s\n",
      "100:\tlearn: 1.2490464\ttotal: 2.74s\tremaining: 10.8s\n",
      "200:\tlearn: 1.2038992\ttotal: 5.33s\tremaining: 7.93s\n",
      "300:\tlearn: 1.1811843\ttotal: 8.95s\tremaining: 5.92s\n",
      "400:\tlearn: 1.1637426\ttotal: 11.5s\tremaining: 2.84s\n",
      "499:\tlearn: 1.1491024\ttotal: 14s\tremaining: 0us\n",
      "Training_column: 5\n",
      "0:\tlearn: 3.5345708\ttotal: 28.9ms\tremaining: 14.4s\n",
      "100:\tlearn: 2.5568859\ttotal: 2.71s\tremaining: 10.7s\n",
      "200:\tlearn: 2.4670998\ttotal: 5.29s\tremaining: 7.87s\n",
      "300:\tlearn: 2.4208373\ttotal: 8.19s\tremaining: 5.41s\n",
      "400:\tlearn: 2.3853012\ttotal: 10.7s\tremaining: 2.65s\n",
      "499:\tlearn: 2.3526767\ttotal: 13.3s\tremaining: 0us\n",
      "Training_column: 6\n",
      "0:\tlearn: 5.1919720\ttotal: 27.5ms\tremaining: 13.7s\n",
      "100:\tlearn: 2.8738063\ttotal: 2.6s\tremaining: 10.3s\n",
      "200:\tlearn: 2.7402469\ttotal: 5.13s\tremaining: 7.62s\n",
      "300:\tlearn: 2.6674878\ttotal: 7.63s\tremaining: 5.04s\n",
      "400:\tlearn: 2.6092070\ttotal: 10.5s\tremaining: 2.6s\n",
      "499:\tlearn: 2.5593753\ttotal: 13.1s\tremaining: 0us\n",
      "Training_column: 7\n",
      "0:\tlearn: 1.0511962\ttotal: 35.9ms\tremaining: 14.3s\n",
      "100:\tlearn: 0.9565328\ttotal: 2.53s\tremaining: 7.49s\n",
      "200:\tlearn: 0.9440935\ttotal: 4.93s\tremaining: 4.88s\n",
      "300:\tlearn: 0.9366246\ttotal: 7.44s\tremaining: 2.45s\n",
      "399:\tlearn: 0.9297992\ttotal: 10.2s\tremaining: 0us\n",
      "Training_column: 8\n",
      "0:\tlearn: 0.8539888\ttotal: 27.1ms\tremaining: 10.8s\n",
      "100:\tlearn: 0.7662726\ttotal: 2.7s\tremaining: 7.99s\n",
      "200:\tlearn: 0.7587391\ttotal: 5.67s\tremaining: 5.61s\n",
      "300:\tlearn: 0.7534909\ttotal: 8.28s\tremaining: 2.72s\n",
      "399:\tlearn: 0.7484489\ttotal: 10.9s\tremaining: 0us\n",
      "Training_column: 9\n",
      "0:\tlearn: 1.2372475\ttotal: 29.1ms\tremaining: 11.6s\n",
      "100:\tlearn: 1.0511632\ttotal: 2.63s\tremaining: 7.78s\n",
      "200:\tlearn: 1.0346380\ttotal: 5.09s\tremaining: 5.04s\n",
      "300:\tlearn: 1.0250182\ttotal: 8.53s\tremaining: 2.81s\n",
      "399:\tlearn: 1.0168127\ttotal: 11.1s\tremaining: 0us\n",
      "Training_column: 10\n",
      "0:\tlearn: 0.8466707\ttotal: 27.7ms\tremaining: 8.29s\n",
      "100:\tlearn: 0.8349249\ttotal: 2.88s\tremaining: 5.67s\n",
      "200:\tlearn: 0.8308837\ttotal: 5.4s\tremaining: 2.66s\n",
      "299:\tlearn: 0.8277737\ttotal: 7.86s\tremaining: 0us\n",
      "Training_column: 11\n",
      "0:\tlearn: 1.1547223\ttotal: 27.2ms\tremaining: 8.12s\n",
      "100:\tlearn: 1.1298439\ttotal: 2.5s\tremaining: 4.93s\n",
      "200:\tlearn: 1.1226814\ttotal: 4.96s\tremaining: 2.44s\n",
      "299:\tlearn: 1.1174444\ttotal: 8.11s\tremaining: 0us\n",
      "Training_column: 12\n",
      "0:\tlearn: 1.0441886\ttotal: 27.7ms\tremaining: 8.29s\n",
      "100:\tlearn: 0.9987802\ttotal: 2.58s\tremaining: 5.08s\n",
      "200:\tlearn: 0.9893992\ttotal: 5.24s\tremaining: 2.58s\n",
      "299:\tlearn: 0.9831483\ttotal: 7.93s\tremaining: 0us\n",
      "Training_column: 13\n",
      "0:\tlearn: 1.1611189\ttotal: 25.5ms\tremaining: 10.2s\n",
      "100:\tlearn: 1.0266231\ttotal: 2.71s\tremaining: 8.03s\n",
      "200:\tlearn: 1.0079705\ttotal: 5.09s\tremaining: 5.04s\n",
      "300:\tlearn: 0.9993504\ttotal: 7.5s\tremaining: 2.46s\n",
      "399:\tlearn: 0.9920209\ttotal: 10.6s\tremaining: 0us\n",
      "Training_column: 14\n",
      "0:\tlearn: 4.9136628\ttotal: 30ms\tremaining: 15s\n",
      "100:\tlearn: 3.6351560\ttotal: 2.73s\tremaining: 10.8s\n",
      "200:\tlearn: 3.5380432\ttotal: 5.33s\tremaining: 7.93s\n",
      "300:\tlearn: 3.4773453\ttotal: 8.3s\tremaining: 5.49s\n",
      "400:\tlearn: 3.4277666\ttotal: 11s\tremaining: 2.71s\n",
      "499:\tlearn: 3.3828022\ttotal: 13.6s\tremaining: 0us\n",
      "Training_column: 15\n",
      "0:\tlearn: 2.7237763\ttotal: 24.6ms\tremaining: 12.3s\n",
      "100:\tlearn: 2.0790633\ttotal: 2.43s\tremaining: 9.6s\n",
      "200:\tlearn: 2.0162773\ttotal: 5.81s\tremaining: 8.64s\n",
      "300:\tlearn: 1.9723038\ttotal: 8.56s\tremaining: 5.66s\n",
      "400:\tlearn: 1.9378675\ttotal: 11s\tremaining: 2.71s\n",
      "499:\tlearn: 1.9086435\ttotal: 13.4s\tremaining: 0us\n",
      "Training_column: 16\n",
      "0:\tlearn: 3.2011770\ttotal: 30.3ms\tremaining: 15.1s\n",
      "100:\tlearn: 2.2956680\ttotal: 2.76s\tremaining: 10.9s\n",
      "200:\tlearn: 2.2089559\ttotal: 5.32s\tremaining: 7.92s\n",
      "300:\tlearn: 2.1622787\ttotal: 7.86s\tremaining: 5.2s\n",
      "400:\tlearn: 2.1261506\ttotal: 10.8s\tremaining: 2.67s\n",
      "499:\tlearn: 2.0949609\ttotal: 13.4s\tremaining: 0us\n",
      "Training_column: 17\n",
      "0:\tlearn: 2.6453677\ttotal: 34.3ms\tremaining: 17.1s\n",
      "100:\tlearn: 1.6007034\ttotal: 2.79s\tremaining: 11s\n",
      "200:\tlearn: 1.5338182\ttotal: 5.34s\tremaining: 7.95s\n",
      "300:\tlearn: 1.4980457\ttotal: 7.87s\tremaining: 5.21s\n",
      "400:\tlearn: 1.4694052\ttotal: 10.5s\tremaining: 2.6s\n",
      "499:\tlearn: 1.4451056\ttotal: 13.4s\tremaining: 0us\n",
      "Training_column: 18\n",
      "0:\tlearn: 2.7186909\ttotal: 29.2ms\tremaining: 14.6s\n",
      "100:\tlearn: 1.9867413\ttotal: 3.1s\tremaining: 12.2s\n",
      "200:\tlearn: 1.9390152\ttotal: 6.18s\tremaining: 9.19s\n",
      "300:\tlearn: 1.9050547\ttotal: 9.23s\tremaining: 6.1s\n",
      "400:\tlearn: 1.8758345\ttotal: 11.8s\tremaining: 2.92s\n",
      "499:\tlearn: 1.8488439\ttotal: 14.6s\tremaining: 0us\n",
      "Training_column: 19\n",
      "0:\tlearn: 2.0237538\ttotal: 27.8ms\tremaining: 11.1s\n",
      "100:\tlearn: 1.9271533\ttotal: 2.67s\tremaining: 7.91s\n",
      "200:\tlearn: 1.9018726\ttotal: 5.31s\tremaining: 5.26s\n",
      "300:\tlearn: 1.8844654\ttotal: 7.72s\tremaining: 2.54s\n",
      "399:\tlearn: 1.8671897\ttotal: 10.1s\tremaining: 0us\n",
      "Training_column: 20\n",
      "0:\tlearn: 0.9199203\ttotal: 26.7ms\tremaining: 10.7s\n",
      "100:\tlearn: 0.8402042\ttotal: 2.61s\tremaining: 7.73s\n",
      "200:\tlearn: 0.8281638\ttotal: 5.21s\tremaining: 5.16s\n",
      "300:\tlearn: 0.8208617\ttotal: 8.18s\tremaining: 2.69s\n",
      "399:\tlearn: 0.8145764\ttotal: 10.7s\tremaining: 0us\n",
      "Training_column: 21\n",
      "0:\tlearn: 2.1753181\ttotal: 41ms\tremaining: 20.4s\n",
      "100:\tlearn: 1.3728662\ttotal: 2.97s\tremaining: 11.7s\n",
      "200:\tlearn: 1.3264352\ttotal: 5.53s\tremaining: 8.22s\n",
      "300:\tlearn: 1.3023419\ttotal: 8.07s\tremaining: 5.33s\n",
      "400:\tlearn: 1.2834753\ttotal: 10.6s\tremaining: 2.61s\n",
      "499:\tlearn: 1.2670463\ttotal: 13.9s\tremaining: 0us\n",
      "Training_column: 22\n",
      "0:\tlearn: 1.0302637\ttotal: 27.2ms\tremaining: 8.12s\n",
      "100:\tlearn: 1.0192286\ttotal: 2.85s\tremaining: 5.63s\n",
      "200:\tlearn: 1.0140641\ttotal: 5.33s\tremaining: 2.63s\n",
      "299:\tlearn: 1.0095744\ttotal: 7.78s\tremaining: 0us\n",
      "Training_column: 23\n",
      "0:\tlearn: 1.2310067\ttotal: 28.8ms\tremaining: 11.5s\n",
      "100:\tlearn: 1.1290836\ttotal: 2.61s\tremaining: 7.73s\n",
      "200:\tlearn: 1.1147695\ttotal: 5.14s\tremaining: 5.09s\n",
      "300:\tlearn: 1.1047929\ttotal: 7.66s\tremaining: 2.52s\n",
      "399:\tlearn: 1.0965374\ttotal: 11.1s\tremaining: 0us\n",
      "Training_column: 24\n",
      "0:\tlearn: 4.4497661\ttotal: 34.6ms\tremaining: 17.2s\n",
      "100:\tlearn: 3.1424713\ttotal: 2.94s\tremaining: 11.6s\n",
      "200:\tlearn: 3.0488288\ttotal: 5.62s\tremaining: 8.36s\n",
      "300:\tlearn: 2.9975131\ttotal: 8.19s\tremaining: 5.42s\n",
      "400:\tlearn: 2.9549480\ttotal: 10.8s\tremaining: 2.66s\n",
      "499:\tlearn: 2.9154562\ttotal: 13.5s\tremaining: 0us\n",
      "Training_column: 25\n",
      "0:\tlearn: 0.9692372\ttotal: 27ms\tremaining: 8.08s\n",
      "100:\tlearn: 0.9349129\ttotal: 2.5s\tremaining: 4.92s\n",
      "200:\tlearn: 0.9281838\ttotal: 5.89s\tremaining: 2.9s\n",
      "299:\tlearn: 0.9245029\ttotal: 8.35s\tremaining: 0us\n",
      "Training_column: 26\n",
      "0:\tlearn: 0.9034324\ttotal: 25.7ms\tremaining: 7.7s\n",
      "100:\tlearn: 0.8941394\ttotal: 2.84s\tremaining: 5.59s\n",
      "200:\tlearn: 0.8901664\ttotal: 5.31s\tremaining: 2.62s\n",
      "299:\tlearn: 0.8867797\ttotal: 7.81s\tremaining: 0us\n",
      "Training_column: 27\n",
      "0:\tlearn: 0.8953543\ttotal: 27.5ms\tremaining: 8.21s\n",
      "100:\tlearn: 0.8721600\ttotal: 2.5s\tremaining: 4.92s\n",
      "200:\tlearn: 0.8666719\ttotal: 4.9s\tremaining: 2.42s\n",
      "299:\tlearn: 0.8628971\ttotal: 8.16s\tremaining: 0us\n",
      "Training_column: 28\n",
      "0:\tlearn: 0.8686476\ttotal: 29.1ms\tremaining: 11.6s\n",
      "100:\tlearn: 0.7693665\ttotal: 2.6s\tremaining: 7.69s\n",
      "200:\tlearn: 0.7617887\ttotal: 5.49s\tremaining: 5.44s\n",
      "300:\tlearn: 0.7562519\ttotal: 8.07s\tremaining: 2.65s\n",
      "399:\tlearn: 0.7511826\ttotal: 10.6s\tremaining: 0us\n",
      "Training_column: 29\n",
      "0:\tlearn: 0.7972452\ttotal: 26.6ms\tremaining: 10.6s\n",
      "100:\tlearn: 0.7456164\ttotal: 2.49s\tremaining: 7.38s\n",
      "200:\tlearn: 0.7397071\ttotal: 4.94s\tremaining: 4.89s\n",
      "300:\tlearn: 0.7351491\ttotal: 8.41s\tremaining: 2.77s\n",
      "399:\tlearn: 0.7305246\ttotal: 11.2s\tremaining: 0us\n",
      "Training_column: 30\n",
      "0:\tlearn: 0.8340698\ttotal: 27.7ms\tremaining: 8.28s\n",
      "100:\tlearn: 0.7952695\ttotal: 2.97s\tremaining: 5.85s\n",
      "200:\tlearn: 0.7898666\ttotal: 5.57s\tremaining: 2.74s\n",
      "299:\tlearn: 0.7860844\ttotal: 8.14s\tremaining: 0us\n",
      "Training_column: 31\n",
      "0:\tlearn: 0.8579145\ttotal: 29.5ms\tremaining: 8.82s\n",
      "100:\tlearn: 0.8184640\ttotal: 2.65s\tremaining: 5.23s\n",
      "200:\tlearn: 0.8129777\ttotal: 5.94s\tremaining: 2.93s\n",
      "299:\tlearn: 0.8092031\ttotal: 8.68s\tremaining: 0us\n",
      "Training_column: 32\n",
      "0:\tlearn: 0.7806137\ttotal: 27.5ms\tremaining: 8.22s\n",
      "100:\tlearn: 0.7573224\ttotal: 2.54s\tremaining: 5.01s\n",
      "200:\tlearn: 0.7535788\ttotal: 5.45s\tremaining: 2.69s\n",
      "299:\tlearn: 0.7503295\ttotal: 8.02s\tremaining: 0us\n",
      "Training_column: 33\n",
      "0:\tlearn: 0.7848020\ttotal: 26.3ms\tremaining: 7.87s\n",
      "100:\tlearn: 0.7658082\ttotal: 2.87s\tremaining: 5.66s\n",
      "200:\tlearn: 0.7619512\ttotal: 5.49s\tremaining: 2.71s\n",
      "299:\tlearn: 0.7587803\ttotal: 8.81s\tremaining: 0us\n",
      "Training_column: 34\n",
      "0:\tlearn: 0.9356923\ttotal: 27.2ms\tremaining: 8.13s\n",
      "100:\tlearn: 0.9181775\ttotal: 2.59s\tremaining: 5.1s\n",
      "200:\tlearn: 0.9135384\ttotal: 5.12s\tremaining: 2.52s\n",
      "299:\tlearn: 0.9098100\ttotal: 7.89s\tremaining: 0us\n",
      "Training_column: 35\n",
      "0:\tlearn: 0.8842525\ttotal: 25.1ms\tremaining: 7.49s\n",
      "100:\tlearn: 0.8497915\ttotal: 2.42s\tremaining: 4.78s\n",
      "200:\tlearn: 0.8425180\ttotal: 5.18s\tremaining: 2.55s\n",
      "299:\tlearn: 0.8386518\ttotal: 7.64s\tremaining: 0us\n",
      "Training_column: 36\n",
      "0:\tlearn: 1.0429273\ttotal: 37.3ms\tremaining: 14.9s\n",
      "100:\tlearn: 0.9322772\ttotal: 2.76s\tremaining: 8.17s\n",
      "200:\tlearn: 0.9236097\ttotal: 5.4s\tremaining: 5.35s\n",
      "300:\tlearn: 0.9171599\ttotal: 8.01s\tremaining: 2.63s\n",
      "399:\tlearn: 0.9111811\ttotal: 10.8s\tremaining: 0us\n",
      "Training_column: 37\n",
      "0:\tlearn: 6.0467690\ttotal: 29.1ms\tremaining: 14.5s\n",
      "100:\tlearn: 4.2006597\ttotal: 2.7s\tremaining: 10.7s\n",
      "200:\tlearn: 4.0676622\ttotal: 5.59s\tremaining: 8.32s\n",
      "300:\tlearn: 3.9947278\ttotal: 8.66s\tremaining: 5.72s\n",
      "400:\tlearn: 3.9385661\ttotal: 11.4s\tremaining: 2.81s\n",
      "499:\tlearn: 3.8866882\ttotal: 14s\tremaining: 0us\n",
      "Training_column: 38\n",
      "0:\tlearn: 0.9549333\ttotal: 28.2ms\tremaining: 11.3s\n",
      "100:\tlearn: 0.8574820\ttotal: 2.65s\tremaining: 7.84s\n",
      "200:\tlearn: 0.8500365\ttotal: 5.38s\tremaining: 5.33s\n",
      "300:\tlearn: 0.8443501\ttotal: 8.12s\tremaining: 2.67s\n",
      "399:\tlearn: 0.8389696\ttotal: 10.7s\tremaining: 0us\n",
      "Training_column: 39\n",
      "0:\tlearn: 1.2170162\ttotal: 28.6ms\tremaining: 8.55s\n",
      "100:\tlearn: 1.1972137\ttotal: 3.36s\tremaining: 6.62s\n",
      "200:\tlearn: 1.1904827\ttotal: 5.84s\tremaining: 2.88s\n",
      "299:\tlearn: 1.1852733\ttotal: 8.27s\tremaining: 0us\n",
      "Training_column: 40\n",
      "0:\tlearn: 0.9436458\ttotal: 27.6ms\tremaining: 8.26s\n",
      "100:\tlearn: 0.9164473\ttotal: 2.46s\tremaining: 4.84s\n",
      "200:\tlearn: 0.9057392\ttotal: 4.82s\tremaining: 2.37s\n",
      "299:\tlearn: 0.8980666\ttotal: 7.55s\tremaining: 0us\n",
      "Training_column: 41\n",
      "0:\tlearn: 1.0298502\ttotal: 26.7ms\tremaining: 10.7s\n",
      "100:\tlearn: 0.9783694\ttotal: 2.62s\tremaining: 7.77s\n",
      "200:\tlearn: 0.9709168\ttotal: 6.08s\tremaining: 6.02s\n",
      "300:\tlearn: 0.9651619\ttotal: 8.54s\tremaining: 2.81s\n",
      "399:\tlearn: 0.9590195\ttotal: 11s\tremaining: 0us\n",
      "Training_column: 42\n",
      "0:\tlearn: 0.9640819\ttotal: 25.3ms\tremaining: 10.1s\n",
      "100:\tlearn: 0.9031310\ttotal: 2.53s\tremaining: 7.5s\n",
      "200:\tlearn: 0.8957656\ttotal: 5.07s\tremaining: 5.02s\n",
      "300:\tlearn: 0.8902613\ttotal: 7.88s\tremaining: 2.59s\n",
      "399:\tlearn: 0.8848727\ttotal: 10.4s\tremaining: 0us\n",
      "Training_column: 43\n",
      "0:\tlearn: 4.8105237\ttotal: 28.7ms\tremaining: 14.3s\n",
      "100:\tlearn: 2.4284607\ttotal: 3.54s\tremaining: 14s\n",
      "200:\tlearn: 2.3165170\ttotal: 6.01s\tremaining: 8.94s\n",
      "300:\tlearn: 2.2493406\ttotal: 8.48s\tremaining: 5.61s\n",
      "400:\tlearn: 2.1958745\ttotal: 11s\tremaining: 2.71s\n",
      "499:\tlearn: 2.1476826\ttotal: 13.8s\tremaining: 0us\n",
      "Training_column: 44\n",
      "0:\tlearn: 0.8960224\ttotal: 24.5ms\tremaining: 9.78s\n",
      "100:\tlearn: 0.8462351\ttotal: 2.83s\tremaining: 8.37s\n",
      "200:\tlearn: 0.8386308\ttotal: 5.26s\tremaining: 5.21s\n",
      "300:\tlearn: 0.8333617\ttotal: 7.74s\tremaining: 2.55s\n",
      "399:\tlearn: 0.8287247\ttotal: 10.2s\tremaining: 0us\n",
      "Training_column: 45\n",
      "0:\tlearn: 0.8454352\ttotal: 26.7ms\tremaining: 7.98s\n",
      "100:\tlearn: 0.8167942\ttotal: 2.52s\tremaining: 4.97s\n",
      "200:\tlearn: 0.8124156\ttotal: 5.11s\tremaining: 2.51s\n",
      "299:\tlearn: 0.8089030\ttotal: 8.03s\tremaining: 0us\n",
      "Training_column: 46\n",
      "0:\tlearn: 1.4141115\ttotal: 29.1ms\tremaining: 11.6s\n",
      "100:\tlearn: 1.2610907\ttotal: 2.73s\tremaining: 8.07s\n",
      "200:\tlearn: 1.2460330\ttotal: 5.38s\tremaining: 5.32s\n",
      "300:\tlearn: 1.2363301\ttotal: 7.86s\tremaining: 2.58s\n",
      "399:\tlearn: 1.2280425\ttotal: 10.3s\tremaining: 0us\n",
      "Training_column: 47\n",
      "0:\tlearn: 1.1434921\ttotal: 25.5ms\tremaining: 10.2s\n",
      "100:\tlearn: 1.0438510\ttotal: 2.41s\tremaining: 7.14s\n",
      "200:\tlearn: 1.0265446\ttotal: 4.8s\tremaining: 4.75s\n",
      "300:\tlearn: 1.0165451\ttotal: 7.43s\tremaining: 2.44s\n",
      "399:\tlearn: 1.0084098\ttotal: 9.93s\tremaining: 0us\n",
      "Training_column: 48\n",
      "0:\tlearn: 4.8476795\ttotal: 28.4ms\tremaining: 14.2s\n",
      "100:\tlearn: 2.8902682\ttotal: 2.98s\tremaining: 11.8s\n",
      "200:\tlearn: 2.7803384\ttotal: 5.6s\tremaining: 8.33s\n",
      "300:\tlearn: 2.7177661\ttotal: 8.22s\tremaining: 5.43s\n",
      "400:\tlearn: 2.6720487\ttotal: 11.7s\tremaining: 2.89s\n",
      "499:\tlearn: 2.6318210\ttotal: 14.5s\tremaining: 0us\n",
      "Training_column: 49\n",
      "0:\tlearn: 0.8460452\ttotal: 28.6ms\tremaining: 8.56s\n",
      "100:\tlearn: 0.8288013\ttotal: 2.91s\tremaining: 5.73s\n",
      "200:\tlearn: 0.8245489\ttotal: 5.55s\tremaining: 2.73s\n",
      "299:\tlearn: 0.8210306\ttotal: 8.09s\tremaining: 0us\n",
      "Training_column: 50\n",
      "0:\tlearn: 0.8775568\ttotal: 27ms\tremaining: 10.8s\n",
      "100:\tlearn: 0.7930824\ttotal: 2.75s\tremaining: 8.13s\n",
      "200:\tlearn: 0.7853644\ttotal: 5.79s\tremaining: 5.73s\n",
      "300:\tlearn: 0.7797766\ttotal: 8.69s\tremaining: 2.86s\n",
      "399:\tlearn: 0.7746207\ttotal: 11.6s\tremaining: 0us\n",
      "Training_column: 51\n",
      "0:\tlearn: 0.9176258\ttotal: 27.8ms\tremaining: 8.3s\n",
      "100:\tlearn: 0.8795400\ttotal: 2.96s\tremaining: 5.83s\n",
      "200:\tlearn: 0.8730439\ttotal: 5.58s\tremaining: 2.75s\n",
      "299:\tlearn: 0.8685723\ttotal: 8.11s\tremaining: 0us\n",
      "Training_column: 52\n",
      "0:\tlearn: 1.9029184\ttotal: 30ms\tremaining: 12s\n",
      "100:\tlearn: 1.5002499\ttotal: 2.77s\tremaining: 8.19s\n",
      "200:\tlearn: 1.4674284\ttotal: 6.12s\tremaining: 6.06s\n",
      "300:\tlearn: 1.4479013\ttotal: 8.71s\tremaining: 2.86s\n",
      "399:\tlearn: 1.4317350\ttotal: 11.6s\tremaining: 0us\n",
      "Training_column: 53\n",
      "0:\tlearn: 1.0215740\ttotal: 25.6ms\tremaining: 7.66s\n",
      "100:\tlearn: 0.9871483\ttotal: 2.73s\tremaining: 5.38s\n",
      "200:\tlearn: 0.9750440\ttotal: 5.11s\tremaining: 2.52s\n",
      "299:\tlearn: 0.9693456\ttotal: 7.5s\tremaining: 0us\n",
      "Training_column: 54\n",
      "0:\tlearn: 1.3379327\ttotal: 28.2ms\tremaining: 11.2s\n",
      "100:\tlearn: 1.1155789\ttotal: 2.73s\tremaining: 8.1s\n",
      "200:\tlearn: 1.0988225\ttotal: 6.09s\tremaining: 6.03s\n",
      "300:\tlearn: 1.0895501\ttotal: 8.64s\tremaining: 2.84s\n",
      "399:\tlearn: 1.0815958\ttotal: 11.5s\tremaining: 0us\n",
      "Training_column: 55\n",
      "0:\tlearn: 1.2037677\ttotal: 30.9ms\tremaining: 15.4s\n",
      "100:\tlearn: 0.9305042\ttotal: 2.84s\tremaining: 11.2s\n",
      "200:\tlearn: 0.9063471\ttotal: 5.57s\tremaining: 8.29s\n",
      "300:\tlearn: 0.8919818\ttotal: 8.1s\tremaining: 5.35s\n",
      "400:\tlearn: 0.8795303\ttotal: 10.6s\tremaining: 2.63s\n",
      "499:\tlearn: 0.8688476\ttotal: 13.2s\tremaining: 0us\n",
      "Training_column: 56\n",
      "0:\tlearn: 0.8943767\ttotal: 27.5ms\tremaining: 8.23s\n",
      "100:\tlearn: 0.8550895\ttotal: 2.64s\tremaining: 5.2s\n",
      "200:\tlearn: 0.8493598\ttotal: 5.53s\tremaining: 2.72s\n",
      "299:\tlearn: 0.8453141\ttotal: 8.1s\tremaining: 0us\n",
      "Training_column: 57\n",
      "0:\tlearn: 2.6920662\ttotal: 79.2ms\tremaining: 39.5s\n",
      "100:\tlearn: 1.7521613\ttotal: 3.01s\tremaining: 11.9s\n",
      "200:\tlearn: 1.6826622\ttotal: 5.63s\tremaining: 8.38s\n",
      "300:\tlearn: 1.6451771\ttotal: 8.22s\tremaining: 5.44s\n",
      "400:\tlearn: 1.6159550\ttotal: 10.9s\tremaining: 2.69s\n",
      "499:\tlearn: 1.5906895\ttotal: 14.5s\tremaining: 0us\n",
      "Training_column: 58\n",
      "0:\tlearn: 1.1919272\ttotal: 26.3ms\tremaining: 10.5s\n",
      "100:\tlearn: 1.0423818\ttotal: 2.67s\tremaining: 7.91s\n",
      "200:\tlearn: 1.0129086\ttotal: 5.02s\tremaining: 4.97s\n",
      "300:\tlearn: 1.0023728\ttotal: 7.4s\tremaining: 2.43s\n",
      "399:\tlearn: 0.9957536\ttotal: 9.79s\tremaining: 0us\n",
      "Training_column: 59\n",
      "0:\tlearn: 1.0481771\ttotal: 24.5ms\tremaining: 7.31s\n",
      "100:\tlearn: 1.0228784\ttotal: 2.44s\tremaining: 4.8s\n",
      "200:\tlearn: 1.0166304\ttotal: 4.9s\tremaining: 2.41s\n",
      "299:\tlearn: 1.0123276\ttotal: 8.3s\tremaining: 0us\n",
      "Training_column: 60\n",
      "0:\tlearn: 1.1580563\ttotal: 25.1ms\tremaining: 10s\n",
      "100:\tlearn: 1.0252601\ttotal: 2.62s\tremaining: 7.75s\n",
      "200:\tlearn: 1.0123019\ttotal: 5.29s\tremaining: 5.23s\n",
      "300:\tlearn: 1.0042454\ttotal: 7.79s\tremaining: 2.56s\n",
      "399:\tlearn: 0.9973054\ttotal: 10.3s\tremaining: 0us\n",
      "Training_column: 61\n",
      "0:\tlearn: 0.8295972\ttotal: 28.2ms\tremaining: 11.3s\n",
      "100:\tlearn: 0.7046121\ttotal: 2.6s\tremaining: 7.7s\n",
      "200:\tlearn: 0.6941744\ttotal: 5.15s\tremaining: 5.1s\n",
      "300:\tlearn: 0.6887895\ttotal: 8.62s\tremaining: 2.84s\n",
      "399:\tlearn: 0.6842560\ttotal: 11.1s\tremaining: 0us\n",
      "Training_column: 62\n",
      "0:\tlearn: 0.9142149\ttotal: 27.5ms\tremaining: 11s\n",
      "100:\tlearn: 0.8147900\ttotal: 2.96s\tremaining: 8.75s\n",
      "200:\tlearn: 0.8069458\ttotal: 5.56s\tremaining: 5.5s\n",
      "300:\tlearn: 0.8014374\ttotal: 8.14s\tremaining: 2.68s\n",
      "399:\tlearn: 0.7961534\ttotal: 10.7s\tremaining: 0us\n",
      "Training_column: 63\n",
      "0:\tlearn: 0.8413425\ttotal: 27.9ms\tremaining: 11.1s\n",
      "100:\tlearn: 0.7873188\ttotal: 2.71s\tremaining: 8.04s\n",
      "200:\tlearn: 0.7812224\ttotal: 6.42s\tremaining: 6.36s\n",
      "300:\tlearn: 0.7761221\ttotal: 9s\tremaining: 2.96s\n",
      "399:\tlearn: 0.7710962\ttotal: 11.5s\tremaining: 0us\n",
      "Training_column: 64\n",
      "0:\tlearn: 1.3725989\ttotal: 27.1ms\tremaining: 10.8s\n",
      "100:\tlearn: 1.1477973\ttotal: 2.44s\tremaining: 7.22s\n",
      "200:\tlearn: 1.1148523\ttotal: 4.82s\tremaining: 4.77s\n",
      "300:\tlearn: 1.0945642\ttotal: 7.22s\tremaining: 2.38s\n",
      "399:\tlearn: 1.0776106\ttotal: 10s\tremaining: 0us\n",
      "Training_column: 65\n",
      "0:\tlearn: 1.1198455\ttotal: 26.1ms\tremaining: 10.4s\n",
      "100:\tlearn: 1.0264278\ttotal: 3.45s\tremaining: 10.2s\n",
      "200:\tlearn: 1.0147502\ttotal: 5.94s\tremaining: 5.88s\n",
      "300:\tlearn: 1.0077436\ttotal: 8.38s\tremaining: 2.75s\n",
      "399:\tlearn: 1.0017021\ttotal: 10.8s\tremaining: 0us\n",
      "Training_column: 66\n",
      "0:\tlearn: 1.4233721\ttotal: 28.2ms\tremaining: 14.1s\n",
      "100:\tlearn: 1.2042124\ttotal: 2.61s\tremaining: 10.3s\n",
      "200:\tlearn: 1.1887862\ttotal: 5.12s\tremaining: 7.61s\n",
      "300:\tlearn: 1.1750467\ttotal: 7.82s\tremaining: 5.17s\n",
      "400:\tlearn: 1.1612462\ttotal: 10.5s\tremaining: 2.59s\n",
      "499:\tlearn: 1.1475886\ttotal: 13.1s\tremaining: 0us\n",
      "Training_column: 67\n",
      "0:\tlearn: 1.0520048\ttotal: 30.7ms\tremaining: 12.2s\n",
      "100:\tlearn: 0.9602206\ttotal: 2.76s\tremaining: 8.16s\n",
      "200:\tlearn: 0.9502657\ttotal: 5.45s\tremaining: 5.39s\n",
      "300:\tlearn: 0.9430103\ttotal: 8.11s\tremaining: 2.67s\n",
      "399:\tlearn: 0.9364388\ttotal: 11.1s\tremaining: 0us\n",
      "Training_column: 68\n",
      "0:\tlearn: 1.5847888\ttotal: 28.1ms\tremaining: 14s\n",
      "100:\tlearn: 1.1658123\ttotal: 2.89s\tremaining: 11.4s\n",
      "200:\tlearn: 1.1433738\ttotal: 5.5s\tremaining: 8.18s\n",
      "300:\tlearn: 1.1290168\ttotal: 8.04s\tremaining: 5.32s\n",
      "400:\tlearn: 1.1160038\ttotal: 11.3s\tremaining: 2.79s\n",
      "499:\tlearn: 1.1037909\ttotal: 14.2s\tremaining: 0us\n",
      "Training_column: 69\n",
      "0:\tlearn: 1.1417653\ttotal: 25.2ms\tremaining: 10s\n",
      "100:\tlearn: 0.9742742\ttotal: 2.44s\tremaining: 7.23s\n",
      "200:\tlearn: 0.9505845\ttotal: 5.12s\tremaining: 5.07s\n",
      "300:\tlearn: 0.9384775\ttotal: 7.53s\tremaining: 2.48s\n",
      "399:\tlearn: 0.9286662\ttotal: 9.93s\tremaining: 0us\n",
      "Training_column: 70\n",
      "0:\tlearn: 0.7263595\ttotal: 29.1ms\tremaining: 11.6s\n",
      "100:\tlearn: 0.6756538\ttotal: 2.68s\tremaining: 7.93s\n",
      "200:\tlearn: 0.6696967\ttotal: 5.92s\tremaining: 5.86s\n",
      "300:\tlearn: 0.6653235\ttotal: 8.43s\tremaining: 2.77s\n",
      "399:\tlearn: 0.6611908\ttotal: 11.3s\tremaining: 0us\n",
      "Training_column: 71\n",
      "0:\tlearn: 0.7347408\ttotal: 27.1ms\tremaining: 10.8s\n",
      "100:\tlearn: 0.6890281\ttotal: 2.84s\tremaining: 8.41s\n",
      "200:\tlearn: 0.6831334\ttotal: 5.27s\tremaining: 5.22s\n",
      "300:\tlearn: 0.6788551\ttotal: 7.77s\tremaining: 2.56s\n",
      "399:\tlearn: 0.6749995\ttotal: 10.2s\tremaining: 0us\n",
      "Training_column: 72\n",
      "0:\tlearn: 1.0676159\ttotal: 42.5ms\tremaining: 12.7s\n",
      "100:\tlearn: 1.0328506\ttotal: 2.68s\tremaining: 5.28s\n",
      "200:\tlearn: 1.0247316\ttotal: 5.12s\tremaining: 2.52s\n",
      "299:\tlearn: 1.0203621\ttotal: 7.79s\tremaining: 0us\n",
      "Training_column: 73\n",
      "0:\tlearn: 5.4300570\ttotal: 32.6ms\tremaining: 16.3s\n",
      "100:\tlearn: 3.4099853\ttotal: 2.84s\tremaining: 11.2s\n",
      "200:\tlearn: 3.2514502\ttotal: 5.61s\tremaining: 8.35s\n",
      "300:\tlearn: 3.1731988\ttotal: 8.24s\tremaining: 5.45s\n",
      "400:\tlearn: 3.1171159\ttotal: 10.8s\tremaining: 2.67s\n",
      "499:\tlearn: 3.0701965\ttotal: 13.4s\tremaining: 0us\n",
      "Training_column: 74\n",
      "0:\tlearn: 0.8498352\ttotal: 27.2ms\tremaining: 8.13s\n",
      "100:\tlearn: 0.8231473\ttotal: 2.61s\tremaining: 5.14s\n",
      "200:\tlearn: 0.8177956\ttotal: 5.53s\tremaining: 2.73s\n",
      "299:\tlearn: 0.8141468\ttotal: 8.04s\tremaining: 0us\n",
      "Training_column: 75\n",
      "0:\tlearn: 4.2338464\ttotal: 29.2ms\tremaining: 14.6s\n",
      "100:\tlearn: 2.7953603\ttotal: 3.07s\tremaining: 12.1s\n",
      "200:\tlearn: 2.6764930\ttotal: 5.79s\tremaining: 8.61s\n",
      "300:\tlearn: 2.6154487\ttotal: 8.44s\tremaining: 5.58s\n",
      "400:\tlearn: 2.5719602\ttotal: 11s\tremaining: 2.73s\n",
      "499:\tlearn: 2.5339033\ttotal: 14.5s\tremaining: 0us\n",
      "Training_column: 76\n",
      "0:\tlearn: 0.9879537\ttotal: 28.2ms\tremaining: 8.43s\n",
      "100:\tlearn: 0.9607310\ttotal: 3.05s\tremaining: 6s\n",
      "200:\tlearn: 0.9551652\ttotal: 5.63s\tremaining: 2.77s\n",
      "299:\tlearn: 0.9510111\ttotal: 8.18s\tremaining: 0us\n",
      "Training_column: 77\n",
      "0:\tlearn: 2.3184422\ttotal: 27.6ms\tremaining: 11s\n",
      "100:\tlearn: 1.9926929\ttotal: 2.63s\tremaining: 7.77s\n",
      "200:\tlearn: 1.9352339\ttotal: 5.23s\tremaining: 5.18s\n",
      "300:\tlearn: 1.9034643\ttotal: 7.81s\tremaining: 2.57s\n",
      "399:\tlearn: 1.8802632\ttotal: 11.3s\tremaining: 0us\n",
      "Training_column: 78\n",
      "0:\tlearn: 1.0891073\ttotal: 26.2ms\tremaining: 7.83s\n",
      "100:\tlearn: 1.0533448\ttotal: 2.74s\tremaining: 5.4s\n",
      "200:\tlearn: 1.0446749\ttotal: 5.15s\tremaining: 2.53s\n",
      "299:\tlearn: 1.0403913\ttotal: 7.52s\tremaining: 0us\n",
      "Training_column: 79\n",
      "0:\tlearn: 1.0965266\ttotal: 27.8ms\tremaining: 11.1s\n",
      "100:\tlearn: 0.9762170\ttotal: 2.52s\tremaining: 7.47s\n",
      "200:\tlearn: 0.9644809\ttotal: 5s\tremaining: 4.95s\n",
      "300:\tlearn: 0.9578474\ttotal: 7.49s\tremaining: 2.46s\n",
      "399:\tlearn: 0.9515712\ttotal: 11.1s\tremaining: 0us\n",
      "Training_column: 80\n",
      "0:\tlearn: 3.1229921\ttotal: 27.3ms\tremaining: 13.6s\n",
      "100:\tlearn: 2.3485604\ttotal: 2.94s\tremaining: 11.6s\n",
      "200:\tlearn: 2.2748205\ttotal: 5.45s\tremaining: 8.1s\n",
      "300:\tlearn: 2.2285818\ttotal: 7.93s\tremaining: 5.24s\n",
      "400:\tlearn: 2.1914302\ttotal: 10.4s\tremaining: 2.57s\n",
      "499:\tlearn: 2.1593244\ttotal: 13s\tremaining: 0us\n",
      "Training_column: 81\n",
      "0:\tlearn: 1.0996198\ttotal: 26.6ms\tremaining: 10.6s\n",
      "100:\tlearn: 0.9545826\ttotal: 2.52s\tremaining: 7.47s\n",
      "200:\tlearn: 0.9332261\ttotal: 5.81s\tremaining: 5.75s\n",
      "300:\tlearn: 0.9188379\ttotal: 8.21s\tremaining: 2.7s\n",
      "399:\tlearn: 0.9080852\ttotal: 10.6s\tremaining: 0us\n",
      "Training_column: 82\n",
      "0:\tlearn: 0.9741441\ttotal: 29.8ms\tremaining: 11.9s\n",
      "100:\tlearn: 0.8481877\ttotal: 2.69s\tremaining: 7.98s\n",
      "200:\tlearn: 0.8386755\ttotal: 5.25s\tremaining: 5.2s\n",
      "300:\tlearn: 0.8322932\ttotal: 7.82s\tremaining: 2.57s\n",
      "399:\tlearn: 0.8267621\ttotal: 10.6s\tremaining: 0us\n",
      "Training_column: 83\n",
      "0:\tlearn: 0.9635481\ttotal: 35.6ms\tremaining: 14.2s\n",
      "100:\tlearn: 0.9105917\ttotal: 3.38s\tremaining: 10s\n",
      "200:\tlearn: 0.9036246\ttotal: 5.94s\tremaining: 5.88s\n",
      "300:\tlearn: 0.8977149\ttotal: 8.57s\tremaining: 2.82s\n",
      "399:\tlearn: 0.8920614\ttotal: 11.1s\tremaining: 0us\n",
      "Training_column: 84\n",
      "0:\tlearn: 0.8049414\ttotal: 28.8ms\tremaining: 8.62s\n",
      "100:\tlearn: 0.7890437\ttotal: 2.64s\tremaining: 5.2s\n",
      "200:\tlearn: 0.7849070\ttotal: 5.19s\tremaining: 2.56s\n",
      "299:\tlearn: 0.7816832\ttotal: 8.02s\tremaining: 0us\n",
      "Training_column: 85\n",
      "0:\tlearn: 0.7911618\ttotal: 30.1ms\tremaining: 9.01s\n",
      "100:\tlearn: 0.7709033\ttotal: 2.99s\tremaining: 5.9s\n",
      "200:\tlearn: 0.7662212\ttotal: 5.57s\tremaining: 2.74s\n",
      "299:\tlearn: 0.7627155\ttotal: 8.1s\tremaining: 0us\n",
      "Training_column: 86\n",
      "0:\tlearn: 0.8890813\ttotal: 27.3ms\tremaining: 10.9s\n",
      "100:\tlearn: 0.8184975\ttotal: 2.58s\tremaining: 7.65s\n",
      "200:\tlearn: 0.8115908\ttotal: 5.12s\tremaining: 5.07s\n",
      "300:\tlearn: 0.8059102\ttotal: 7.71s\tremaining: 2.53s\n",
      "399:\tlearn: 0.8004489\ttotal: 10.6s\tremaining: 0us\n",
      "Training_column: 87\n",
      "0:\tlearn: 0.8705162\ttotal: 26.9ms\tremaining: 8.04s\n",
      "100:\tlearn: 0.8523912\ttotal: 2.84s\tremaining: 5.59s\n",
      "200:\tlearn: 0.8485398\ttotal: 5.5s\tremaining: 2.71s\n",
      "299:\tlearn: 0.8452628\ttotal: 8.02s\tremaining: 0us\n",
      "Training_column: 88\n",
      "0:\tlearn: 1.7154906\ttotal: 27.9ms\tremaining: 13.9s\n",
      "100:\tlearn: 1.1573896\ttotal: 2.56s\tremaining: 10.1s\n",
      "200:\tlearn: 1.1317935\ttotal: 5.08s\tremaining: 7.55s\n",
      "300:\tlearn: 1.1140782\ttotal: 7.57s\tremaining: 5s\n",
      "400:\tlearn: 1.0987780\ttotal: 10.4s\tremaining: 2.58s\n",
      "499:\tlearn: 1.0849633\ttotal: 12.9s\tremaining: 0us\n",
      "Training_column: 89\n",
      "0:\tlearn: 5.1750031\ttotal: 27.7ms\tremaining: 13.8s\n",
      "100:\tlearn: 2.1373090\ttotal: 2.86s\tremaining: 11.3s\n",
      "200:\tlearn: 2.0336736\ttotal: 5.36s\tremaining: 7.98s\n",
      "300:\tlearn: 1.9689539\ttotal: 7.81s\tremaining: 5.16s\n",
      "400:\tlearn: 1.9178060\ttotal: 10.3s\tremaining: 2.54s\n",
      "499:\tlearn: 1.8737615\ttotal: 13.1s\tremaining: 0us\n",
      "Training_column: 90\n",
      "0:\tlearn: 1.0429808\ttotal: 28.7ms\tremaining: 11.4s\n",
      "100:\tlearn: 0.9097439\ttotal: 2.88s\tremaining: 8.54s\n",
      "200:\tlearn: 0.9002857\ttotal: 5.63s\tremaining: 5.58s\n",
      "300:\tlearn: 0.8939929\ttotal: 8.99s\tremaining: 2.96s\n",
      "399:\tlearn: 0.8880462\ttotal: 11.5s\tremaining: 0us\n",
      "Training_column: 91\n",
      "0:\tlearn: 0.7804915\ttotal: 29.5ms\tremaining: 8.83s\n",
      "100:\tlearn: 0.7505141\ttotal: 2.62s\tremaining: 5.16s\n",
      "200:\tlearn: 0.7461396\ttotal: 5.17s\tremaining: 2.54s\n",
      "299:\tlearn: 0.7429241\ttotal: 8.07s\tremaining: 0us\n",
      "Training_column: 92\n",
      "0:\tlearn: 0.7538804\ttotal: 27.8ms\tremaining: 11.1s\n",
      "100:\tlearn: 0.6816131\ttotal: 2.79s\tremaining: 8.27s\n",
      "200:\tlearn: 0.6752280\ttotal: 6.09s\tremaining: 6.03s\n",
      "300:\tlearn: 0.6707192\ttotal: 8.85s\tremaining: 2.91s\n",
      "399:\tlearn: 0.6664979\ttotal: 11.4s\tremaining: 0us\n",
      "Training_column: 93\n",
      "0:\tlearn: 0.9857803\ttotal: 27.7ms\tremaining: 11.1s\n",
      "100:\tlearn: 0.8808025\ttotal: 2.7s\tremaining: 7.98s\n",
      "200:\tlearn: 0.8722437\ttotal: 5.31s\tremaining: 5.25s\n",
      "300:\tlearn: 0.8660306\ttotal: 8.23s\tremaining: 2.71s\n",
      "399:\tlearn: 0.8602671\ttotal: 10.8s\tremaining: 0us\n",
      "Training_column: 94\n",
      "0:\tlearn: 2.2290307\ttotal: 38.5ms\tremaining: 19.2s\n",
      "100:\tlearn: 1.3985857\ttotal: 3.63s\tremaining: 14.3s\n",
      "200:\tlearn: 1.3594936\ttotal: 6.29s\tremaining: 9.35s\n",
      "300:\tlearn: 1.3343962\ttotal: 8.86s\tremaining: 5.86s\n",
      "400:\tlearn: 1.3139757\ttotal: 11.8s\tremaining: 2.91s\n",
      "499:\tlearn: 1.2960484\ttotal: 14.3s\tremaining: 0us\n",
      "Training_column: 95\n",
      "0:\tlearn: 1.7783108\ttotal: 25.4ms\tremaining: 7.59s\n",
      "100:\tlearn: 1.6949992\ttotal: 2.73s\tremaining: 5.39s\n",
      "200:\tlearn: 1.6646420\ttotal: 5.12s\tremaining: 2.52s\n",
      "299:\tlearn: 1.6464229\ttotal: 7.48s\tremaining: 0us\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:22: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training_column: 96\n",
      "0:\tlearn: 1.0488591\ttotal: 26.2ms\tremaining: 10.5s\n",
      "100:\tlearn: 0.9616378\ttotal: 2.5s\tremaining: 7.4s\n",
      "200:\tlearn: 0.9527839\ttotal: 4.93s\tremaining: 4.88s\n",
      "300:\tlearn: 0.9465803\ttotal: 7.47s\tremaining: 2.46s\n",
      "399:\tlearn: 0.9406954\ttotal: 10.1s\tremaining: 0us\n",
      "Training_column: 97\n",
      "0:\tlearn: 2.6639259\ttotal: 30.5ms\tremaining: 15.2s\n",
      "100:\tlearn: 1.7158407\ttotal: 2.99s\tremaining: 11.8s\n",
      "200:\tlearn: 1.6561376\ttotal: 5.58s\tremaining: 8.3s\n",
      "300:\tlearn: 1.6281572\ttotal: 8.11s\tremaining: 5.36s\n",
      "400:\tlearn: 1.6067617\ttotal: 10.6s\tremaining: 2.61s\n",
      "499:\tlearn: 1.5870562\ttotal: 14s\tremaining: 0us\n",
      "Training_column: 98\n",
      "0:\tlearn: 0.8464169\ttotal: 27.4ms\tremaining: 8.19s\n",
      "100:\tlearn: 0.8191111\ttotal: 2.6s\tremaining: 5.12s\n",
      "200:\tlearn: 0.8143471\ttotal: 5.58s\tremaining: 2.75s\n",
      "299:\tlearn: 0.8106361\ttotal: 8.16s\tremaining: 0us\n",
      "Training_column: 99\n",
      "0:\tlearn: 4.0724575\ttotal: 35.7ms\tremaining: 17.8s\n",
      "100:\tlearn: 2.4278199\ttotal: 2.75s\tremaining: 10.9s\n",
      "200:\tlearn: 2.3398703\ttotal: 5.36s\tremaining: 7.97s\n",
      "300:\tlearn: 2.2901525\ttotal: 7.92s\tremaining: 5.24s\n",
      "400:\tlearn: 2.2518166\ttotal: 11.6s\tremaining: 2.87s\n",
      "499:\tlearn: 2.2192978\ttotal: 14.2s\tremaining: 0us\n",
      "Training_column: 100\n",
      "0:\tlearn: 2.1825563\ttotal: 30.5ms\tremaining: 15.2s\n",
      "100:\tlearn: 1.4748152\ttotal: 3.04s\tremaining: 12s\n",
      "200:\tlearn: 1.4248946\ttotal: 5.63s\tremaining: 8.38s\n",
      "300:\tlearn: 1.3943877\ttotal: 8.24s\tremaining: 5.45s\n",
      "400:\tlearn: 1.3708605\ttotal: 10.9s\tremaining: 2.68s\n",
      "499:\tlearn: 1.3519044\ttotal: 13.6s\tremaining: 0us\n",
      "Training_column: 101\n",
      "0:\tlearn: 1.0922172\ttotal: 60.6ms\tremaining: 18.1s\n",
      "100:\tlearn: 1.0469247\ttotal: 2.84s\tremaining: 5.6s\n",
      "200:\tlearn: 1.0334147\ttotal: 5.31s\tremaining: 2.62s\n",
      "299:\tlearn: 1.0268339\ttotal: 7.68s\tremaining: 0us\n",
      "Training_column: 102\n",
      "0:\tlearn: 3.5480410\ttotal: 29ms\tremaining: 14.5s\n",
      "100:\tlearn: 2.9637260\ttotal: 2.62s\tremaining: 10.4s\n",
      "200:\tlearn: 2.8976309\ttotal: 5.18s\tremaining: 7.71s\n",
      "300:\tlearn: 2.8517071\ttotal: 7.72s\tremaining: 5.1s\n",
      "400:\tlearn: 2.8125348\ttotal: 10.6s\tremaining: 2.62s\n",
      "499:\tlearn: 2.7763226\ttotal: 13.2s\tremaining: 0us\n",
      "Training_column: 103\n",
      "0:\tlearn: 1.0681383\ttotal: 30.6ms\tremaining: 12.2s\n",
      "100:\tlearn: 0.9820457\ttotal: 2.71s\tremaining: 8.03s\n",
      "200:\tlearn: 0.9731327\ttotal: 5.31s\tremaining: 5.26s\n",
      "300:\tlearn: 0.9666261\ttotal: 7.86s\tremaining: 2.59s\n",
      "399:\tlearn: 0.9600808\ttotal: 10.6s\tremaining: 0us\n",
      "Training_column: 104\n",
      "0:\tlearn: 2.9145794\ttotal: 28.5ms\tremaining: 14.2s\n",
      "100:\tlearn: 1.9357613\ttotal: 2.67s\tremaining: 10.6s\n",
      "200:\tlearn: 1.8665796\ttotal: 5.55s\tremaining: 8.26s\n",
      "300:\tlearn: 1.8229536\ttotal: 8.16s\tremaining: 5.39s\n",
      "400:\tlearn: 1.7888760\ttotal: 11.4s\tremaining: 2.8s\n",
      "499:\tlearn: 1.7577161\ttotal: 13.9s\tremaining: 0us\n",
      "Training_column: 105\n",
      "0:\tlearn: 1.0869964\ttotal: 27.2ms\tremaining: 10.9s\n",
      "100:\tlearn: 1.0280321\ttotal: 2.53s\tremaining: 7.48s\n",
      "200:\tlearn: 1.0198517\ttotal: 5.13s\tremaining: 5.08s\n",
      "300:\tlearn: 1.0133282\ttotal: 7.88s\tremaining: 2.59s\n",
      "399:\tlearn: 1.0072076\ttotal: 10.4s\tremaining: 0us\n",
      "Training_column: 106\n",
      "0:\tlearn: 3.3120536\ttotal: 29ms\tremaining: 14.5s\n",
      "100:\tlearn: 2.2371329\ttotal: 2.62s\tremaining: 10.3s\n",
      "200:\tlearn: 2.1746295\ttotal: 5.91s\tremaining: 8.79s\n",
      "300:\tlearn: 2.1365766\ttotal: 8.44s\tremaining: 5.58s\n",
      "400:\tlearn: 2.1066479\ttotal: 11.3s\tremaining: 2.8s\n",
      "499:\tlearn: 2.0795646\ttotal: 13.9s\tremaining: 0us\n",
      "Training_column: 107\n",
      "0:\tlearn: 1.5288544\ttotal: 25.5ms\tremaining: 10.2s\n",
      "100:\tlearn: 1.3899523\ttotal: 2.75s\tremaining: 8.16s\n",
      "200:\tlearn: 1.3671985\ttotal: 5.15s\tremaining: 5.1s\n",
      "300:\tlearn: 1.3535422\ttotal: 7.6s\tremaining: 2.5s\n",
      "399:\tlearn: 1.3419491\ttotal: 10s\tremaining: 0us\n",
      "Training_column: 108\n",
      "0:\tlearn: 3.7011304\ttotal: 28.9ms\tremaining: 14.4s\n",
      "100:\tlearn: 1.8761226\ttotal: 2.69s\tremaining: 10.6s\n",
      "200:\tlearn: 1.8015200\ttotal: 5.59s\tremaining: 8.32s\n",
      "300:\tlearn: 1.7612727\ttotal: 8.03s\tremaining: 5.31s\n",
      "400:\tlearn: 1.7305897\ttotal: 10.5s\tremaining: 2.58s\n",
      "499:\tlearn: 1.7065179\ttotal: 12.9s\tremaining: 0us\n",
      "Training_column: 109\n",
      "0:\tlearn: 2.0949569\ttotal: 28.6ms\tremaining: 14.3s\n",
      "100:\tlearn: 1.6100949\ttotal: 2.56s\tremaining: 10.1s\n",
      "200:\tlearn: 1.5705256\ttotal: 5.01s\tremaining: 7.45s\n",
      "300:\tlearn: 1.5432348\ttotal: 7.73s\tremaining: 5.11s\n",
      "400:\tlearn: 1.5198734\ttotal: 10.7s\tremaining: 2.65s\n",
      "499:\tlearn: 1.4982787\ttotal: 13.4s\tremaining: 0us\n",
      "Training_column: 110\n",
      "0:\tlearn: 1.6318410\ttotal: 26.6ms\tremaining: 13.3s\n",
      "100:\tlearn: 1.1343147\ttotal: 2.6s\tremaining: 10.3s\n",
      "200:\tlearn: 1.1129642\ttotal: 5.07s\tremaining: 7.54s\n",
      "300:\tlearn: 1.0975369\ttotal: 7.55s\tremaining: 5s\n",
      "400:\tlearn: 1.0840380\ttotal: 10.4s\tremaining: 2.57s\n",
      "499:\tlearn: 1.0718238\ttotal: 12.9s\tremaining: 0us\n",
      "Training_column: 111\n",
      "0:\tlearn: 2.2903533\ttotal: 28.1ms\tremaining: 14s\n",
      "100:\tlearn: 1.6079120\ttotal: 3.78s\tremaining: 14.9s\n",
      "200:\tlearn: 1.5607569\ttotal: 6.36s\tremaining: 9.46s\n",
      "300:\tlearn: 1.5315960\ttotal: 8.95s\tremaining: 5.92s\n",
      "400:\tlearn: 1.5068580\ttotal: 11.6s\tremaining: 2.87s\n",
      "499:\tlearn: 1.4863297\ttotal: 14.3s\tremaining: 0us\n",
      "Training_column: 112\n",
      "0:\tlearn: 1.2451573\ttotal: 26.5ms\tremaining: 13.2s\n",
      "100:\tlearn: 0.9476787\ttotal: 2.8s\tremaining: 11.1s\n",
      "200:\tlearn: 0.9322356\ttotal: 5.26s\tremaining: 7.83s\n",
      "300:\tlearn: 0.9212843\ttotal: 7.75s\tremaining: 5.13s\n",
      "400:\tlearn: 0.9111557\ttotal: 10.3s\tremaining: 2.54s\n",
      "499:\tlearn: 0.9012538\ttotal: 13.5s\tremaining: 0us\n",
      "Training_column: 113\n",
      "0:\tlearn: 7.6205376\ttotal: 38.1ms\tremaining: 19s\n",
      "100:\tlearn: 4.0151850\ttotal: 2.59s\tremaining: 10.2s\n",
      "200:\tlearn: 3.8400966\ttotal: 5.36s\tremaining: 7.98s\n",
      "300:\tlearn: 3.7101149\ttotal: 7.81s\tremaining: 5.16s\n",
      "400:\tlearn: 3.5917505\ttotal: 10.3s\tremaining: 2.55s\n",
      "499:\tlearn: 3.4896998\ttotal: 12.8s\tremaining: 0us\n",
      "Training_column: 114\n",
      "0:\tlearn: 1.0596154\ttotal: 26ms\tremaining: 10.4s\n",
      "100:\tlearn: 0.9847998\ttotal: 2.46s\tremaining: 7.28s\n",
      "200:\tlearn: 0.9743490\ttotal: 5.35s\tremaining: 5.3s\n",
      "300:\tlearn: 0.9681594\ttotal: 8.41s\tremaining: 2.77s\n",
      "399:\tlearn: 0.9619982\ttotal: 10.9s\tremaining: 0us\n",
      "Training_column: 115\n",
      "0:\tlearn: 3.1533252\ttotal: 29.8ms\tremaining: 14.9s\n",
      "100:\tlearn: 2.6592466\ttotal: 2.56s\tremaining: 10.1s\n",
      "200:\tlearn: 2.5997910\ttotal: 5.05s\tremaining: 7.51s\n",
      "300:\tlearn: 2.5603842\ttotal: 7.5s\tremaining: 4.96s\n",
      "400:\tlearn: 2.5254043\ttotal: 10.2s\tremaining: 2.52s\n",
      "499:\tlearn: 2.4906602\ttotal: 12.8s\tremaining: 0us\n",
      "Training_column: 116\n",
      "0:\tlearn: 2.0002752\ttotal: 27.4ms\tremaining: 13.7s\n",
      "100:\tlearn: 1.5219969\ttotal: 2.96s\tremaining: 11.7s\n",
      "200:\tlearn: 1.4887847\ttotal: 5.42s\tremaining: 8.06s\n",
      "300:\tlearn: 1.4656694\ttotal: 7.91s\tremaining: 5.23s\n",
      "400:\tlearn: 1.4452526\ttotal: 10.4s\tremaining: 2.57s\n",
      "499:\tlearn: 1.4272096\ttotal: 13.3s\tremaining: 0us\n",
      "Training_column: 117\n",
      "0:\tlearn: 1.0441556\ttotal: 27.6ms\tremaining: 11s\n",
      "100:\tlearn: 0.9893507\ttotal: 2.59s\tremaining: 7.67s\n",
      "200:\tlearn: 0.9803211\ttotal: 5.48s\tremaining: 5.42s\n",
      "300:\tlearn: 0.9733640\ttotal: 8.05s\tremaining: 2.65s\n",
      "399:\tlearn: 0.9670608\ttotal: 10.8s\tremaining: 0us\n",
      "Training_column: 118\n",
      "0:\tlearn: 1.2532105\ttotal: 28.6ms\tremaining: 11.4s\n",
      "100:\tlearn: 1.1705380\ttotal: 2.6s\tremaining: 7.68s\n",
      "200:\tlearn: 1.1552282\ttotal: 5.06s\tremaining: 5.01s\n",
      "300:\tlearn: 1.1455970\ttotal: 7.88s\tremaining: 2.59s\n",
      "399:\tlearn: 1.1373108\ttotal: 10.3s\tremaining: 0us\n",
      "Training_column: 119\n",
      "0:\tlearn: 3.6561627\ttotal: 29.6ms\tremaining: 14.7s\n",
      "100:\tlearn: 2.5913701\ttotal: 3.02s\tremaining: 11.9s\n",
      "200:\tlearn: 2.5048438\ttotal: 5.58s\tremaining: 8.3s\n",
      "300:\tlearn: 2.4574110\ttotal: 8.87s\tremaining: 5.87s\n",
      "400:\tlearn: 2.4227109\ttotal: 11.4s\tremaining: 2.82s\n",
      "499:\tlearn: 2.3912895\ttotal: 14.3s\tremaining: 0us\n",
      "Training_column: 120\n",
      "0:\tlearn: 0.9589395\ttotal: 26.6ms\tremaining: 7.96s\n",
      "100:\tlearn: 0.9190537\ttotal: 2.77s\tremaining: 5.46s\n",
      "200:\tlearn: 0.9126187\ttotal: 5.19s\tremaining: 2.56s\n",
      "299:\tlearn: 0.9087709\ttotal: 7.66s\tremaining: 0us\n",
      "Training_column: 121\n",
      "0:\tlearn: 4.5677775\ttotal: 28.8ms\tremaining: 14.4s\n",
      "100:\tlearn: 3.1453229\ttotal: 2.61s\tremaining: 10.3s\n",
      "200:\tlearn: 3.0593242\ttotal: 5.84s\tremaining: 8.68s\n",
      "300:\tlearn: 2.9994840\ttotal: 8.37s\tremaining: 5.53s\n",
      "400:\tlearn: 2.9491239\ttotal: 11.3s\tremaining: 2.78s\n",
      "499:\tlearn: 2.9049004\ttotal: 13.8s\tremaining: 0us\n",
      "Training_column: 122\n",
      "0:\tlearn: 0.9778287\ttotal: 35.8ms\tremaining: 10.7s\n",
      "100:\tlearn: 0.9637852\ttotal: 2.63s\tremaining: 5.19s\n",
      "200:\tlearn: 0.9591687\ttotal: 5.18s\tremaining: 2.55s\n",
      "299:\tlearn: 0.9550480\ttotal: 7.7s\tremaining: 0us\n",
      "Training_column: 123\n",
      "0:\tlearn: 1.0662182\ttotal: 56.7ms\tremaining: 17s\n",
      "100:\tlearn: 1.0541429\ttotal: 3.34s\tremaining: 6.58s\n",
      "200:\tlearn: 1.0494342\ttotal: 5.9s\tremaining: 2.9s\n",
      "299:\tlearn: 1.0450730\ttotal: 8.76s\tremaining: 0us\n",
      "Training_column: 124\n",
      "0:\tlearn: 1.0098004\ttotal: 30.7ms\tremaining: 12.3s\n",
      "100:\tlearn: 0.9385827\ttotal: 2.95s\tremaining: 8.73s\n",
      "200:\tlearn: 0.9307654\ttotal: 5.64s\tremaining: 5.58s\n",
      "300:\tlearn: 0.9246816\ttotal: 8.22s\tremaining: 2.7s\n",
      "399:\tlearn: 0.9184776\ttotal: 10.8s\tremaining: 0us\n",
      "Training_column: 125\n",
      "0:\tlearn: 0.9888188\ttotal: 28ms\tremaining: 11.2s\n",
      "100:\tlearn: 0.9268954\ttotal: 2.67s\tremaining: 7.92s\n",
      "200:\tlearn: 0.9190063\ttotal: 5.29s\tremaining: 5.24s\n",
      "300:\tlearn: 0.9127711\ttotal: 8.22s\tremaining: 2.7s\n",
      "399:\tlearn: 0.9065335\ttotal: 10.8s\tremaining: 0us\n",
      "Training_column: 126\n",
      "0:\tlearn: 0.8975341\ttotal: 28.9ms\tremaining: 8.65s\n",
      "100:\tlearn: 0.8787332\ttotal: 2.69s\tremaining: 5.29s\n",
      "200:\tlearn: 0.8744136\ttotal: 5.33s\tremaining: 2.63s\n",
      "299:\tlearn: 0.8706003\ttotal: 7.9s\tremaining: 0us\n",
      "Training_column: 127\n",
      "0:\tlearn: 1.2313813\ttotal: 28.3ms\tremaining: 14.1s\n",
      "100:\tlearn: 0.9888617\ttotal: 2.64s\tremaining: 10.4s\n",
      "200:\tlearn: 0.9662883\ttotal: 5.2s\tremaining: 7.73s\n",
      "300:\tlearn: 0.9503245\ttotal: 8.09s\tremaining: 5.35s\n",
      "400:\tlearn: 0.9369873\ttotal: 10.6s\tremaining: 2.63s\n",
      "499:\tlearn: 0.9246594\ttotal: 13.2s\tremaining: 0us\n",
      "Training_column: 128\n",
      "0:\tlearn: 0.8944703\ttotal: 26.4ms\tremaining: 7.91s\n",
      "100:\tlearn: 0.8646491\ttotal: 2.46s\tremaining: 4.84s\n",
      "200:\tlearn: 0.8601903\ttotal: 4.95s\tremaining: 2.44s\n",
      "299:\tlearn: 0.8566961\ttotal: 7.46s\tremaining: 0us\n",
      "Training_column: 129\n",
      "0:\tlearn: 0.9523624\ttotal: 25.5ms\tremaining: 10.2s\n",
      "100:\tlearn: 0.8967905\ttotal: 2.44s\tremaining: 7.23s\n",
      "200:\tlearn: 0.8878906\ttotal: 5.28s\tremaining: 5.23s\n",
      "300:\tlearn: 0.8816978\ttotal: 7.79s\tremaining: 2.56s\n",
      "399:\tlearn: 0.8760538\ttotal: 10.3s\tremaining: 0us\n",
      "Training_column: 130\n",
      "0:\tlearn: 0.8342364\ttotal: 29.2ms\tremaining: 8.74s\n",
      "100:\tlearn: 0.8094468\ttotal: 2.63s\tremaining: 5.19s\n",
      "200:\tlearn: 0.8043268\ttotal: 5.25s\tremaining: 2.59s\n",
      "299:\tlearn: 0.8004779\ttotal: 7.82s\tremaining: 0us\n",
      "Training_column: 131\n",
      "0:\tlearn: 3.1416527\ttotal: 29.4ms\tremaining: 14.7s\n",
      "100:\tlearn: 2.4206631\ttotal: 2.7s\tremaining: 10.7s\n",
      "200:\tlearn: 2.3491943\ttotal: 5.73s\tremaining: 8.52s\n",
      "300:\tlearn: 2.3077081\ttotal: 8.36s\tremaining: 5.53s\n",
      "400:\tlearn: 2.2723641\ttotal: 11s\tremaining: 2.72s\n",
      "499:\tlearn: 2.2424668\ttotal: 13.6s\tremaining: 0us\n",
      "Training_column: 132\n",
      "0:\tlearn: 1.1295782\ttotal: 28.7ms\tremaining: 11.5s\n",
      "100:\tlearn: 1.0030874\ttotal: 2.69s\tremaining: 7.97s\n",
      "200:\tlearn: 0.9925777\ttotal: 5.91s\tremaining: 5.85s\n",
      "300:\tlearn: 0.9853810\ttotal: 8.79s\tremaining: 2.89s\n",
      "399:\tlearn: 0.9785577\ttotal: 11.3s\tremaining: 0us\n",
      "Training_column: 133\n",
      "0:\tlearn: 0.7348033\ttotal: 28.3ms\tremaining: 11.3s\n",
      "100:\tlearn: 0.6940246\ttotal: 2.67s\tremaining: 7.92s\n",
      "200:\tlearn: 0.6880968\ttotal: 5.23s\tremaining: 5.18s\n",
      "300:\tlearn: 0.6835648\ttotal: 7.74s\tremaining: 2.55s\n",
      "399:\tlearn: 0.6793574\ttotal: 10.5s\tremaining: 0us\n",
      "Training_column: 134\n",
      "0:\tlearn: 1.7759725\ttotal: 27.7ms\tremaining: 13.8s\n",
      "100:\tlearn: 1.2154586\ttotal: 3.41s\tremaining: 13.5s\n",
      "200:\tlearn: 1.1652191\ttotal: 6.16s\tremaining: 9.17s\n",
      "300:\tlearn: 1.1384456\ttotal: 8.55s\tremaining: 5.66s\n",
      "400:\tlearn: 1.1191147\ttotal: 10.9s\tremaining: 2.7s\n",
      "499:\tlearn: 1.1038398\ttotal: 13.3s\tremaining: 0us\n",
      "Training_column: 135\n",
      "0:\tlearn: 0.9397048\ttotal: 27.7ms\tremaining: 11s\n",
      "100:\tlearn: 0.8691725\ttotal: 2.61s\tremaining: 7.73s\n",
      "200:\tlearn: 0.8620371\ttotal: 5.16s\tremaining: 5.11s\n",
      "300:\tlearn: 0.8558482\ttotal: 8.1s\tremaining: 2.66s\n",
      "399:\tlearn: 0.8499284\ttotal: 10.6s\tremaining: 0us\n",
      "Training_column: 136\n",
      "0:\tlearn: 2.8182868\ttotal: 29.3ms\tremaining: 14.6s\n",
      "100:\tlearn: 1.8836204\ttotal: 2.69s\tremaining: 10.6s\n",
      "200:\tlearn: 1.8042636\ttotal: 5.25s\tremaining: 7.81s\n",
      "300:\tlearn: 1.7645586\ttotal: 7.88s\tremaining: 5.21s\n",
      "400:\tlearn: 1.7328562\ttotal: 10.9s\tremaining: 2.68s\n",
      "499:\tlearn: 1.7063341\ttotal: 13.4s\tremaining: 0us\n",
      "Training_column: 137\n",
      "0:\tlearn: 1.3422823\ttotal: 25.3ms\tremaining: 12.6s\n",
      "100:\tlearn: 1.0067532\ttotal: 2.81s\tremaining: 11.1s\n",
      "200:\tlearn: 0.9862860\ttotal: 5.26s\tremaining: 7.82s\n",
      "300:\tlearn: 0.9723822\ttotal: 8.47s\tremaining: 5.6s\n",
      "400:\tlearn: 0.9608692\ttotal: 10.9s\tremaining: 2.69s\n",
      "499:\tlearn: 0.9504502\ttotal: 13.6s\tremaining: 0us\n",
      "Training_column: 138\n",
      "0:\tlearn: 2.6411251\ttotal: 27.8ms\tremaining: 13.9s\n",
      "100:\tlearn: 1.5082842\ttotal: 3s\tremaining: 11.8s\n",
      "200:\tlearn: 1.4661993\ttotal: 5.44s\tremaining: 8.09s\n",
      "300:\tlearn: 1.4400166\ttotal: 7.89s\tremaining: 5.22s\n",
      "400:\tlearn: 1.4178977\ttotal: 10.3s\tremaining: 2.55s\n",
      "499:\tlearn: 1.3998175\ttotal: 12.8s\tremaining: 0us\n",
      "Training_column: 139\n",
      "0:\tlearn: 2.4436847\ttotal: 29.5ms\tremaining: 11.8s\n",
      "100:\tlearn: 1.9771091\ttotal: 2.76s\tremaining: 8.17s\n",
      "200:\tlearn: 1.9333058\ttotal: 5.71s\tremaining: 5.65s\n",
      "300:\tlearn: 1.9093351\ttotal: 8.32s\tremaining: 2.74s\n",
      "399:\tlearn: 1.8905347\ttotal: 10.9s\tremaining: 0us\n"
     ]
    }
   ],
   "source": [
    "# Make predictions.\n",
    "if SUBMIT:\n",
    "    df = add_metadata_features(df)\n",
    "    df_test = add_metadata_features(df_test)\n",
    "    x_cols = [col for col in list(df.columns) if (col.startswith('ENSG')) | (col.startswith('svd'))]\n",
    "    y_cols = [col for col in list(df.columns) if (col.startswith('y_'))]\n",
    "    #y_cols = y_cols[:5]\n",
    "    X = df[x_cols].values\n",
    "    Y = df[y_cols].values\n",
    "    Xt = df_test[x_cols].values\n",
    "    for i in range(len(y_cols)):\n",
    "        print('Training_column: ' + str(i))\n",
    "        #model = lightgbm.LGBMRegressor(n_estimators=n_estimators, **lightgbm_params)\n",
    "        if y_cols[i] in y_slow:\n",
    "            model = CatBoostRegressor(**cat_params_slow)\n",
    "        elif y_cols[i] in y_fast:\n",
    "            model = CatBoostRegressor(**cat_params_fast)\n",
    "        else:\n",
    "            model = CatBoostRegressor(**cat_params_middle)\n",
    "        model.fit(X, Y[:,i].copy())\n",
    "        col_name = y_cols[i]\n",
    "        df_test[col_name] = model.predict(Xt)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9a1ebf74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T13:34:44.767315Z",
     "iopub.status.busy": "2023-03-26T13:34:44.766974Z",
     "iopub.status.idle": "2023-03-26T13:34:46.541820Z",
     "shell.execute_reply": "2023-03-26T13:34:46.540738Z"
    },
    "papermill": {
     "duration": 1.845731,
     "end_time": "2023-03-26T13:34:46.544534",
     "exception": false,
     "start_time": "2023-03-26T13:34:44.698803",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if SUBMIT:\n",
    "    del df, X, Y, Xt\n",
    "    for col in df_test.columns:\n",
    "        if col in x_cols:\n",
    "            del df_test[col]\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "81dc284d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-26T13:34:46.647582Z",
     "iopub.status.busy": "2023-03-26T13:34:46.647260Z",
     "iopub.status.idle": "2023-03-26T13:34:46.797135Z",
     "shell.execute_reply": "2023-03-26T13:34:46.796005Z"
    },
    "papermill": {
     "duration": 0.203638,
     "end_time": "2023-03-26T13:34:46.799802",
     "exception": false,
     "start_time": "2023-03-26T13:34:46.596164",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Export predictions to file.\n",
    "if SUBMIT:\n",
    "    export_file_results = 'results_cite.ftr'\n",
    "    df_test.to_feather(export_file_results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 2703.84364,
   "end_time": "2023-03-26T13:34:47.575853",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-26T12:49:43.732213",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}