demo/supervised-sequence-to-target.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Supervised task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "if not os.path.exists('data/transactions_train.csv'):\n",
    "    ! mkdir -p data\n",
    "    ! curl -OL https://storage.yandexcloud.net/ptls-datasets/age-prediction-nti-sbebank-2019.zip\n",
    "    ! unzip -j -o age-prediction-nti-sbebank-2019.zip 'data/*.csv' -d data\n",
    "    ! mv age-prediction-nti-sbebank-2019.zip data/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare your data\n",
    "\n",
    "- Use `Pyspark` in local or cluster mode for big dataset and `Pandas` for small.\n",
    "- Split data into required parts (train, valid, test, ...).\n",
    "- Use `ptls.preprocessing` for simple data preparation. \n",
    "- Transform features to compatible format using `Pyspark` or `Pandas` functions. \n",
    "You can also use `ptls.data_load.preprocessing` for common data transformation patterns.\n",
    "- Split sequences to `ptls-data` format with `ptls.data_load.split_tools`. Save prepared data into `Parquet` format or \n",
    "keep it in memory (`Pickle` also works).\n",
    "- Use one of the available `ptls.data_load.datasets` to define input for the models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from ptls.preprocessing import PandasDataPreprocessor\n",
    "from ptls.data_load.datasets import MemoryMapDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load and split target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>client_id</th>\n",
       "      <th>bins</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24662</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1046</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34089</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34848</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>47076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   client_id  bins\n",
       "0      24662     2\n",
       "1       1046     0\n",
       "2      34089     2\n",
       "3      34848     1\n",
       "4      47076     3"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_target = pd.read_csv('data/train_target.csv')\n",
    "df_target.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 30000 records to train: 20000, valid: 3000, test: 7000\n"
     ]
    }
   ],
   "source": [
    "df_target_train, df_target_test = train_test_split(\n",
    "    df_target, test_size=7000, stratify=df_target['bins'], random_state=142)\n",
    "df_target_train, df_target_valid = train_test_split(\n",
    "    df_target_train, test_size=3000, stratify=df_target_train['bins'], random_state=142)\n",
    "print('Split {} records to train: {}, valid: {}, test: {}'.format(\n",
    "    *[len(df) for df in [df_target, df_target_train, df_target_valid, df_target_test]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load and split transactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>client_id</th>\n",
       "      <th>trans_date</th>\n",
       "      <th>small_group</th>\n",
       "      <th>amount_rur</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>33172</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>71.463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33172</td>\n",
       "      <td>6</td>\n",
       "      <td>35</td>\n",
       "      <td>45.017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33172</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>13.887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>33172</td>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>15.983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33172</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>21.341</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   client_id  trans_date  small_group  amount_rur\n",
       "0      33172           6            4      71.463\n",
       "1      33172           6           35      45.017\n",
       "2      33172           8           11      13.887\n",
       "3      33172           9           11      15.983\n",
       "4      33172          10           11      21.341"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trx = pd.read_csv('data/transactions_train.csv')\n",
    "df_trx.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 26450577 transactions to train: 17622321, valid: 2634248, test: 6194008\n"
     ]
    }
   ],
   "source": [
    "df_trx_train = pd.merge(df_trx, df_target_train['client_id'], on='client_id', how='inner')\n",
    "df_trx_valid = pd.merge(df_trx, df_target_valid['client_id'], on='client_id', how='inner')\n",
    "df_trx_test = pd.merge(df_trx, df_target_test['client_id'], on='client_id', how='inner')\n",
    "print('Split {} transactions to train: {}, valid: {}, test: {}'.format(\n",
    "    *[len(df) for df in [df_trx, df_trx_train, df_trx_valid, df_trx_test]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform flat table to dictionaries with client features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ptls.preprocessing import PandasDataPreprocessor\n",
    "\n",
    "preprocessor = PandasDataPreprocessor(\n",
    "    col_id='client_id',\n",
    "    col_event_time='trans_date',\n",
    "    event_time_transformation='none',\n",
    "    cols_category=['small_group'],\n",
    "    cols_numerical=['amount_rur'],\n",
    "    return_records=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 37.6 s, sys: 10.2 s, total: 47.8 s\n",
      "Wall time: 1min 2s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_data_train = preprocessor.fit_transform(df_trx_train)\n",
    "df_data_valid = preprocessor.transform(df_trx_valid)\n",
    "df_data_test = preprocessor.transform(df_trx_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Record in dataset, train 20000, valid 3000, test 7000\n",
      "Each record is a client with list of transactions\n"
     ]
    }
   ],
   "source": [
    "print('Record in dataset, train {}, valid {}, test {}\\nEach record is a client with list of transactions'.format(\n",
    "    *[len(df) for df in [df_data_train, df_data_valid, df_data_test]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>client_id</th>\n",
       "      <th>trans_date</th>\n",
       "      <th>small_group</th>\n",
       "      <th>amount_rur</th>\n",
       "      <th>event_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>[tensor(0), tensor(5), tensor(10), tensor(11),...</td>\n",
       "      <td>[tensor(4), tensor(3), tensor(1), tensor(3), t...</td>\n",
       "      <td>[tensor(4.0540, dtype=torch.float64), tensor(1...</td>\n",
       "      <td>[tensor(0), tensor(5), tensor(10), tensor(11),...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>[tensor(1), tensor(2), tensor(12), tensor(13),...</td>\n",
       "      <td>[tensor(3), tensor(53), tensor(1), tensor(5), ...</td>\n",
       "      <td>[tensor(18.3190, dtype=torch.float64), tensor(...</td>\n",
       "      <td>[tensor(1), tensor(2), tensor(12), tensor(13),...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>[tensor(3), tensor(6), tensor(6), tensor(6), t...</td>\n",
       "      <td>[tensor(1), tensor(19), tensor(13), tensor(6),...</td>\n",
       "      <td>[tensor(3.0220, dtype=torch.float64), tensor(2...</td>\n",
       "      <td>[tensor(3), tensor(6), tensor(6), tensor(6), t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   client_id                                         trans_date  \\\n",
       "0          6  [tensor(0), tensor(5), tensor(10), tensor(11),...   \n",
       "1          7  [tensor(1), tensor(2), tensor(12), tensor(13),...   \n",
       "2         12  [tensor(3), tensor(6), tensor(6), tensor(6), t...   \n",
       "\n",
       "                                         small_group  \\\n",
       "0  [tensor(4), tensor(3), tensor(1), tensor(3), t...   \n",
       "1  [tensor(3), tensor(53), tensor(1), tensor(5), ...   \n",
       "2  [tensor(1), tensor(19), tensor(13), tensor(6),...   \n",
       "\n",
       "                                          amount_rur  \\\n",
       "0  [tensor(4.0540, dtype=torch.float64), tensor(1...   \n",
       "1  [tensor(18.3190, dtype=torch.float64), tensor(...   \n",
       "2  [tensor(3.0220, dtype=torch.float64), tensor(2...   \n",
       "\n",
       "                                          event_time  \n",
       "0  [tensor(0), tensor(5), tensor(10), tensor(11),...  \n",
       "1  [tensor(1), tensor(2), tensor(12), tensor(13),...  \n",
       "2  [tensor(3), tensor(6), tensor(6), tensor(6), t...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data_train.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_target = df_target.rename(columns={'bins': 'target_bin'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data_train = pd.merge(df_data_train, df_target, on='client_id')\n",
    "df_data_valid = pd.merge(df_data_valid, df_target, on='client_id')\n",
    "df_data_test = pd.merge(df_data_test, df_target, on='client_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data_train = df_data_train.to_dict(orient='records')\n",
    "df_data_valid = df_data_valid.to_dict(orient='records')\n",
    "df_data_test = df_data_test.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'client_id': 6,\n",
       " 'trans_date': tensor([ 0,  5, 10, 11, 15, 15, 16, 16, 17, 18]),\n",
       " 'small_group': tensor([ 4,  3,  1,  3,  4,  1,  4,  3, 18,  2]),\n",
       " 'amount_rur': tensor([ 4.0540, 13.7380, 20.7010, 21.5640, 13.4990, 23.7220,  4.3040,  8.6250,\n",
       "         12.9380, 28.1620], dtype=torch.float64),\n",
       " 'event_time': tensor([ 0,  5, 10, 11, 15, 15, 16, 16, 17, 18]),\n",
       " 'target_bin': 1}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show first 10 transactions from one record\n",
    "rec = df_data_train[0]\n",
    "{k: v[:10] if type(v) is torch.Tensor else v for k, v in rec.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make torch datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_train = MemoryMapDataset(df_data_train)\n",
    "dataset_valid = MemoryMapDataset(df_data_valid)\n",
    "dataset_test = MemoryMapDataset(df_data_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build encoder\n",
    "\n",
    "- All parts are available in `ptls.nn`.\n",
    "- You can also use pretrained layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchmetrics\n",
    "from ptls.nn import TrxEncoder, RnnSeqEncoder, Head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'small_group': 203}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check preprocessor dictionary sisez for categoryes\n",
    "preprocessor.get_category_dictionary_sizes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_encoder = RnnSeqEncoder(\n",
    "    trx_encoder=TrxEncoder(\n",
    "        embeddings={\n",
    "            'small_group': {'in': 150, 'out': 32},\n",
    "        },\n",
    "        numeric_values={\n",
    "            'amount_rur': 'log',\n",
    "        },\n",
    "        embeddings_noise=0.001,\n",
    "    ),\n",
    "    hidden_size=48,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose framework for encoder train\n",
    "\n",
    "- There are both supervised of unsupervised frameworks in `ptls.frames`.\n",
    "- Keep in mind that each framework requires his own batch format.\n",
    "Tools for batch collate can be found in the selected framework package."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from ptls.frames.supervised import SeqToTargetDataset, SequenceToTarget\n",
    "from ptls.frames import PtlsDataModule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "sup_module = SequenceToTarget(\n",
    "    seq_encoder=seq_encoder,\n",
    "    head=Head(input_size=seq_encoder.embedding_size, objective='classification', num_classes=4),\n",
    "    loss=torch.nn.NLLLoss(),\n",
    "    metric_list=torchmetrics.Accuracy(),\n",
    "    optimizer_partial=partial(torch.optim.Adam),\n",
    "    lr_scheduler_partial=partial(torch.optim.lr_scheduler.StepLR, step_size=4, gamma=0.5),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "sup_data = PtlsDataModule(\n",
    "    train_data=SeqToTargetDataset(dataset_train, target_col_name='target_bin', target_dtype=torch.long),\n",
    "    valid_data=SeqToTargetDataset(dataset_valid, target_col_name='target_bin', target_dtype=torch.long),\n",
    "    test_data=SeqToTargetDataset(dataset_test, target_col_name='target_bin', target_dtype=torch.long),\n",
    "    train_batch_size=128,\n",
    "    valid_batch_size=1024,\n",
    "    train_num_workers=8,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train your encoder with selected framework and `pytorch_lightning`\n",
    "\n",
    "- Provide data with one of the DataLoaders that is compatible with selected framework. \n",
    "- Monitor the progress on tensorboard.\n",
    "- Optionally tune hyperparameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pytorch_lightning as pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the TensorBoard notebook extension\n",
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%tensorboard --logdir lightning_logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "trainer = pl.Trainer(\n",
    "    max_epochs=10,\n",
    "    gpus=1 if torch.cuda.is_available() else 0,\n",
    "    enable_progress_bar=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logger.version = 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "\n",
      "  | Name          | Type          | Params\n",
      "------------------------------------------------\n",
      "0 | seq_encoder   | RnnSeqEncoder | 16.8 K\n",
      "1 | head          | Head          | 196   \n",
      "2 | loss          | NLLLoss       | 0     \n",
      "3 | train_metrics | ModuleDict    | 0     \n",
      "4 | valid_metrics | ModuleDict    | 0     \n",
      "5 | test_metrics  | ModuleDict    | 0     \n",
      "------------------------------------------------\n",
      "17.0 K    Trainable params\n",
      "0         Non-trainable params\n",
      "17.0 K    Total params\n",
      "0.068     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 13s, sys: 20.1 s, total: 3min 33s\n",
      "Wall time: 5min 31s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(f'logger.version = {trainer.logger.version}')\n",
    "trainer.fit(sup_module, sup_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': tensor(0.8221), 'seq_len': tensor(854.8750), 'val_Accuracy': tensor(0.5311), 'train_Accuracy': tensor(0.5223)}\n"
     ]
    }
   ],
   "source": [
    "# train and validation metrics\n",
    "print(trainer.logged_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Restoring states from the checkpoint path at /home/kireev/pycharm-deploy/pytorch-lifestream/demo/lightning_logs/version_2/checkpoints/epoch=9-step=1570.ckpt\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "Loaded model weights from checkpoint at /home/kireev/pycharm-deploy/pytorch-lifestream/demo/lightning_logs/version_2/checkpoints/epoch=9-step=1570.ckpt\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\">        Test metric        </span>┃<span style=\"font-weight: bold\">       DataLoader 0        </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">       test_Accuracy       </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.5738499760627747     </span>│\n",
       "└───────────────────────────┴───────────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[36m \u001b[0m\u001b[36m      test_Accuracy      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.5738499760627747    \u001b[0m\u001b[35m \u001b[0m│\n",
       "└───────────────────────────┴───────────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'test_Accuracy': 0.5738499760627747}]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test metrics\n",
    "trainer.test(ckpt_path='best', dataloaders=sup_data.test_dataloader())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make predict\n",
    "\n",
    "Let's make predict to check metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ptls.data_load.utils import collate_feature_dict\n",
    "from ptls.frames.inference_module import InferenceModule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "inference_dl = torch.utils.data.DataLoader(\n",
    "    dataset=dataset_test,\n",
    "    collate_fn=collate_feature_dict,\n",
    "    shuffle=False,\n",
    "    batch_size=1000,\n",
    "    num_workers=4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "inf_module = InferenceModule(\n",
    "    torch.nn.Sequential(\n",
    "        sup_module,\n",
    "        torch.nn.Softmax(dim=1),\n",
    "    ),\n",
    "    model_out_name='prob',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    }
   ],
   "source": [
    "df_predict = trainer.predict(inf_module, inference_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_predict = pd.concat(df_predict, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>client_id</th>\n",
       "      <th>target_bin</th>\n",
       "      <th>prob_0000</th>\n",
       "      <th>prob_0001</th>\n",
       "      <th>prob_0002</th>\n",
       "      <th>prob_0003</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0.029013</td>\n",
       "      <td>0.801605</td>\n",
       "      <td>0.006939</td>\n",
       "      <td>0.162442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0.426665</td>\n",
       "      <td>0.022428</td>\n",
       "      <td>0.444309</td>\n",
       "      <td>0.106599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>0.082625</td>\n",
       "      <td>0.004733</td>\n",
       "      <td>0.903185</td>\n",
       "      <td>0.009457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>0.223670</td>\n",
       "      <td>0.007996</td>\n",
       "      <td>0.741577</td>\n",
       "      <td>0.026757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0.155119</td>\n",
       "      <td>0.408157</td>\n",
       "      <td>0.018226</td>\n",
       "      <td>0.418498</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   client_id  target_bin  prob_0000  prob_0001  prob_0002  prob_0003\n",
       "0          6           1   0.029013   0.801605   0.006939   0.162442\n",
       "1          7           0   0.426665   0.022428   0.444309   0.106599\n",
       "2         12           2   0.082625   0.004733   0.903185   0.009457\n",
       "3         13           2   0.223670   0.007996   0.741577   0.026757\n",
       "4         14           0   0.155119   0.408157   0.018226   0.418498"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_predict.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 2, ..., 2, 2, 1])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = df_predict[[f'prob_{i:04d}' for i in range(4)]].values.argmax(axis=1)\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 2, ..., 2, 2, 1])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true = df_predict['target_bin'].values\n",
    "y_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.57385"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2272,  305,  784, 1595],\n",
       "       [ 293, 3220,   48, 1437],\n",
       "       [1362,   51, 3354,  273],\n",
       "       [1109, 1158,  108, 2631]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}