Train_Empathy_Adapters.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5c8a5f18",
   "metadata": {},
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9d9c340a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from transformers import RobertaTokenizer\n",
    "from collections import Counter\n",
    "from sklearn.metrics import f1_score, recall_score, precision_score\n",
    "\n",
    "import torch\n",
    "from transformers import AutoConfig, AutoModelWithHeads\n",
    "from transformers.trainer_utils import set_seed\n",
    "from transformers import EvalPrediction\n",
    "from transformers import AdapterTrainer\n",
    "from transformers import TrainingArguments\n",
    "from transformers import EarlyStoppingCallback\n",
    "\n",
    "\n",
    "RANDOM_SEED = 42\n",
    "set_seed(RANDOM_SEED)\n",
    "\n",
    "dateTimeObj = datetime.now()\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "505a2ffd",
   "metadata": {},
   "outputs": [],
   "source": [
    "approach = \"epitome-training\"\n",
    "\n",
    "\"\"\" choose from the options: emotional-reactions, interpretations, explorations\"\"\"\n",
    "adapter_name = 'emotional-reactions'\n",
    "\n",
    "\n",
    "\"\"\" set your desired location for training output \"\"\"\n",
    "training_output_dir = f\"./training_output/{approach}_{adapter_name}_{dateTimeObj.hour}{dateTimeObj.minute}-{dateTimeObj.day}-{dateTimeObj.month}\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f230090",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cb139ace",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sp_id</th>\n",
       "      <th>rp_id</th>\n",
       "      <th>seeker_post</th>\n",
       "      <th>response_post</th>\n",
       "      <th>level</th>\n",
       "      <th>rationales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65m92s</td>\n",
       "      <td>dgbdk7z</td>\n",
       "      <td>Help. Help me. I dunno what I'm doing anymore</td>\n",
       "      <td>That's pretty vague, do you not know what you'...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9ezsfi</td>\n",
       "      <td>e5t3oxh</td>\n",
       "      <td>I'm done saying I love you to her because I do...</td>\n",
       "      <td>idk what a Red pill means exactly but my advic...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6b2cmc</td>\n",
       "      <td>dhj8tcb</td>\n",
       "      <td>Always feel like I'm being criticized and mock...</td>\n",
       "      <td>I think it's social anxiety , that creates par...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8iz0as</td>\n",
       "      <td>dyvq1ne</td>\n",
       "      <td>My diet becomes fucked when i get depressed.. ...</td>\n",
       "      <td>By any chance do you think you're in a loop. J...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aow3l9</td>\n",
       "      <td>eg40ecq</td>\n",
       "      <td>I hate not knowing why. I was diagnosed with d...</td>\n",
       "      <td>depression. not sadness which is caused by som...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3079</th>\n",
       "      <td>8jltcy</td>\n",
       "      <td>dz0kvhi</td>\n",
       "      <td>does anyone else keep forgetting stuff the nee...</td>\n",
       "      <td>All day, every day. It's definitely not just y...</td>\n",
       "      <td>1</td>\n",
       "      <td>All day, every day. It's definitely not just y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3080</th>\n",
       "      <td>94xc3o</td>\n",
       "      <td>e3ok8c0</td>\n",
       "      <td>What does depression feel like?. Honest questi...</td>\n",
       "      <td>like being stuck in a black hole. At times you...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3081</th>\n",
       "      <td>3zbq8e</td>\n",
       "      <td>cykvlsj</td>\n",
       "      <td>I'm to scared to commit suicide.. All I can fe...</td>\n",
       "      <td>I probably would have considered bringing harm...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3082</th>\n",
       "      <td>5kpp98</td>\n",
       "      <td>dbpqi2p</td>\n",
       "      <td>I just want to disappear but I don't want to h...</td>\n",
       "      <td>People barely notice me too</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3083</th>\n",
       "      <td>7gqrti</td>\n",
       "      <td>dqz05c5</td>\n",
       "      <td>26 year old male, living at home, low income j...</td>\n",
       "      <td>I see what you mean, but imagine being 26, liv...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3084 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       sp_id    rp_id                                        seeker_post  \\\n",
       "0     65m92s  dgbdk7z      Help. Help me. I dunno what I'm doing anymore   \n",
       "1     9ezsfi  e5t3oxh  I'm done saying I love you to her because I do...   \n",
       "2     6b2cmc  dhj8tcb  Always feel like I'm being criticized and mock...   \n",
       "3     8iz0as  dyvq1ne  My diet becomes fucked when i get depressed.. ...   \n",
       "4     aow3l9  eg40ecq  I hate not knowing why. I was diagnosed with d...   \n",
       "...      ...      ...                                                ...   \n",
       "3079  8jltcy  dz0kvhi  does anyone else keep forgetting stuff the nee...   \n",
       "3080  94xc3o  e3ok8c0  What does depression feel like?. Honest questi...   \n",
       "3081  3zbq8e  cykvlsj  I'm to scared to commit suicide.. All I can fe...   \n",
       "3082  5kpp98  dbpqi2p  I just want to disappear but I don't want to h...   \n",
       "3083  7gqrti  dqz05c5  26 year old male, living at home, low income j...   \n",
       "\n",
       "                                          response_post  level  \\\n",
       "0     That's pretty vague, do you not know what you'...      0   \n",
       "1     idk what a Red pill means exactly but my advic...      0   \n",
       "2     I think it's social anxiety , that creates par...      0   \n",
       "3     By any chance do you think you're in a loop. J...      0   \n",
       "4     depression. not sadness which is caused by som...      0   \n",
       "...                                                 ...    ...   \n",
       "3079  All day, every day. It's definitely not just y...      1   \n",
       "3080  like being stuck in a black hole. At times you...      0   \n",
       "3081  I probably would have considered bringing harm...      0   \n",
       "3082                        People barely notice me too      0   \n",
       "3083  I see what you mean, but imagine being 26, liv...      0   \n",
       "\n",
       "                                             rationales  \n",
       "0                                                   NaN  \n",
       "1                                                   NaN  \n",
       "2                                                   NaN  \n",
       "3                                                   NaN  \n",
       "4                                                   NaN  \n",
       "...                                                 ...  \n",
       "3079  All day, every day. It's definitely not just y...  \n",
       "3080                                                NaN  \n",
       "3081                                                NaN  \n",
       "3082                                                NaN  \n",
       "3083                                                NaN  \n",
       "\n",
       "[3084 rows x 6 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" \n",
    "decide which adapter to train and load data\n",
    "\"\"\"\n",
    "epitome = pd.read_csv(f'./epitome-data/{adapter_name}-reddit.csv')\n",
    "epitome"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7e510329",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Make train/eval split \"\"\"\n",
    "\n",
    "pretrain_texts = list(epitome['response_post'].values)\n",
    "pretrain_labels = list(epitome['level'].values)\n",
    "\n",
    "train_texts, val_texts, train_labels, val_labels = train_test_split(pretrain_texts, pretrain_labels, test_size=.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a4dba7b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" encode response post text \"\"\"\n",
    "\n",
    "tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\", padding=\"max_length\")\n",
    "train_encodings = tokenizer(train_texts, truncation=True, padding=True)\n",
    "val_encodings = tokenizer(val_texts, truncation=True, padding=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8403f494",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>train</th>\n",
       "      <th>dev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>66.27</td>\n",
       "      <td>64.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>28.72</td>\n",
       "      <td>31.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>5.01</td>\n",
       "      <td>4.21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  train    dev\n",
       "0      0  66.27  64.08\n",
       "2      1  28.72  31.72\n",
       "1      2   5.01   4.21"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" show label distribution in train and val set\"\"\"\n",
    "\n",
    "train_label_counts = Counter(train_labels)\n",
    "val_label_counts = Counter(val_labels)\n",
    "\n",
    "train_total = sum(train_label_counts.values())\n",
    "val_total = sum(val_label_counts.values())\n",
    "label_counts = {'label':[], 'train':[], 'dev':[]}\n",
    "\n",
    "for lab in train_label_counts:\n",
    "    train_prop = round(train_label_counts[lab] / train_total, 4) * 100\n",
    "    val_prop = round(val_label_counts[lab] / val_total, 4) * 100\n",
    "    \n",
    "    label_counts['label'].append(lab)\n",
    "    label_counts['train'].append(train_prop)\n",
    "    label_counts['dev'].append(val_prop)\n",
    "    \n",
    "label_counts = pd.DataFrame(label_counts, columns=label_counts.keys()).sort_values(by=['train'], ascending=False)\n",
    "label_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e219a58f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" create torch dataset \"\"\"\n",
    "\n",
    "class Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx], dtype=torch.long)\n",
    "        return item\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "\n",
    "train_dataset = Dataset(train_encodings, train_labels)\n",
    "val_dataset = Dataset(val_encodings, val_labels)\n",
    "id2label = {id: int(label) for (id, label) in enumerate(sorted(np.unique(train_dataset.labels)))}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "981aaffb",
   "metadata": {},
   "source": [
    "# Adapter Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3d1b7ab5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/home/alahnala/miniconda3/envs/st/lib/python3.9/site-packages/transformers/adapters/models/roberta.py:250: FutureWarning: This class has been renamed to `RobertaAdapterModel` in v3. Please use the new class instead as this class might be removed in a future version.\n",
      "  warnings.warn(\n",
      "/app/home/alahnala/miniconda3/envs/st/lib/python3.9/site-packages/transformers/adapters/models/roberta.py:228: FutureWarning: This class has been renamed to `RobertaAdapterModel` in v3. Please use the new class instead as this class might be removed in a future version.\n",
      "  warnings.warn(\n",
      "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaModelWithHeads: ['lm_head.layer_norm.weight', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.bias']\n",
      "- This IS expected if you are initializing RobertaModelWithHeads from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModelWithHeads from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaModelWithHeads were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.embeddings.position_ids']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "\"\"\" init model \"\"\"\n",
    "\n",
    "config = AutoConfig.from_pretrained(\n",
    "    \"roberta-base\",\n",
    "    id2label=id2label,\n",
    ")\n",
    "model = AutoModelWithHeads.from_pretrained(\n",
    "    \"roberta-base\",\n",
    "    config=config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "81b9c9a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" create adapter with classification head and activate \"\"\"\n",
    "\n",
    "model.add_adapter(adapter_name)\n",
    "\n",
    "model.add_classification_head(\n",
    "    adapter_name,\n",
    "    num_labels=3,\n",
    "    id2label=id2label\n",
    "  )\n",
    "\n",
    "model.train_adapter(adapter_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a2ad5dd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the `WAND_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    }
   ],
   "source": [
    "\"\"\" set training arguments \"\"\"\n",
    "\n",
    "num_train_epochs=20\n",
    "per_device_train_batch_size=8\n",
    "per_device_eval_batch_size=8\n",
    "metric_for_best_model='eval_f1_macro'\n",
    "warmup_steps=1000\n",
    "weight_decay=0.1\n",
    "learning_rate=1e-04\n",
    "early_stopping_patience=10\n",
    "early_stopping_threshold=.005\n",
    "callbacks=[]\n",
    "callbacks=[EarlyStoppingCallback(early_stopping_patience=early_stopping_patience)]\n",
    "# callbacks=[EarlyStoppingCallback(early_stopping_patience=early_stopping_patience,\n",
    "#                                 early_stopping_threshold=early_stopping_threshold)]\n",
    "\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=training_output_dir,\n",
    "    num_train_epochs=num_train_epochs,\n",
    "    per_device_train_batch_size=per_device_train_batch_size,\n",
    "    per_device_eval_batch_size=per_device_eval_batch_size,\n",
    "    learning_rate=learning_rate,\n",
    "    warmup_steps=warmup_steps,\n",
    "    weight_decay=weight_decay,\n",
    "    logging_dir='./logs',\n",
    "    logging_steps=50,\n",
    "    eval_steps=50,\n",
    "    save_steps=50,\n",
    "    evaluation_strategy='steps',\n",
    "    disable_tqdm=False,\n",
    "    overwrite_output_dir=True,\n",
    "    remove_unused_columns=False,\n",
    "    save_strategy='steps',\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=metric_for_best_model,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3e753ba9",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" create compute_metrics function \"\"\"\n",
    "\n",
    "def compute_metrics(p: EvalPrediction):\n",
    "    preds = np.argmax(p.predictions, axis=1)\n",
    "    \n",
    "    metrics = {\"accuracy\": (preds == p.label_ids).mean()}  \n",
    "    \n",
    "    for avg in ['weighted', \"macro\"]:\n",
    "        metrics[f\"f1_{avg}\"] = f1_score(p.label_ids, preds, average=avg)\n",
    "        \n",
    "    for id, lab in id2label.items():\n",
    "        metrics[f\"f1_{id2label[id]}\"] = f1_score(p.label_ids, preds, average='micro', labels=[id])\n",
    "    \n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0db856cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" init trainer \"\"\"\n",
    "\n",
    "trainer = AdapterTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    compute_metrics=compute_metrics,\n",
    "    callbacks=callbacks,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "788e9ae6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/home/alahnala/miniconda3/envs/st/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 2775\n",
      "  Num Epochs = 20\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 6940\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2000' max='6940' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2000/6940 02:53 < 07:09, 11.50 it/s, Epoch 5/20]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1 Weighted</th>\n",
       "      <th>F1 Macro</th>\n",
       "      <th>F1 0</th>\n",
       "      <th>F1 1</th>\n",
       "      <th>F1 2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>1.146300</td>\n",
       "      <td>1.110520</td>\n",
       "      <td>0.042071</td>\n",
       "      <td>0.003397</td>\n",
       "      <td>0.026915</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.080745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>1.070900</td>\n",
       "      <td>1.023623</td>\n",
       "      <td>0.640777</td>\n",
       "      <td>0.500488</td>\n",
       "      <td>0.260355</td>\n",
       "      <td>0.781065</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>0.967100</td>\n",
       "      <td>0.920111</td>\n",
       "      <td>0.640777</td>\n",
       "      <td>0.500488</td>\n",
       "      <td>0.260355</td>\n",
       "      <td>0.781065</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.885700</td>\n",
       "      <td>0.844712</td>\n",
       "      <td>0.640777</td>\n",
       "      <td>0.500488</td>\n",
       "      <td>0.260355</td>\n",
       "      <td>0.781065</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>0.823500</td>\n",
       "      <td>0.808060</td>\n",
       "      <td>0.640777</td>\n",
       "      <td>0.500488</td>\n",
       "      <td>0.260355</td>\n",
       "      <td>0.781065</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.789900</td>\n",
       "      <td>0.779988</td>\n",
       "      <td>0.640777</td>\n",
       "      <td>0.500488</td>\n",
       "      <td>0.260355</td>\n",
       "      <td>0.781065</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>0.708800</td>\n",
       "      <td>0.768778</td>\n",
       "      <td>0.640777</td>\n",
       "      <td>0.500488</td>\n",
       "      <td>0.260355</td>\n",
       "      <td>0.781065</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.721400</td>\n",
       "      <td>0.747500</td>\n",
       "      <td>0.640777</td>\n",
       "      <td>0.500488</td>\n",
       "      <td>0.260355</td>\n",
       "      <td>0.781065</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>0.726200</td>\n",
       "      <td>0.683300</td>\n",
       "      <td>0.650485</td>\n",
       "      <td>0.534767</td>\n",
       "      <td>0.293775</td>\n",
       "      <td>0.788732</td>\n",
       "      <td>0.092593</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>0.617693</td>\n",
       "      <td>0.750809</td>\n",
       "      <td>0.741733</td>\n",
       "      <td>0.498084</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>550</td>\n",
       "      <td>0.547100</td>\n",
       "      <td>0.567895</td>\n",
       "      <td>0.773463</td>\n",
       "      <td>0.757678</td>\n",
       "      <td>0.505273</td>\n",
       "      <td>0.855721</td>\n",
       "      <td>0.660099</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.567100</td>\n",
       "      <td>0.542020</td>\n",
       "      <td>0.786408</td>\n",
       "      <td>0.776357</td>\n",
       "      <td>0.523903</td>\n",
       "      <td>0.858667</td>\n",
       "      <td>0.713043</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>650</td>\n",
       "      <td>0.503700</td>\n",
       "      <td>0.532403</td>\n",
       "      <td>0.786408</td>\n",
       "      <td>0.776357</td>\n",
       "      <td>0.523903</td>\n",
       "      <td>0.858667</td>\n",
       "      <td>0.713043</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>0.576900</td>\n",
       "      <td>0.492483</td>\n",
       "      <td>0.786408</td>\n",
       "      <td>0.772275</td>\n",
       "      <td>0.517179</td>\n",
       "      <td>0.865823</td>\n",
       "      <td>0.685714</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>750</td>\n",
       "      <td>0.436800</td>\n",
       "      <td>0.545008</td>\n",
       "      <td>0.789644</td>\n",
       "      <td>0.779694</td>\n",
       "      <td>0.673913</td>\n",
       "      <td>0.871429</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>0.521739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.403900</td>\n",
       "      <td>0.520518</td>\n",
       "      <td>0.805825</td>\n",
       "      <td>0.803379</td>\n",
       "      <td>0.709291</td>\n",
       "      <td>0.869347</td>\n",
       "      <td>0.702970</td>\n",
       "      <td>0.555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>850</td>\n",
       "      <td>0.433700</td>\n",
       "      <td>0.573300</td>\n",
       "      <td>0.812298</td>\n",
       "      <td>0.816575</td>\n",
       "      <td>0.775658</td>\n",
       "      <td>0.860963</td>\n",
       "      <td>0.738739</td>\n",
       "      <td>0.727273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>0.532900</td>\n",
       "      <td>0.451836</td>\n",
       "      <td>0.822006</td>\n",
       "      <td>0.825044</td>\n",
       "      <td>0.802815</td>\n",
       "      <td>0.869110</td>\n",
       "      <td>0.739336</td>\n",
       "      <td>0.800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>950</td>\n",
       "      <td>0.524700</td>\n",
       "      <td>0.474374</td>\n",
       "      <td>0.828479</td>\n",
       "      <td>0.832226</td>\n",
       "      <td>0.813548</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.753488</td>\n",
       "      <td>0.814815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.513900</td>\n",
       "      <td>0.448331</td>\n",
       "      <td>0.818770</td>\n",
       "      <td>0.820111</td>\n",
       "      <td>0.760498</td>\n",
       "      <td>0.882653</td>\n",
       "      <td>0.711340</td>\n",
       "      <td>0.687500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1050</td>\n",
       "      <td>0.450100</td>\n",
       "      <td>0.471226</td>\n",
       "      <td>0.831715</td>\n",
       "      <td>0.829858</td>\n",
       "      <td>0.838973</td>\n",
       "      <td>0.876847</td>\n",
       "      <td>0.723404</td>\n",
       "      <td>0.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>0.409800</td>\n",
       "      <td>0.513897</td>\n",
       "      <td>0.812298</td>\n",
       "      <td>0.816169</td>\n",
       "      <td>0.801068</td>\n",
       "      <td>0.859416</td>\n",
       "      <td>0.728972</td>\n",
       "      <td>0.814815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1150</td>\n",
       "      <td>0.373600</td>\n",
       "      <td>0.487699</td>\n",
       "      <td>0.831715</td>\n",
       "      <td>0.829104</td>\n",
       "      <td>0.816119</td>\n",
       "      <td>0.881773</td>\n",
       "      <td>0.720430</td>\n",
       "      <td>0.846154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>0.392600</td>\n",
       "      <td>0.698254</td>\n",
       "      <td>0.766990</td>\n",
       "      <td>0.774710</td>\n",
       "      <td>0.796157</td>\n",
       "      <td>0.801153</td>\n",
       "      <td>0.707317</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1250</td>\n",
       "      <td>0.441100</td>\n",
       "      <td>0.482085</td>\n",
       "      <td>0.844660</td>\n",
       "      <td>0.845842</td>\n",
       "      <td>0.842429</td>\n",
       "      <td>0.884910</td>\n",
       "      <td>0.762376</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>0.451900</td>\n",
       "      <td>0.458640</td>\n",
       "      <td>0.841424</td>\n",
       "      <td>0.842601</td>\n",
       "      <td>0.819404</td>\n",
       "      <td>0.887179</td>\n",
       "      <td>0.756219</td>\n",
       "      <td>0.814815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1350</td>\n",
       "      <td>0.359600</td>\n",
       "      <td>0.485957</td>\n",
       "      <td>0.844660</td>\n",
       "      <td>0.844213</td>\n",
       "      <td>0.839932</td>\n",
       "      <td>0.887218</td>\n",
       "      <td>0.752577</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>0.469400</td>\n",
       "      <td>0.504000</td>\n",
       "      <td>0.822006</td>\n",
       "      <td>0.824641</td>\n",
       "      <td>0.772292</td>\n",
       "      <td>0.880829</td>\n",
       "      <td>0.726368</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1450</td>\n",
       "      <td>0.365900</td>\n",
       "      <td>0.486395</td>\n",
       "      <td>0.838188</td>\n",
       "      <td>0.838303</td>\n",
       "      <td>0.815263</td>\n",
       "      <td>0.886076</td>\n",
       "      <td>0.744898</td>\n",
       "      <td>0.814815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.318000</td>\n",
       "      <td>0.489088</td>\n",
       "      <td>0.844660</td>\n",
       "      <td>0.845938</td>\n",
       "      <td>0.853899</td>\n",
       "      <td>0.882653</td>\n",
       "      <td>0.762376</td>\n",
       "      <td>0.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1550</td>\n",
       "      <td>0.390300</td>\n",
       "      <td>0.567186</td>\n",
       "      <td>0.815534</td>\n",
       "      <td>0.820426</td>\n",
       "      <td>0.815639</td>\n",
       "      <td>0.856369</td>\n",
       "      <td>0.744395</td>\n",
       "      <td>0.846154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>0.353700</td>\n",
       "      <td>0.503464</td>\n",
       "      <td>0.838188</td>\n",
       "      <td>0.841387</td>\n",
       "      <td>0.840782</td>\n",
       "      <td>0.875989</td>\n",
       "      <td>0.766355</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1650</td>\n",
       "      <td>0.380000</td>\n",
       "      <td>0.466047</td>\n",
       "      <td>0.851133</td>\n",
       "      <td>0.847752</td>\n",
       "      <td>0.840877</td>\n",
       "      <td>0.895377</td>\n",
       "      <td>0.747253</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>0.416500</td>\n",
       "      <td>0.489680</td>\n",
       "      <td>0.822006</td>\n",
       "      <td>0.825712</td>\n",
       "      <td>0.818354</td>\n",
       "      <td>0.864721</td>\n",
       "      <td>0.744186</td>\n",
       "      <td>0.846154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1750</td>\n",
       "      <td>0.298700</td>\n",
       "      <td>0.478809</td>\n",
       "      <td>0.851133</td>\n",
       "      <td>0.851514</td>\n",
       "      <td>0.846272</td>\n",
       "      <td>0.891139</td>\n",
       "      <td>0.767677</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>0.265400</td>\n",
       "      <td>0.593758</td>\n",
       "      <td>0.838188</td>\n",
       "      <td>0.841105</td>\n",
       "      <td>0.840264</td>\n",
       "      <td>0.876640</td>\n",
       "      <td>0.764151</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1850</td>\n",
       "      <td>0.359400</td>\n",
       "      <td>0.569641</td>\n",
       "      <td>0.831715</td>\n",
       "      <td>0.835527</td>\n",
       "      <td>0.806794</td>\n",
       "      <td>0.877660</td>\n",
       "      <td>0.757009</td>\n",
       "      <td>0.785714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>0.374000</td>\n",
       "      <td>0.473424</td>\n",
       "      <td>0.844660</td>\n",
       "      <td>0.846202</td>\n",
       "      <td>0.843008</td>\n",
       "      <td>0.884319</td>\n",
       "      <td>0.764706</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1950</td>\n",
       "      <td>0.327300</td>\n",
       "      <td>0.458725</td>\n",
       "      <td>0.847896</td>\n",
       "      <td>0.842323</td>\n",
       "      <td>0.835388</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.731429</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.332000</td>\n",
       "      <td>0.428954</td>\n",
       "      <td>0.864078</td>\n",
       "      <td>0.861457</td>\n",
       "      <td>0.852128</td>\n",
       "      <td>0.904645</td>\n",
       "      <td>0.771739</td>\n",
       "      <td>0.880000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-50/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-100/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-150/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-200/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-250/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-300/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350/emotional-reactions/adapter_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-350/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-400/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-450/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-500/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-550/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-600/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650/emotional-reactions/head_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-650/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-700/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-750/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-800/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-850/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-900/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950/emotional-reactions/head_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-950/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1000/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1050/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1100/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1150/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1200/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250/emotional-reactions/head_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1250/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1300/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1350/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1400/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1450/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1550/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1600/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1650/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1700/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1750/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1800/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1850/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900/emotional-reactions/adapter_config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1900/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1950/emotional-reactions/pytorch_model_head.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000/emotional-reactions/pytorch_model_head.bin\n",
      "Configuration saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000/emotional-reactions/head_config.json\n",
      "Module weights saved in ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-2000/emotional-reactions/pytorch_model_head.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500 (score: 0.8538986551716397).\n",
      "Could not locate the best model at ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/pytorch_model.bin, if you are running a distributed training on multiple nodes, you should activate `--save_on_each_node`.\n",
      "Loading best adapter(s) from ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500 (score: 0.8538986551716397).\n",
      "Loading module configuration from ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/adapter_config.json\n",
      "Overwriting existing adapter 'emotional-reactions'.\n",
      "Loading module weights from ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/pytorch_adapter.bin\n",
      "Loading module configuration from ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/head_config.json\n",
      "Overwriting existing head 'emotional-reactions'\n",
      "Adding head 'emotional-reactions' with config {'head_type': 'classification', 'num_labels': 3, 'layers': 2, 'activation_function': 'tanh', 'label2id': {'0': 0, '1': 1, '2': 2}, 'use_pooler': False, 'bias': True}.\n",
      "Loading module weights from ./training_output/epitome-training_emotional-reactions_1536-21-4/checkpoint-1500/emotional-reactions/pytorch_model_head.bin\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=2000, training_loss=0.5272480578422546, metrics={'train_runtime': 174.4239, 'train_samples_per_second': 318.19, 'train_steps_per_second': 39.788, 'total_flos': 4281472923356160.0, 'train_loss': 0.5272480578422546, 'epoch': 5.76})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" train \"\"\"\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f8511f9",
   "metadata": {},
   "source": [
    "# Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e90dbdea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='78' max='39' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [39/39 00:02]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>eval_loss</td>\n",
       "      <td>0.489088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>eval_accuracy</td>\n",
       "      <td>0.844660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eval_f1_weighted</td>\n",
       "      <td>0.845938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>eval_f1_macro</td>\n",
       "      <td>0.853899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>eval_f1_0</td>\n",
       "      <td>0.882653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>eval_f1_1</td>\n",
       "      <td>0.762376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>eval_f1_2</td>\n",
       "      <td>0.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>eval_runtime</td>\n",
       "      <td>1.056800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>eval_samples_per_second</td>\n",
       "      <td>292.384000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>eval_steps_per_second</td>\n",
       "      <td>36.903000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>epoch</td>\n",
       "      <td>5.760000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     metric       value\n",
       "0                 eval_loss    0.489088\n",
       "1             eval_accuracy    0.844660\n",
       "2          eval_f1_weighted    0.845938\n",
       "3             eval_f1_macro    0.853899\n",
       "4                 eval_f1_0    0.882653\n",
       "5                 eval_f1_1    0.762376\n",
       "6                 eval_f1_2    0.916667\n",
       "7              eval_runtime    1.056800\n",
       "8   eval_samples_per_second  292.384000\n",
       "9     eval_steps_per_second   36.903000\n",
       "10                    epoch    5.760000"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" evaluate best checkpoint (loaded at end of training) \"\"\"\n",
    "\n",
    "trainer.model.cuda()\n",
    "eval_output = trainer.evaluate()\n",
    "eval_metric_result = eval_output[metric_for_best_model]\n",
    "pd.DataFrame({'metric':list(eval_output.keys()), 'value': list(eval_output.values())}, columns=['metric', 'value'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2e1cf00f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Prediction *****\n",
      "  Num examples = 309\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgYAAAG3CAYAAADPQZ3tAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA2aElEQVR4nO3deZhcZZmw8fsJELYkBAiDrCKMEBBCxLCIJCQExAU1juACosgoIAiKH4OgIAgqyCLM6MjisOoHOmEwsqhEsoHCEIKEkIDIqogfS4MhwIQteb4/6nRNpdKd7k6q63R33b/rqut0vec95zyns9RT73YiM5EkSQIYVHYAkiSp7zAxkCRJVSYGkiSpysRAkiRVmRhIkqSq1csOYGVFhNMpWtTOO+9cdggqyeqr99v/srQKnnjiCdra2qJZ12vw58stmfm+Bp6v1/mvTP3O9OnTyw5BJdlggw3KDkElGDNmTNkhrIoRZQfQUyYGkiTViWhMA0V/XCvIxECSpDqtnBg4+FCSJFXZYiBJUp1GtRj0RyYGkiTViAgGDWpMg/qSJUsacp5msitBkiRV2WIgSVIduxIkSVJVKycGdiVIkqQqWwwkSarTyi0GJgaSJNVp5cTArgRJklRli4EkSTUioqVbDEwMJEmq06gFjvqj1r1zSZK0HFsMJEmqY1eCJEmqauXEwK4ESZJUZYuBJEk1nJUgSZKW0cqJgV0JkiSpysRAkqQ67d0Jq/rq4TU3j4ipEZG9dFvdYmIgSVKdQYMGNeTVXRFxCHAHsFUX9TaNiB9HxL0RcX9E/CUifhERm9XV2yAiLouIP0XEgxHx24jYqVv33u2oJUlSw0XEesCRwHgqyUFn9d5a7J+Rme/MzJ2ADwL7AZvU1FsduAXYDNgpM7cH7gRmRcRWXcVjYiBJUo1GdSP0oCvhJWB8Zj7WRb0fAFMz85r2gsy8HzgAeLSm3qHAGOCEzHytKDsTWAKc3lUwJgaSJNVpZmKQmUszc2kX8WxGJQH4ZQfHz8zMv9cUHQQ8l5nza+q8AdwGfCwiVlvRtUwMJEnq+94DBLA0Iq4qxhf8qfh5m7q6o4HHOzjHo8AQYOsVXcjEQJKkOg1sMRgREXNqXkesZEhbFtufAr8ARgF7UhlHMLtu7MAIYFEH51hUs79TLnAkSVKdBi5w1JaZYxpwnrWK7Q2ZOaX93BHxReBPwInA0Q24jomBJEn1+uDKhy8V2z/UFmbmwxGxCNitprgNGNrBOYbV7O+UXQmSJPV9DxTbjj6336wrn0vH4wi2Bl4GVjj7wcRAkqQaEdH0BY664Xbg78DOdbFuAWwAzK4pvg7YKCLeUVNvdWAccH1mLlnRhUwMJEmqU8aSyCuSma8CJwGfjIixRYxrAhcCLwBn11S/GpgDnFvUATiFyvCB07q6lmMMJEkqWURcDOxBMfsgIuYWu8Zn5kKAzLw0IhYDP4iItakMSJwN7JGZT7SfKzPfjIj9gfOA+yNiCfDX4lzVep0xMZAkqU6zBx9m5lHdrPcT4CfdqPcCcPjKxGJiIElSnT44K6FpHGMgSZKqbDGQJKlGowcO9jcmBpIk1WnlxMCuBEmSVGWLgSRJdRq8OFG/YmIgSVIduxIkSZKwxUCSpGU4K0GSJC2jlRMDuxIkSVKVLQaSJNVp5RYDEwNJkuq08nTF1r1zSZK0HFsMJEmq4awESZK0jFZODOxKkCRJVbYYSJJUp5UHH5oYSJJUx64ElWqzzTbjlltuITPLDkWS1OJsMSjZwQcfzFlnncWrr77a4f69996b66+/nr/85S/L7Rs5ciSXXnopX/7ylwEYN24chx12GLvssgtLly5lyJAhPPzww5xxxhncddddvXofWnUPPfQQV111FTNnziQiWLJkCSNHjuRrX/sa22+/fbXeMcccw1133cW66667zPF77bUX3/nOd5odtnrR4sWLOfXUU7nxxhtZY401GDZsGGeddRZ777132aENaBFhV4LKMWzYMI488kjGjx/PN7/5TbbddtsO691www187nOfW6Zs44035sknn+SnP/1ptezYY49l+PDh7L333rz44ousueaaXHHFFdx6663ssssuPPzww716P1o1xx57LGuuuSY33XQTG2ywAa+88gpHHnkk++23H7/+9a/ZaaedqnUvvPBC9tprrxKjVTMcfPDBPPHEE9x1110MHz6cyy+/nP3335/bbruN3XbbrezwBjS7EkoQEWtHxHkR8VBEzI+IOyKipdLgl156ifHjx/P44493WufBBx/kkksuWa788MMPZ968edx9993VsgULFvCNb3yDF198EYDXXnuNc845hyFDhvC+972v8Teghvv617/OBhtsAMC6667LGWecweLFi7n88stLjkzNNmPGDKZMmcKZZ57J8OHDgcq/+5EjR3LiiSeWG5wGtDJbDK4BtgJ2z8yFEXE4cEtEjMvM2SXG1TTdGVPw7LPP8uyzzy5TFhF84QtfWK7Z+PTTT1/u+KFDhwLw3HPPrXygaoqbbrqJwYMHL1O2ySabALBw4cISIlKZJk+eTEQwfvz4ZconTpzIBRdcwDPPPMPGG29cTnAtwBaDJouICcAk4NTMXAiQmZcDfwTOKSOm/uS9730v66+/Ptdee+0K640cOZLzzjuPX/3qV0yePLlJ0Wll1ScFAI888gjAct0GkydP5kMf+hB77rknEydO5Lzzzut0nIr6p7lz57LRRhsxZMiQZcq32WYbMpN58+aVFNnA1z7GoBGv/qisqA8CEphZVz4NGBcRpsErcOSRR3L11VfzP//zPx3u/9CHPsSTTz7JggULWLBgAQcffDBLlixpcpRqhMsvv5ztttuOQw45pFo2ZMgQNtxwQ372s59xxx13cPbZZ3PllVdy4IEH8uabb5YYrRqpra2NYcOGLVfeXtbW1tbskNQiykoMRgPPZebLdeWPAgGManpE/cQmm2zCAQccwMUXX9xpnRtvvJEtttiCzTbbjKFDh3L//fcvM6pd/cMtt9zCzTffzBVXXMFaa61VLf/e977HN7/5zeqshF133ZWTTz6ZO++8k1/+8pdlhSsNKO3PS1jVV39UVmIwAljUQfmimv3LiYgjImJORMzptcj6uH/+53/m97//PQ8++GCXdZ9++mk+85nPMHToUC644IImRKdGufPOO/nqV7/Kz372M7bbbrsu67/rXe8CYPbslhie0xJGjBjBSy+9tFz5okWLqvvVe+xK6Ccy89LMHJOZY8qOpQwRwec//3kuuuiiDvevvfbay5UtXryYhx9+mDFjWvJX1i/dfvvtHHXUUVx77bXssssuy+xbsmQJf//735c7ZrXVVqvu18AwevRonn32WV555ZVlyh977DEiglGjbFhV7ygrMWgDhnZQPqxmv+q8//3vZ8011+T666/vcH9bW9tyGeqgQYPYfPPN7Y/sJ2bMmMGxxx7Ltddeu8x//O1jDJ566in233//5Y677777ANh5552bE6h63YEHHkhmMnPmzGXKp02bxtixY52R0MvsSmi+ucA/RMS6deVbUxmU6HDbDhx55JFcdtllnQ4wW2eddTj77LNZY401AFhjjTU455xz2GSTTbjwwgubGKlWxtSpU/n0pz/NgQceyPz58/nP//zP6mvBggXVeo8++ugy6xo88sgjnHXWWWy77bYceOCBZYSuXrDPPvswadIkTj311OraJFdeeSUPPPAA5557bsnRDWxlzUqIiM0jYmpEdDmXPSK2j4g3ImLmyt5nZ8pax+A64IvAeODmmvKJwO2Z+UwZQZXhoosuYo899mDLLbcE4N577wVg/Pjx1f8MoPI8hf33359jjz2203N96lOf4lOf+hR/+MMfWLJkCWuttRZ/+ctfOOCAA7j55ps7PU59w0knncSrr77a4XiQLbbYAoC3vOUtXHDBBUyZMoUrrriC119/nddee4399tuPk08+ucPuJPVf11xzDaeccgq77rorgwcPZujQoUydOtVVDwegiDgEOAvo7rzjC1jBZ3hEbACcC4wFlgB/Bb6amfd3GUtZD+6JiF8AbwUmZOaLEXEYcAkwtjsLHHUno9LA9Pzzz5cdgkrSviqkWsuYMWOYM2dO09rlhwwZko0aw3HnnXfe09W4uIhYD7gROAz4JvDZzOz0fiPig8C3gQ2BxzJzfN3+1YE7geeBj2TmaxFxBvAlYJfMfGJF8ZQ5+PBgYAZwd0TMB44E3tsqqx5KkvquJo8xeAkYn5mPdSOuNYDzgeOBpZ1UOxQYA5yQma8VZWdSaTk4vatrlJYYZObizPw/mbltZu6Yme/OzFllxSNJUhkyc2lmdvYhX+8Y4IHMnLmCOgdRWStofs013gBuAz4WEaut6AI+XVGSpBp99bHLETECOAl4TxdVRwMdPZ3vUWAIlYH+nT5ut+/duSRJJWtgV8KI9oX5itcRqxDWGcDVmfloF/VWahHBdrYYSJLUe9oasShfROwIfATo9fXtTQwkSarTB7sSLgC+lZkdtQTUW6VFBE0MJEmq05dWLYyIYcBOwMYRcXTNrk2pdFXMLd7vlpmvU1lEsKNWiq2Bl4EVzn4wMZAkqQ8rWgneUl8eEU8AT9SvY0BlEcH3R8Q7MnNBUXd1YBxwfWau8KEqfa6tRJKkMjVq4GGJrQ5XA3OAcyNizaLsFCqNAad1dbAtBpIk1Wn2GIOIuBjYA9iyeD+32DU+MxfW1b0aGMWyXQm/zcx/AcjMNyNif+A84P6IaF8SeXxXqx6CiYEkSaXLzKN6UPcz3ajzAnD4ysRiYiBJUp2+NPiw2UwMJEmq0VdXPmyW1r1zSZK0HFsMJEmqY1eCJEmqMjGQJElVjjGQJEnCFgNJkpZR8qqFpTMxkCSpjl0JkiRJ2GIgSdJy7EqQJEmAKx+27p1LkqTl2GIgSVIduxIkSVJVKycGdiVIkqQqWwwkSarTyi0GJgaSJNVo9ZUP7UqQJElVthhIklSnlVsMTAwkSarTyomBXQmSJKnKFgNJkuq0couBiYEkSXVaOTGwK0GSJFXZYiBJUo1Wf7qiiYEkSXXsSpAkScIWA0mSltPKLQYmBpIk1WnlxMCuBEmS+oCI2DwipkZElhmHiYEkSTXan67YiFcPrnkIcAewVSf7h0bEFyPi9xHxQPGaHRGfiQ4uFBEbRMRlEfGniHgwIn4bETt1JxYTA0mS6jQzMYiI9YAjgfFUkoOOHAKcD5ycmTtk5g7AvwFXAd+tO9/qwC3AZsBOmbk9cCcwKyK26ioeEwNJksr1EjA+Mx/rot61mXlb+5vM/Cnwe+C4IhlodygwBjghM18rys4ElgCndxWMiYEkSXWa2WKQmUszc2kX1X4MHNFB+VPAOsBaNWUHAc9l5vyaa7wB3AZ8LCJWW9GFnJUgSVKdvjYrITOXdLJrO+DuzHy5pmw08HgHdR8FhgBbAw93di1bDCRJ6j0jImJOzaujb/0rJSL2BHYCTqy/JrCog0MW1ezvlC0GkiTVaWCLQVtmjmnUydpFxPrAFcDXM3NmI89tYiBJUo2eTjVstogYCtwMTM7M73VQpQ0Y2kH5sJr9nbIrQZKkfqKY2jgV+E1mntJJtblUxhHU2xp4GVjh7AcTA0mS6jR7gaNuxrQ+cCvwy8w8o6b8tIh4V03V64CNIuIdNXVWB8YB169gICNgV4IkScvpa10JETEC+C2VNQ/+GhGfrtk9EZhV8/5q4IvAuRHx0WItg1OofOaf1tW1TAwkSSpZRFwM7AFsWbyfW+wan5kLgaOpTEMEGLuic2XmmxGxP3AecH9ELAH+Wpzria5iMTGQJKlOs1sMMvOoLvafAZyxojp19V8ADl+ZWEwMJEmq09e6EprJwYeSJKmq37YYjB49mpkzZ5Ydhkrw61//uuwQVJJPfvKTZYegFtDX1zHobf02MZAkqbe0cmJgV4IkSaqyxUCSpDqt3GJgYiBJUp1WTgzsSpAkSVW2GEiSVKeVWwxMDCRJqtHq0xXtSpAkSVW2GEiSVKeVWwxMDCRJqtPKiYFdCZIkqcoWA0mS6rRyi4GJgSRJdUwMJEkS4HRFxxhIkqQqWwwkSarTyi0GJgaSJNUZNKh1G9Rb984lSdJybDGQJKmOXQmSJAlwVoJdCZIkqcoWA0mS6thi0AARsWujziVJUpnauxNW9dUfNbIr4ZIGnkuSJJWg066EiHish+fadBVjkSSpT+iv3/YbYUVjDNYDbujmeQI4YNXDkSSpfCYGHftLZn6uuyeKiHsbEI8kSSrRisYY7NbDc/W0viRJfU6jBh72tNUhIjaPiKkRkb10a93SaWKQmW90VB4Re0XEqRFxds37dTurL0lSf9PsxCAiDgHuALbqot4/RcQfImJeRPwxIr4REat1UG+DiLgsIv4UEQ9GxG8jYqfuxNLtWQkRsW5E/AaYBXwL+Eyx65+A+yNiy+6eS5IkVUTEesCRwHgqyUFn9SYBPwf+JTNHAfsBRwPn1dVbHbgF2AzYKTO3B+4EZkXEVl3F05Ppit8F/gH4OLAD8BxAZn4V+DZwVg/OJUlSn9XkFoOXgPGZ2elswKic7ALgl5k5DSAznwTOBY6NiG1qqh8KjAFOyMzXirIzgSXA6V0F05PE4IPAPpn5X5n5x+ICFMFdDozswbkkSeqzmpkYZObSzFzaRbVdqXQzTK8rnwasBnyspuwg4LnMnF9zjTeA24CPddT1UKsnicEbmblwBfuH9OBckiSp+0YX28fryh8ttjvX1a2v1153CLD1ii7Uk8TgzYgY39GOiBgHLO7BuSRJ6pMigkGDBjXkBYyIiDk1ryNWMqwRxXZRbWFm/g+VFvwRdXWXqVd37IgO9lX15CFK3wd+GxHXUxkcMTwiPg+8k8pAxKN6cC5JkvqsBi5w1JaZYxp1smbodmKQmVdExPpUZiQcVBRfCrwMfCMz/28vxCdJkqCt2A6tLYyIdaiMMWirq7tMvcKwunN1qEePXc7M70fEj4E9gQ2Lk9+RmS/35DySJPVlfXBJ5LnFtn58QPv7++rqdtRKsTWVL/MrfBZSjxIDgMx8icr8SEmSBqQ+mBjcDfwZmAD8qKZ8IrAUuL6m7Drg/RHxjsxcANW1DcYB12fmElagR49djojhEfGtYsnGBcX29IgY3pPzSJKk7svMBI4HJkXEBKgsoQz8C/BvmflITfWrgTnAuRGxZlF2CpXGgNO6ula3WwwiYjSV+ZLrAwuBv1OZV7kv8KWI2Ccz53X3fJIk9VXNbjGIiIuBPYAti/dzi13j25cKyMxfRMQngO8XaxEMBi6mboHBzHwzIvansiLi/RGxBPhrca4nuoqlJ10JPwRmAF/LzPZ5k0TE24GzgX8HxvbgfJIk9Tkr8wCkVZWZ3ZrZl5nXs2y3QWf1XgAOX5lYepIYvB0YV786U2Y+XGQwT61MAJIkqe/oSWLwWGdLNhbNFk82KCZJkkrVBwcfNk1PBh9eHxFHdrSjKP9NY0KSJKlczX7scl/SaYtBRFzeQfEHIuJ4YD6VpRXXA95B5dGO/9UrEUqSpKZZUVfCIcDf6soWA2sC76orbwM+CRzWsMgkSSpJf/223wgrSgweyMx3dvdEEXFvA+KRJKl0rZwYrGiMwdE9PFdP60uSpD6m0xaDzLyzh+c6DujpMZIk9Sntj11uVT16VkLxFKcPANtQGWtQa+9GBSVJUplauSuhJ0sivx24FdgCSKD+t5YNjEuSJJWgJ20l5wA/ANYB5mXmoMwcBGwKXAIc0wvxSZLUdK5j0D1bZ+ZHASKi2jqQmU9HxNFUWhMubnB8kiQ1XX/9UG+EnrQYvFbz8+rFs52B6uMgt2xYVJIkqRQ9SQwyInYqfv4TcF5EDI+I9SLiO8CbjQ9PkqTmalQ3Qn9tdehJYvBLYFZEbEtlvMGRwPPAC8BJwPcaH566cvXVV7PFFltw1llndV1ZktQtrZwYdHuMQWZ+F/hu+/uI2A34FLAGcHNmzmx4dC3ooYce4oorrmDmzJlEBEuWLGHkyJGcdNJJ7LDDDtV6zz77LMcddxx/+9vfeOmll0qMWI1y0UUX8dBDD7HWWmstU/7aa6/x9NNP893vfpfFixfz/e9/nxEjRix3/N/+9jf22WcfDjvssCZFrN62ePFiTj31VG688UbWWGMNhg0bxllnncXeezs7XL2nR+sY1MrM+4H7299HRBRjDXokIjYHLgf2y8z+mV410DHHHMPgwYP51a9+xQYbbMArr7zCF77wBSZOnMgtt9zCqFGjALjkkkuYMGEC+++/P6NHjy43aDXMEUccsUwCCHDTTTcxc+ZM3va2t/HAAw/wrne9iy9+8YvL1Fm4cCFf+tKX2GuvvZoZrnrZwQcfzBNPPMFdd93F8OHDufzyy9l///257bbb2G233coOb0Drr9/2G6GRSzvd09MDIuIQ4A5gqwbG0e+dcsopbLDBBgCsu+66fPvb32bx4sVcdtll1Tonn3wyRx55ZEv/5R1oJk6cyKabbrpc+fTp09l3330B2HTTTZk4ceJydWbOnMmWW27JP/7jP/Z6nGqOGTNmMGXKFM4880yGDx8OwOGHH87IkSM58cQTyw2uBdiV0IFOHru8Ij2alRAR61EZpzAe+Cbw9h5eb0D61a9+xeDBg5cp22STTYDKt8J2q6++0o096qO23Xbb5coWLFjACy+8wLhx4wAYPnx49UOi3dKlS5k+fTqTJk1qQpRqlsmTJxMRjB8/fpnyiRMncsEFF/DMM8+w8cYblxOcBrSePnZ5RYb08NovAeMzc2l/zap6Q31SAPDwww8DMHbs2GaHo5JNmzaNPffck3XWWafTOvPmzeOVV15hzz33bGJk6m1z585lo402YsiQZf9r3WabbchM5s2bx3777VdSdANfK38ulfbY5cxc2pP6reyyyy5j5MiRfPrTny47FDXRokWLuPvuuznjjDNWWG/atGmMHTt2uUGL6t/a2toYNmzYcuXtZW1tbc0OqWX0526ARvCxy33cb37zG2666Sauuuoq/+NvMTNnzuStb30rb3vb2zqt88ILL3DvvfdWxyBI0qrqNDHo6WOXV+IxzT0WEUdExJyImPP888/39uVKd8cdd3D88cczefJktttuu7LDURNlJtOnT++yqXjmzJlsu+22bL755k2KTM0yYsSIDqciL1q0qLpfvWfQoEENefVH/SrqzLw0M8dk5pgNN9yw7HB61W233cYRRxzBz372M3bZZZeyw1GTzZ8/n5dffpl3v/vdndZZunQpM2bMsJ95gBo9ejTPPvssr7zyyjLljz32GBFRnbqs3tHKsxL6VWLQKqZPn84xxxzDz3/+c3beeedq+Sc/+ckSo1IzTZs2jb333rvDwajt5s6dyxtvvMGuu+7axMjULAceeCCZycyZM5cpbx9T4owE9RbnvPUxt9xyC5/97Gc55phjmD9/PvPnz6/uW7BgQYmRqVkWLlzIPffcwznnnLPCetOmTWPChAlOXR2g9tlnHyZNmsSpp57KXnvtxXrrrceVV17JAw88wO233152eANef/223wj+j9LHnHjiibz66qucf/75y+3bYostqj/PmzePo48+mjfeeAOAK664gptvvpmPfexjHH/88U2LV403c+ZMRo4cWV2/oiPPP/888+bNc/njAe6aa67hlFNOYdddd2Xw4MEMHTqUqVOnuuphL+vP3QCNsFKJQUSsBqyfmc6XabD77ruvW/VGjRrF7373u16ORmWYNGlSl4sVbbjhhvzkJz9pTkAqzdprr83555/f4RcF9a7+OnCwEXp05xGxV0TcCrwM3FeU/XtErNRUxYi4OCLmAh8u3s8tXsNX5nySJGnVdLvFICImAr8GHgKmAO0jnq4GfhARr2Zmj5ZRzsyjelJfkqRmaOWuhJ60GJwBHJGZO2Xmp4BFAJl5F3AAleceSJLU7zldsXtGZOaVHe3IzGeBNRoSkSRJLSQiRkfELyPijxFxf0TMi4gTImL1unrjIuKOiJgfEQ9FxPkRsXaj4+nJ4MM1I2KNzHyjfkdErAUM7BWHJEktoZnf9iNiS2AGMBXYOTNfi4g9gJnARsDXinq7FXWOzMyrirF4s4CtgY82MqaetBj8HpgSEdvXFkbExlTGGcxoZGCSJJWliV0JHwSGA9/LzNcAMvO/qSQBh9bUOxeYn5lXFXUWAqcBkyJifKPuG3qWGJwIvAOYHxEvACMj4ingKWAX4KRGBiZJUgtYUmzrW/DXaC+LiLcAY4HpdXWmAwl8vJEBdTsxyMyngHcC36UyM+GvwJ+BbwNjMvPpRgYmSVJZmvgQpWuBBcDp7VP1I+KDwL7AhUWdUUAAj9cemJmLgDZgZxqoRwscZebfgVOLlyRJA06DxxiMiIg5Ne8vzcxL299k5ksRsQ/wH8BzEfE8MBg4KjMvaz9HsV3UwfkX1exviIYtiRwRZ2amCYMkSf+rLTPHdLYzIt4O3ArcDmyYmYuKRGFyRAzLzAuaFWi7nixwNK6LKp/AlgRJ0gDQxDUIvg28hUoLwcsAmTk9In4MnBsRN1LpLgAY2sHxw4CHGxlQT1oMZlIZ5CBJ0oDWxMRgFPD/2pOCGg8Bq1EZ23c7lc/frWsrRMRQKt0I1zUyoJ4kBo8Cn68rGwJsD3wE+H6jgpIkqUU8A+weEWu2T1csbFVs2zLz6Yj4HTCh7th9qAxKnNzIgHoyXfG8zJxV97o5M88DPgb8UyMDkySpLE2clfCvwFrAOcWTi4mIHYGjgXuptBYAnACMiohDizrDgW8BUzKzoesI9WS64iUr2PcsleYQSZL6tUYtbtSd7ojM/AWVqYk7UFkn6H7gP6nMUpiYmW8W9WYD+wFfjIj5wGwq6xgc3Oj7X+VZCRGxPnAQlW4FSZLUA5k5DZjWjXq3AXv2djw9mZWwlM4HHybwpYZEJElSyfrrkxEboSctBs8AF9eVLSnKb8vMhxoWlSRJJTIx6J7fZea3ei0SSZJUup7MSlg9Iq6PiM17LRpJkvqAJj5dsc/pSYvBB4DPUOk6kCRpQIqI7k41HJB6cuf3ZebPM/ONjnZGxLoNikmSJJWkJ4nB7IhY0VoFt69gnyRJ/YZdCd2zAPh5RNwKPAjUr+u8QcOikiSpRP31Q70RepIY/Hux3a6T/T5gSZKkfq4nicGDVAYgdiSAm1c9HEmSymeLQfdcnJl/7mxnRJzdgHgkSSpVq89KWGFiEBHTix9nZ+ZJK6qbmT9pWFSSJKkUXbUYbAF8HnihCbFIktQn2JXQuZczc1ZTIpEkqY9o5cSgq06Ubs80iIgrVy0USZJUtq5aDIZGxFgqsw660uvPiJYkqRlaucWgq8RgG2BmN84TuI6BJGmAMDHo3JPAN7txngBOX+VoJElSqbpKDJ7PzKu6c6KI+GQD4pEkqVSuY9Agmfm+Rp1LkqQytXJXQlcp0eCI2CIiNmpKNJIkqVRdtRgMAWYVr8/1fjiSJJWvlVsMVpgYZOZWTYpDkqQ+o5UTg9YdXSFJkpbTsMGHkiQNBM5KkCRJy7ArQZIkCRMDSZJUw64ESZLq2JUgSZKELQaSJC3HFgNJkgRUkoJGvXpwzfdFxLSImBMRj0TEgog4va7OuIi4IyLmR8RDEXF+RKzd6Ps3MZAkqUQR8QXgPOALmTkGeDswGTiwps5uwFTgkszcEdgd2Be4ptHx2JUgSVKdZnUlRMTmwL8C+2bmYwCZmRFxHvD7mqrnAvMz86qizsKIOA34RUSMz8yZjYrJFgNJkuo0sSvhM8DLmXlHbWFmvpyZvy1ieQswFphed+x0IIGPr/od/y8TA0mSyrMX8HhETIqIWRHxYETcExEnR8QaRZ1RQACP1x6YmYuANmDnRgZkV4IkSXUa2JUwIiLm1Ly/NDMvrXm/JbAVcArw0cx8MiLGA1OAMcDHgBFF3UUdnH9Rzf6GMDGQJKlOAxODtmJAYWfWAtYF/k9mPgmQmTMj4t+BrxeDDpvKrgRJksrzUrH9Q115eyvDblS6CwCGdnD8sJr9DWFiIElSeR4otvWfx2/WlM+jMshw69oKETGUSjfCfY0MyMRAkqQaTV7g6JfFtn4AYfv72Zn5NPA7YEJdnX2oDEqcvLL32pF+O8ZgtdVWY7311is7DJXgE5/4RNkhqCSvvvpq2SGoBJnZ9Gs2cUnk66h86H83Ij6YmS9GxA7AscAvMvO/i3onALdHxKGZ+ZOIGA58C5iSmTMaGZAtBpIklSQzlwIfBO4F5kbEH6nMSPgR8ImaerOB/YAvRsR8YDaVdQwObnRM/bbFQJKk3tLMhygV6xEcW7xWVO82YM/ejsfEQJKkOj5dUZIkCVsMJElaTiu3GJgYSJJUowdTDQckuxIkSVKVLQaSJNWxxUCSJAkTA0mSVMOuBEmS6rRyV4KJgSRJdVo5MbArQZIkVdliIElSnVZuMTAxkCSphgscSZIkFWwxkCSpTiu3GJgYSJJUp5UTA7sSJElSlS0GkiTVaeUWAxMDSZLqtHJiYFeCJEmqMjGQJElVdiVIklTDBY4kSZIKthhIklSnlVsMTAwkSarTyomBXQmSJKnKFgNJkuq0couBiYEkSXVaOTGwK0GSJFXZYiBJUg3XMZAkSX1GRHwnIjIiDivj+iYGkiT1ERGxFfDVFewfFxF3RMT8iHgoIs6PiLUbGYOJgSRJddq7E1b1tRLOBX7bSUy7AVOBSzJzR2B3YF/gmpW9z46YGEiSVKeMxCAixgGjgB90UuVcYH5mXgWQmQuB04BJETF+Ze+1nomBJEkli4hBwIXAvwBvdLD/LcBYYHrdrulAAh9vVCwmBpIkle9zwPOZeUMn+0cBATxeW5iZi4A2YOdGBeJ0RUmS6jRwuuKIiJhT8/7SzLy07lpDgTOA963oPMV2UQf7FtXsX2UmBpIk9Z62zBzTRZ1TgBsz8/5mBNQVEwNJkuo0a4GjiNgaOAzYsYuqbcV2aAf7hgEPNyomEwNJkmo0eeXDicCrwG9rrjmk2J4REV8BfkNlYGICW9ceXHRDjACua1RAJgaSJJUkM38M/Li2rJh6OAP4ZmZeWVP+O2BC3Sn2oTIocXKjYnJWgiRJ/cMJwKiIOBQgIoYD3wKmZOaMRl3ExECSpDolLXC0VUTMBf6jKDojIuZGxFiAzJwN7Ad8MSLmA7OprGNwcMNuHLsSJEnqEzLzCWB0F3VuA/bszThMDCRJquNjlyVJkjAxkCRJNexKkCSpTit3JZgYSJJUx8RAkiQBTV/5sM9xjIEkSaoyMZAkSVV2JUiSVMeuBElS6f76179ywAEHsPbaa3daZ8mSJZx77rmst956/OQnP2lidGoVJgb9xOLFiznhhBPYbrvt2HHHHdlzzz2ZNWtW2WGplz344IN85StfYccdd2SnnXZihx124KCDDmL+/Pllh6YGu/baa5kwYQJ//vOfO63z6KOPMnHiRK6//npef/31JkbXesp4VkJfYWLQTxx88MFMmzaNu+66i/nz5/P5z3+e/fffn9mzZ5cdmnrR4Ycfzty5c5k1axb3338/d999N2+++Sa77747c+fOLTs8NciLL77IZZddxtSpU9ljjz06rXfuuedyzDHH8L3vfa+J0bUmE4Mmi4jtI+LCiFgQEfMj4sGIuC4idiwjnr5uxowZTJkyhTPPPJPhw4cDlQ+MkSNHcuKJJ5YbnHrdmWeeyYYbbgjAuuuuy7nnnsvixYv50Y9+VHJkapShQ4cydepU3va2t62w3g9/+EMOOuigJkWlVlVWi8EVwDuBcZm5IzCGykDI2RExuqSY+qzJkycTEYwfP36Z8okTJ3LbbbfxzDPPlBOYet2sWbMYO3bsMmWbbbYZAAsXLiwhIvWGQYMGMWhQ1/8dr76648XV+8rsSjglM58HyMxXgBOAtYGjS4ypT5o7dy4bbbQRQ4YMWaZ8m222ITOZN29eSZGptw0ePHi5soceeghguURRUuO0cldCWennuMysHznzVLFdv9nB9HVtbW0MGzZsufL2sra2tmaHpBL96Ec/YocdduDwww8vOxRJA1ApiUEHSQHAdsV2RmfHRcQRwBEAW265ZS9EJvVtN910E1OmTGHWrFmstdZaZYcjDUj9+dt+I/SlWQlHAw8Al3dWITMvzcwxmTlmo402al5kJRsxYgQvvfTScuWLFi2q7tfAd/vtt3PUUUdx8803s/3225cdjqQBqk8kBhFxAPBR4MDMfLXsePqa0aNH8+yzz/LKK68sU/7YY48REYwaNaqkyNQsM2bM4NBDD+XGG29k1113LTscSQNY6YlBRIwFLgE+kJkPlh1PX3TggQeSmcycOXOZ8mnTpjF27Fg23njjcgJTU0ydOpXDDz+cG2+8kXe+853V8o985CMlRiUNbA4+LElETACuBA7IzHvLjKUv22effZg0aRKnnnoqe+21F+uttx5XXnklDzzwALfffnvZ4akX3XzzzRx00EEcf/zx3Hfffdx3333Vfc5GkXpPf/1Qb4TSEoOIeC9wKZWk4P6a8hsy88NlxdVXXXPNNZxyyinsuuuuDB48uLogym677VZ2aOpFxx13HK+++ipnnXXWcvve+ta3lhCResuxxx7L7NmzefLJJwHYfffdAbjllluqC5v95je/4bTTTuPll18G4IwzzuCHP/whxx13HIccckgpcWvgicxs/kUjPghcB3wfqO8++HZmbtXVOcaMGZNz5szphejU15Xxd1Z9w2uvvVZ2CCrBe97zHu65556mfYUfPXp0Tps2rSHnGjFixD2ZOaYhJ2uSsloMfgCsBXy9g32dP0FEkiT1qrLWMdi6jOtKkqQVc+FtSZLqOPhQkiQBrnxY+joGkiSp7zAxkCRJVXYlSJJUx64ESZIkTAwkSVpOs56VEBHbR8SFEbEgIuZHxIMRcV1E7NhB3XERcUdR76GIOD8i1m70vZsYSJJUniuAdwLjMnNHYAyVbv7ZETG6vVJE7AZMBS4p6u0O7Atc0+iATAwkSSrXKZn5PEBmvgKcAKwNHF1T51xgfmZeVdRbCJwGTIqI8Y0MxsRAkqQ6TXzs8rjMrH9M7lPFdv0ilrcAY4HpdfWmAwl8fBVudTkmBpIklSQzX++geLtiO6PYjgICeLzu2EVAG7BzI2NyuqIkSb1nRETUPgr40sy8tItjjgYeAC5vP0exXdRB3UU1+xvCxECSpBoNXhK5rSePXY6IA4CPUulieLVRQfSEXQmSJPUBETEWuAT4QGY+WLOrrdgO7eCwYTX7G8LEQJKkkkXEBOCnwAGZeXfd7nlUBhluXXfMUCrdCPc1MhYTA0mS6jRxVgIR8V4q6xkckJn31pTfAJCZTwO/AybUHboPlUGJkxtxz+0cYyBJUkki4oPAdcD3gZ0jonaGwaian08Abo+IQzPzJxExHPgWMCUzZ9BAJgaSJJXnB8BawNc72Pfn9h8yc3ZE7AecHRFfAwYDNwHfaHRAJgaSJNVp1tMVM3PrrmtV694G7NmL4QCOMZAkSTVMDCRJUpVdCZIk1WlWV0JfZIuBJEmqMjGQJElVdiVIklTHrgRJkiRMDCRJUg27EiRJqtHgxy73O7YYSJKkKhMDSZJUZWIgSZKqHGMgSVIdxxhIkiRhYiBJkmrYlSBJUh27EiRJkjAxkCRJNexKkCSpjl0JkiRJmBhIkqQadiVIklTHrgRJkiRMDCRJUg27EiRJqhERdiVIkiSBiYEkSaphV4IkSXXsSpAkScLEQJIk1bArQZKkOnYlSJKkUkTE2hFxXkQ8FBHzI+KOiNi7rHhMDCRJKtc1wERg98zcEfgP4JaI2K2MYEwMJEmq077I0aq+unGdCcAk4NTMXAiQmZcDfwTO6cVb7JSJgSRJ5TkISGBmXfk0YFxEbNzsgEwMJEkqz2jgucx8ua78USCAUc0OqN/OSrjnnnvaIuLPZcdRkhFAW9lBqBT+2beuVv6zf2szL3bPPffcEhEjGnS6tSJiTs37SzPz0pr3I4BFHRy3qGZ/U/XbxCAzNyo7hrJExJzMHFN2HGo+/+xbl3/2zZOZ7ys7hjLZlSBJUnnagKEdlA+r2d9UJgaSJJVnLvAPEbFuXfnWVAYlzmt2QCYG/dOlXVfRAOWffevyz35guo7KIMPxdeUTgdsz85lmBxSZ2exrSpKkQkT8gsoAywmZ+WJEHAZcAozNzNnNjqffDj6UJGmAOBj4NnB3RLwOvAS8t4ykAGwxkCRJNRxjIPUTEbF5REyNCLN5Sb3GxKCf6GtP31JzRcQhwB3AViWHoiaJiO0j4sKIWFD8m38wIq6LiB3Ljk0Dm4lB/9Gnnr6l5omI9YAjqYxavqPcaNREVwDvBMYV/+bHUBkXNjsiRpcZmAY2E4N+oC8+fUtN9RIwPjMfKzsQNd0pmfk8QGa+ApwArA0cXWpUGtBMDPqHPvf0LTVPZi7NzKVlx6GmG5eZt9eVPVVs1292MGodJgb9w2j62NO3JPWuzHy9g+Ltiu2MZsai1mJi0D/0uadvSSrF0cADwOVlB6KBywWOJKkfiIgDgI9S6WJ4tex4NHDZYtA/9Lmnb0lqnogYS2WJ3A9k5oNlx6OBzcSgf5hLH3v6lqTmKGYl/RQ4IDPvLjseDXwmBv1Dn3v6lqTeFxHvpbKewQGZeW9N+Q3lRaWBzjEG/UBmTo+IKcCZEfG7mqdv7QCMLTU4Sb0iIj5I5UvB94GdI2Lnmt3ORFKv8SFK/URErE3l6VsfAtqfvnVSZs4qNTA1RURcDOwBbEllDvt9xa7x7YteaWCJiMeAt3Wy+8+ZuVUTw1ELMTGQJElVjjGQJElVJgaSJKnKxECSJFWZGEiSpCoTA0mSVGViIEmSqkwMJElSlYmBtJIi4oyIeCQiMiLGlx1PRyLiwxExt4jx9C7q3hoRT0dEjxc3WZVjOznf2X39dysNVCYG6nciYoviw+6F4oNjbvF6JCL+WHyodPQ0yobKzG8Cn+8kvuci4ks9PWdETIqIrzQiPoDMvCEzR3ez7r7AxSt5nZU+tpPznUQHv1tJvc/EQP1OZj5ZfNjdULwfXbz+ETgSOB64JSLK+vv9OvBn4IWVOHYS8JVGBiNJPWFioAGleHbEfwHvBt5TUgzPZOaYzLymjOtL0qowMdBA9GSx3TwiDqzpYz8zIr4dEXdGxCsRMbf9gIg4ICLujoiHI+KJiLgqIv6h9qQRsXZEXBQRbRExLyJ+DmxUV2dMcb3XI+LKun3DIuIHxfnnRcT9EXFJROxU7J8BfBjYtKZ75KSa43eIiBuL4x+LiN9GxLvqbz4ivhwRf46IP0XE9IgYsyq/zIiYEBE3RMS9RUxzIuKQFdTfNSJmFNf/S0R8uYM63boXSc1nYqCB6O3F9rHMvK6mj/1zwMzMfDewb3vliDiQSrfEDzPz7cBIYGNgWkSsWXPei4GPAntm5ijg68WrKjPnFNf7W215RAwGfgvsDryzOP69wATgn4tjJxRx/K2me+Ts4vhtgN8DzwJbZ+bWxftZEbFtzXU+B1wAfDkzt6XyNM4TevLL68AngAXALsW9HQr8a0R8pJP6/wc4oLj+V4ALi7jaY+zWvUgqh4mBBoyIGBQRHwc+Avw6M++qqzIvM28tfr4L+FhEBHA+8IfMvAogM18FTgF2pPKhSPGB9Wngksz8U1HvUeC6boZ3CLAbcEpm/r04/v9R+RB/oxvHnw6sA/xLZi4tyr4LvAac1H7/Rb3bM3NKcY1XgB90M8bOfBc4I4tHsWbmg8CtVMZzdOTfiuuSmdcDvwNOL37X3boXSeVZvewApFVV0yWwDvAMlW+s/95B1Qfafyg+kB6NiJHAlsCUurrzi+0E4Goq3/QHAXM6qdeV9xbb2bWFmXlRN4/fD/hjZlYHNGbm6xHxcBEjwGZU7qU+WelujJ15icoH+z7AmsDS4jrPdVK//npzgL2K+P5K9+5FUklMDNTvdXc6HvByB2UbFtuDImLvun1PA2sVP29abBfW1Xmxm9ceASzNzPrju2tDYJ3acRGF9Wt+7jDGzHzxf7+s90zxLf8mKvHvn5lPFOVXAuM7OiYzF9UV/b3YticG3bkXSSUxMVCrayu2V2XmySuo1z5moP7Da3gPrjMoItbLzO4mE/XHP1GMj+hMhzFGxPCVuF67twN7Aie0JwVd6eAeNyi2TxXb7tyLpJI4xkCt7k9U1hwYXb8jIk4uBiZCZUzCUqB+hP87unmdqcV217prHB4R59cUvQFEsW/diPhwzfHbRcRadcd/KP53RcOngL+sQowdGVxsl9aVv2UFx+xY937XIq72xKA79yKpJCYGamnFgLrjgf2KgYsARMT7gOOAu4t6fwJ+ChwREW8v6mwDfLabl/q/VMYXfLv9G3xEbAWcBtxSU+9xYEQxG2JP4MKi/PRi+72IWK04futi/71FjEuLentFxKSizjqs2oC+h4BHgH+OiA2Lc46jZlZHB04qrktE/FNxH6e3D17szr1IKlFm+vLVr17AFsBcKisLZvHzdZ3U3bfYn1TGDMwFtuug3vuB/6bywXwPcCOwc12ddYCLqDSF31/U+Wxx7keAs6l8W59LZfXDF4qfVyuOXw/4IZUWinlUEoV/qrvGPwAzqLRkzAc+XLNvJJVBkn8F/kBlit9HO7iXL1P5hv5wUWefmvu/dQW/11uLOu2/048W5dtTmWr5NHAbcBlwc3GPc4Ht6o6dCNxRXP8vVKZO1l9rhfdS/C4fqfndfqfsv3e+fLXKKzIb8swTSZI0ANiVIEmSqkwMJElSlYmBJEmqMjGQJElVJgaSJKnKxECSJFWZGEiSpCoTA0mSVGViIEmSqv4/wzyglzOKQr4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\" plot confusion matrix of dev set predictions \"\"\"\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "matplotlib.rcParams.update({\n",
    "    'font.family': 'serif',\n",
    "    'font.size': 16.4,\n",
    "    'figure.figsize': [10,7]\n",
    "})\n",
    "\n",
    "\n",
    "\n",
    "p = trainer.predict(val_dataset)\n",
    "preds = np.argmax(p.predictions, axis=1)\n",
    "\n",
    "cm = confusion_matrix(p.label_ids, preds, labels=[0,1,2])\n",
    "\n",
    "disp = ConfusionMatrixDisplay(confusion_matrix=cm,\n",
    "                               display_labels=[id2label[i] for i in range(3)])\n",
    "\n",
    "disp.plot(cmap='Greys')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe0a799d",
   "metadata": {},
   "source": [
    "# Optional - save adapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "956b9c65",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Configuration saved in ./trained_adapters/emotional-reactions/adapter_config.json\n",
      "Module weights saved in ./trained_adapters/emotional-reactions/pytorch_adapter.bin\n",
      "Configuration saved in ./trained_adapters/emotional-reactions/head_config.json\n",
      "Module weights saved in ./trained_adapters/emotional-reactions/pytorch_model_head.bin\n"
     ]
    }
   ],
   "source": [
    "\"\"\" set path for where to save the adapter \"\"\"\n",
    "adapter_save_path = f\"./trained_adapters/{adapter_name}\"\n",
    "\n",
    "\"\"\" save \"\"\"\n",
    "trainer.model.save_adapter(adapter_save_path, adapter_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}