baseline2_BiGRU+ResNet+Attention.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define 'EPOCHS' (total epochs) and 'EP_INT' (interval epochs) and then run from the starting..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fVNqmL5Yy4N6",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch, torchvision\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchtext import  vocab,data\n",
    "from torchtext.datasets import TranslationDataset, Multi30k\n",
    "from torchtext.data import Field, BucketIterator, TabularDataset\n",
    "from torchvision.transforms import ToTensor\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "import os, csv, sys, random, re, time, math, spacy, nltk\n",
    "\n",
    "from PIL import Image\n",
    "from numpy.random import RandomState\n",
    "from tensorboardX import SummaryWriter\n",
    "from nltk.tokenize.treebank import TreebankWordDetokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "#Define the logger\n",
    "#log_writer_train = SummaryWriter('TBlogs/train/')\n",
    "#log_writer_val = SummaryWriter('TBlogs/val/')\n",
    "#log_writer_test = SummaryWriter('TBlogs/test/')\n",
    "\n",
    "log_writer = SummaryWriter('TBlogs/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 3     # Total epochs to train for\n",
    "EP_INT = 1     # In the intervals of 'EP_INT' epochs\n",
    "CLIP = 1\n",
    "\n",
    "NF=1000        # For ResNet/VGG Feature Extraction\n",
    "#NF = 36*2048  # For Faster RCNN Feature Extraction using Object Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3efEa7DccC4y"
   },
   "outputs": [],
   "source": [
    "SEED = 1234\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "J5X-C_9Oy6zb"
   },
   "outputs": [],
   "source": [
    "dataset=pd.read_csv(r\"train_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 343
    },
    "colab_type": "code",
    "id": "mqeWgOxjzCvI",
    "outputId": "71f0a67b-516d-42c5-b4fd-47cb59ac6e33",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>trg</th>\n",
       "      <th>img_path</th>\n",
       "      <th>img_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>it was a dry year at the globes for films dire...</td>\n",
       "      <td>so welldeserved she was brilliant in this movi...</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/139...</td>\n",
       "      <td>1398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a person was taken into custody in the killing...</td>\n",
       "      <td>probably knew the clinton cabal</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/170...</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>new jersey at very dangerous place over corona...</td>\n",
       "      <td>trump</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/277...</td>\n",
       "      <td>2770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>image copyright getty images image caption rut...</td>\n",
       "      <td>rbg you stay alive stay alive</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/203...</td>\n",
       "      <td>2036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>per person albertans donate more money to char...</td>\n",
       "      <td>this is a great example of how socialist and c...</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/329...</td>\n",
       "      <td>329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>image copyright getty images image caption the...</td>\n",
       "      <td>we look like we have snouts try a face shield ...</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/317...</td>\n",
       "      <td>3177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>dr anthony fauci the nations leading infectiou...</td>\n",
       "      <td>hard pass</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/370...</td>\n",
       "      <td>3709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>a florida sheriff has banned his employees and...</td>\n",
       "      <td>the employees should walk out due to an unsafe...</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/402...</td>\n",
       "      <td>4027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>a federal appeals court friday overturned the ...</td>\n",
       "      <td>great now us tax payers have to pay to keep hi...</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/259...</td>\n",
       "      <td>2594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>a federal appeals court friday overturned the ...</td>\n",
       "      <td>great now us tax payers have to pay to keep hi...</td>\n",
       "      <td>/home/puneet/code/Multimodal Feedback/data/259...</td>\n",
       "      <td>2594</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   src  \\\n",
       "0    it was a dry year at the globes for films dire...   \n",
       "1    a person was taken into custody in the killing...   \n",
       "2    new jersey at very dangerous place over corona...   \n",
       "3    image copyright getty images image caption rut...   \n",
       "4    per person albertans donate more money to char...   \n",
       "..                                                 ...   \n",
       "995  image copyright getty images image caption the...   \n",
       "996  dr anthony fauci the nations leading infectiou...   \n",
       "997  a florida sheriff has banned his employees and...   \n",
       "998  a federal appeals court friday overturned the ...   \n",
       "999  a federal appeals court friday overturned the ...   \n",
       "\n",
       "                                                   trg  \\\n",
       "0    so welldeserved she was brilliant in this movi...   \n",
       "1                      probably knew the clinton cabal   \n",
       "2                                                trump   \n",
       "3                        rbg you stay alive stay alive   \n",
       "4    this is a great example of how socialist and c...   \n",
       "..                                                 ...   \n",
       "995  we look like we have snouts try a face shield ...   \n",
       "996                                          hard pass   \n",
       "997  the employees should walk out due to an unsafe...   \n",
       "998  great now us tax payers have to pay to keep hi...   \n",
       "999  great now us tax payers have to pay to keep hi...   \n",
       "\n",
       "                                              img_path  img_id  \n",
       "0    /home/puneet/code/Multimodal Feedback/data/139...    1398  \n",
       "1    /home/puneet/code/Multimodal Feedback/data/170...    1703  \n",
       "2    /home/puneet/code/Multimodal Feedback/data/277...    2770  \n",
       "3    /home/puneet/code/Multimodal Feedback/data/203...    2036  \n",
       "4    /home/puneet/code/Multimodal Feedback/data/329...     329  \n",
       "..                                                 ...     ...  \n",
       "995  /home/puneet/code/Multimodal Feedback/data/317...    3177  \n",
       "996  /home/puneet/code/Multimodal Feedback/data/370...    3709  \n",
       "997  /home/puneet/code/Multimodal Feedback/data/402...    4027  \n",
       "998  /home/puneet/code/Multimodal Feedback/data/259...    2594  \n",
       "999  /home/puneet/code/Multimodal Feedback/data/259...    2594  \n",
       "\n",
       "[1000 rows x 4 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nR3QYw8jzbOc"
   },
   "outputs": [],
   "source": [
    "def tokenize_text(text):\n",
    "    \"\"\"\n",
    "    Tokenizes English text from a string into a list of strings (tokens)\n",
    "    \"\"\"\n",
    "    return [tok.text for tok in nlp.tokenizer(text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "colab_type": "code",
    "id": "5o3ybO1Dzn0W",
    "outputId": "83ab3576-0835-4cbe-9ac1-fe4c985abc8e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('src', <torchtext.data.field.Field at 0x7f8bc8d2b310>),\n",
       " ('trg', <torchtext.data.field.Field at 0x7f8bc8d2b110>),\n",
       " ('img_path', None),\n",
       " ('img_id', <torchtext.data.field.Field at 0x7f8bc8d2b250>)]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ID = data.Field(sequential=False,use_vocab=False)\n",
    "SRC = Field(tokenize = tokenize_text, \n",
    "            init_token = '<sos>', \n",
    "            eos_token = '<eos>', \n",
    "            lower = True,\n",
    "            include_lengths = True)\n",
    "TRG = Field(tokenize = tokenize_text, \n",
    "            init_token = '<sos>', \n",
    "            eos_token = '<eos>', \n",
    "            lower = True,\n",
    "            )\n",
    "datafields=[('src', SRC), ('trg', TRG),('img_path',None),('img_id',ID)]\n",
    "datafields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'df = pd.read_csv(\"data.csv\")\\n\\nrng = RandomState()\\ntrain_data = df.sample(frac=0.7, random_state=rng)\\nval_data = df.loc[~df.index.isin(train_data.index)]\\n\\n#cols= [\"src\", \"trg\", \"img_path\", \"img_id\"]\\ntrain_data.to_csv(\\'train_data.csv\\', index= False) #columns=cols\\nval_data.to_csv(\\'val_data.csv\\', index= False) #columns=cols\\n'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''df = pd.read_csv(\"data.csv\")\n",
    "\n",
    "rng = RandomState()\n",
    "train_data = df.sample(frac=0.7, random_state=rng)\n",
    "val_data = df.loc[~df.index.isin(train_data.index)]\n",
    "\n",
    "#cols= [\"src\", \"trg\", \"img_path\", \"img_id\"]\n",
    "train_data.to_csv('train_data.csv', index= False) #columns=cols\n",
    "val_data.to_csv('val_data.csv', index= False) #columns=cols\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZgbvVh-izqPS"
   },
   "outputs": [],
   "source": [
    "train_data, val_data = data.TabularDataset.splits(path=r\"\",train=\"train_data.csv\", validation=\"val_data.csv\", format='csv', skip_header=True, fields=datafields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1001\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data))\n",
    "print(len(val_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "mwZexYjE0EE_",
    "outputId": "7e4f6e56-a179-495c-baa5-050b07a2be0a"
   },
   "outputs": [],
   "source": [
    "#print(vars(train_data.examples[4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Nl09s_RG0Xgq",
    "outputId": "e8751d89-3058-45e7-e934-7f615a078c57"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9223372036854775807"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.maxsize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UUJ1Ca-G0cDz"
   },
   "outputs": [],
   "source": [
    "SRC.build_vocab(train_data, vectors=\"glove.6B.100d\")\n",
    "TRG.build_vocab(train_data, vectors=\"glove.6B.100d\")\n",
    "SRC.build_vocab(train_data, min_freq=2)\n",
    "TRG.build_vocab(train_data, min_freq=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "CyiRg-hxbrcQ",
    "outputId": "7c92eae4-b6c0-4038-8896-9f9a5d9d8b7f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique tokens in target (en) vocabulary: 18500\n"
     ]
    }
   ],
   "source": [
    "print(f\"Unique tokens in target (en) vocabulary: {len(SRC.vocab)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique tokens in target (en) vocabulary: 4025\n"
     ]
    }
   ],
   "source": [
    "print(f\"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "A7LVmXZ8b843"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "vKvvpyErd42e",
    "outputId": "be52d098-b8a5-481a-e108-5ac6a19eb78f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_77VstPccGke"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 4\n",
    "\n",
    "train_iterator, x_iterator = BucketIterator.splits(\n",
    "    (train_data, train_data), \n",
    "    batch_size = BATCH_SIZE,\n",
    "    sort_within_batch = True,\n",
    "    sort_key = lambda x : len(x.src), \n",
    "    device = device)\n",
    "\n",
    "valid_iterator, y_iterator = BucketIterator.splits(\n",
    "    (val_data, val_data), \n",
    "    batch_size = BATCH_SIZE, \n",
    "    sort_within_batch = True,\n",
    "    sort_key = lambda x : len(x.src),\n",
    "    device = device)\n",
    "\n",
    "#valid_iterator, y_iterator = BucketIterator.splits(\n",
    "#    (val_data, val_data), \n",
    "#    batch_size = BATCH_SIZE, \n",
    "#    device = device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "djUCQ7A-o7yF",
    "outputId": "30410a05-ad19-4ea4-ceb8-56c9026d6fc5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "251\n",
      "25\n"
     ]
    }
   ],
   "source": [
    "print(len(train_iterator))\n",
    "print(len(valid_iterator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "I7A5VZ6GmLg-",
    "outputId": "b2df7af1-4ee6-48ac-dcf9-324ef3708823"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 1202x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 43x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "1 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 254x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 55x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "2 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 705x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 54x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "3 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 1030x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 41x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "4 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 1666x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 16x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "5 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 386x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 32x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "6 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 370x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 27x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "7 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 187x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 35x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "8 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 1875x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 52x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "9 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 613x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 25x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "10 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 264x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 29x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "11 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 563x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 54x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "12 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 88x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 32x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "13 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 1166x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 35x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "14 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 934x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 27x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "15 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 288x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 56x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "16 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 1361x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 46x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "17 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 215x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 29x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "18 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 1616x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 45x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "19 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 452x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 35x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "20 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 160x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 54x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "21 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 65x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 40x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "22 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 2731x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 37x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "23 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 1072x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 29x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n",
      "24 \n",
      "[torchtext.data.batch.Batch of size 4]\n",
      "\t[.src]:('[torch.cuda.LongTensor of size 529x4 (GPU 0)]', '[torch.cuda.LongTensor of size 4 (GPU 0)]')\n",
      "\t[.trg]:[torch.cuda.LongTensor of size 29x4 (GPU 0)]\n",
      "\t[.img_id]:[torch.cuda.LongTensor of size 4 (GPU 0)]\n"
     ]
    }
   ],
   "source": [
    "for i, batch in enumerate(valid_iterator):\n",
    "  print(i,batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7oAQpgo1cJ2g"
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, input_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.GRU(emb_dim, enc_hid_dim, bidirectional = True)\n",
    "        \n",
    "        self.fc = nn.Linear(enc_hid_dim * 2, dec_hid_dim)\n",
    "        self.vis_fc=nn.Linear(enc_hid_dim * 8, dec_hid_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, src,visual_features, src_len):\n",
    "        \n",
    "        #src = [src len, batch size]\n",
    "        #src_len = [src len]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(src))\n",
    "        \n",
    "        #embedded = [src len, batch size, emb dim]\n",
    "                \n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, src_len)\n",
    "                \n",
    "        packed_outputs, hidden = self.rnn(packed_embedded)\n",
    "                                 \n",
    "        #packed_outputs is a packed sequence containing all hidden states\n",
    "        #hidden is now from the final non-padded element in the batch\n",
    "            \n",
    "        outputs, _ = nn.utils.rnn.pad_packed_sequence(packed_outputs) \n",
    "\n",
    "        vis_features= torch.tanh(self.vis_fc(visual_features))\n",
    "            \n",
    "        #outputs is now a non-packed sequence, all hidden states obtained\n",
    "        #  when the input is a pad token are all zeros\n",
    "            \n",
    "        #outputs = [src len, batch size, hid dim * num directions]\n",
    "        #hidden = [n layers * num directions, batch size, hid dim]\n",
    "        \n",
    "        #hidden is stacked [forward_1, backward_1, forward_2, backward_2, ...]\n",
    "        #outputs are always from the last layer\n",
    "        \n",
    "        #hidden [-2, :, : ] is the last of the forwards RNN \n",
    "        #hidden [-1, :, : ] is the last of the backwards RNN\n",
    "        \n",
    "        #initial decoder hidden is final hidden state of the forwards and backwards \n",
    "        #  encoder RNNs fed through a linear layer\n",
    "        text_hidden = torch.tanh(self.fc(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1)))\n",
    "        hidden = torch.tanh(self.fc(torch.cat((text_hidden,vis_features), 1)))\n",
    "        #outputs = [src len, batch size, enc hid dim * 2]\n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        \n",
    "        return outputs, hidden ,vis_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-VE6FuBm3Ti8"
   },
   "outputs": [],
   "source": [
    "class Attention(nn.Module):\n",
    "    def __init__(self, enc_hid_dim, dec_hid_dim):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.attn = nn.Linear((enc_hid_dim * 2) + dec_hid_dim, dec_hid_dim)\n",
    "        self.v = nn.Parameter(torch.rand(dec_hid_dim))\n",
    "        \n",
    "    def forward(self, hidden, encoder_outputs,vis_features, mask):\n",
    "        \n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        #encoder_outputs = [src len, batch size, enc hid dim * 2]\n",
    "        #mask = [batch size, src len]\n",
    "        \n",
    "        batch_size = encoder_outputs.shape[1]\n",
    "        src_len = encoder_outputs.shape[0]\n",
    "        \n",
    "        #repeat encoder hidden state src_len times\n",
    "        hidden = hidden.unsqueeze(1).repeat(1, src_len, 1)\n",
    "        vis_features=vis_features.unsqueeze(1).repeat(1, src_len, 1)\n",
    "\n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        \n",
    "        #hidden = [batch size, src len, dec hid dim]\n",
    "        #encoder_outputs = [batch size, src len, enc hid dim * 2]\n",
    "        \n",
    "        energy = torch.tanh(self.attn(torch.cat((hidden, encoder_outputs), dim = 2))) \n",
    "        vis_energy=torch.tanh(self.attn(torch.cat((vis_features, encoder_outputs), dim = 2))) \n",
    "        \n",
    "        #energy = [batch size, src len, dec hid dim]\n",
    "                \n",
    "        energy = energy.permute(0, 2, 1)\n",
    "        vis_energy = vis_energy.permute(0, 2, 1)\n",
    "        \n",
    "        #energy = [batch size, dec hid dim, src len]\n",
    "        \n",
    "        #v = [dec hid dim]\n",
    "        \n",
    "        v = self.v.repeat(batch_size, 1).unsqueeze(1)\n",
    "        \n",
    "        #v = [batch size, 1, dec hid dim]\n",
    "            \n",
    "        attention = torch.bmm(v, energy).squeeze(1)\n",
    "        vis_attention = torch.bmm(v, vis_energy).squeeze(1)\n",
    "        \n",
    "        #attention = [batch size, src len]\n",
    "        \n",
    "        attention = attention.masked_fill(mask == 0, -1e10)\n",
    "        vis_attention = vis_attention.masked_fill(mask == 0, -1e10)\n",
    "        \n",
    "        return F.softmax(attention, dim = 1),F.softmax(vis_attention, dim = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1xXbwIMh3j2A"
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, output_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout, attention):\n",
    "        super().__init__()\n",
    "\n",
    "        self.output_dim = output_dim\n",
    "        self.attention = attention\n",
    "        \n",
    "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
    "        \n",
    "        self.rnn = nn.GRU((enc_hid_dim * 4) + emb_dim, dec_hid_dim)\n",
    "        \n",
    "        self.fc_out = nn.Linear((enc_hid_dim * 4) + dec_hid_dim + emb_dim, output_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, input, hidden, encoder_outputs,vis_features, mask):\n",
    "             \n",
    "        #input = [batch size]\n",
    "        #hidden = [batch size, dec hid dim]\n",
    "        #encoder_outputs = [src len, batch size, enc hid dim * 2]\n",
    "        #mask = [batch size, src len]\n",
    "        \n",
    "        input = input.unsqueeze(0)\n",
    "        \n",
    "        #input = [1, batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(input))\n",
    "        \n",
    "        #embedded = [1, batch size, emb dim]\n",
    "        \n",
    "        a,a_vis = self.attention(hidden, encoder_outputs,vis_features, mask)\n",
    "                \n",
    "        #a = [batch size, src len]\n",
    "        \n",
    "        a = a.unsqueeze(1)\n",
    "        a_vis = a_vis.unsqueeze(1)\n",
    "        \n",
    "        #a = [batch size, 1, src len]\n",
    "        \n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        \n",
    "        #encoder_outputs = [batch size, src len, enc hid dim * 2]\n",
    "        \n",
    "        weighted = torch.bmm(a, encoder_outputs)\n",
    "        vis_weighted = torch.bmm(a_vis, encoder_outputs)\n",
    "        \n",
    "        #weighted = [batch size, 1, enc hid dim * 2]\n",
    "        \n",
    "        weighted = weighted.permute(1, 0, 2)\n",
    "        vis_weighted = vis_weighted.permute(1, 0, 2)\n",
    "        \n",
    "        #weighted = [1, batch size, enc hid dim * 2]\n",
    "        \n",
    "        rnn_input = torch.cat((embedded, weighted,vis_weighted), dim = 2)\n",
    "        \n",
    "        #rnn_input = [1, batch size, (enc hid dim * 4) + emb dim]\n",
    "            \n",
    "        output, hidden = self.rnn(rnn_input, hidden.unsqueeze(0))\n",
    "        \n",
    "        #output = [seq len, batch size, dec hid dim * n directions]\n",
    "        #hidden = [n layers * n directions, batch size, dec hid dim]\n",
    "        \n",
    "        #seq len, n layers and n directions will always be 1 in this decoder, therefore:\n",
    "        #output = [1, batch size, dec hid dim]\n",
    "        #hidden = [1, batch size, dec hid dim]\n",
    "        #this also means that output == hidden\n",
    "        assert (output == hidden).all()\n",
    "        \n",
    "        embedded = embedded.squeeze(0)\n",
    "        output = output.squeeze(0)\n",
    "        weighted = weighted.squeeze(0)\n",
    "        vis_weighted = vis_weighted.squeeze(0)\n",
    "        \n",
    "        prediction = self.fc_out(torch.cat((output, weighted,vis_weighted, embedded), dim = 1))\n",
    "        \n",
    "        #prediction = [batch size, output dim]\n",
    "        \n",
    "        return prediction, hidden.squeeze(0), a.squeeze(1), a_vis.squeeze(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Oxymxf3l3hXO"
   },
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self, encoder, decoder, src_pad_idx, device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.src_pad_idx = src_pad_idx\n",
    "        self.device = device\n",
    "        \n",
    "    def create_mask(self, src):\n",
    "        mask = (src != self.src_pad_idx).permute(1, 0)\n",
    "        return mask\n",
    "        \n",
    "    def forward(self, src,visual_features, src_len, trg, teacher_forcing_ratio = 0.5):\n",
    "        \n",
    "        #src = [src len, batch size]\n",
    "        #src_len = [batch size]\n",
    "        #trg = [trg len, batch size]\n",
    "        #teacher_forcing_ratio is probability to use teacher forcing\n",
    "        #e.g. if teacher_forcing_ratio is 0.75 we use teacher forcing 75% of the time\n",
    "                    \n",
    "        batch_size = src.shape[1]\n",
    "        trg_len = trg.shape[0]\n",
    "        trg_vocab_size = self.decoder.output_dim\n",
    "        \n",
    "        #tensor to store decoder outputs\n",
    "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n",
    "        \n",
    "        #encoder_outputs is all hidden states of the input sequence, back and forwards\n",
    "        #hidden is the final forward and backward hidden states, passed through a linear layer\n",
    "        encoder_outputs, hidden,vis_features = self.encoder(src,visual_features, src_len)\n",
    "                \n",
    "        #first input to the decoder is the <sos> tokens\n",
    "        input = trg[0,:]\n",
    "        \n",
    "        mask = self.create_mask(src)\n",
    "\n",
    "        #mask = [batch size, src len]\n",
    "                \n",
    "        for t in range(1, trg_len):\n",
    "            \n",
    "            #insert input token embedding, previous hidden state, all encoder hidden states \n",
    "            #  and mask\n",
    "            #receive output tensor (predictions) and new hidden state\n",
    "            output, hidden, rand_1,rand_2 = self.decoder(input, hidden, encoder_outputs,vis_features, mask)\n",
    "            \n",
    "            #place predictions in a tensor holding predictions for each token\n",
    "            outputs[t] = output\n",
    "            \n",
    "            #decide if we are going to use teacher forcing or not\n",
    "            teacher_force = random.random() < teacher_forcing_ratio\n",
    "            \n",
    "            #get the highest predicted token from our predictions\n",
    "            top1 = output.argmax(1) \n",
    "            \n",
    "            #if teacher forcing, use actual next token as next input\n",
    "            #if not, use predicted token\n",
    "            input = trg[t] if teacher_force else top1\n",
    "            \n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "39nFMh_QcTFf"
   },
   "outputs": [],
   "source": [
    "INPUT_DIM = len(SRC.vocab)\n",
    "OUTPUT_DIM = len(TRG.vocab)\n",
    "ENC_EMB_DIM = 100\n",
    "DEC_EMB_DIM = 100\n",
    "ENC_HID_DIM = 125\n",
    "#ENC_HID_DIM = 512\n",
    "DEC_HID_DIM = 125\n",
    "#DEC_HID_DIM = 512\n",
    "ENC_DROPOUT = 0.5\n",
    "DEC_DROPOUT = 0.5\n",
    "SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token]\n",
    "\n",
    "attn = Attention(ENC_HID_DIM, DEC_HID_DIM)\n",
    "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, ENC_DROPOUT)\n",
    "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, ENC_HID_DIM, DEC_HID_DIM, DEC_DROPOUT, attn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token] \n",
    "TRG_PAD_IDX = TRG.vocab.stoi[TRG.pad_token]\n",
    "\n",
    "model = Seq2Seq(enc, dec, SRC_PAD_IDX, device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "O3xLbMEwcVGd",
    "outputId": "8c5f8207-7a83-40db-8d43-3a889d9226e6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18500"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "INPUT_DIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 319
    },
    "colab_type": "code",
    "id": "Kg2yIkSJcXIc",
    "outputId": "4b737749-5363-48db-9211-96107c160fb3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Seq2Seq(\n",
       "  (encoder): Encoder(\n",
       "    (embedding): Embedding(18500, 100)\n",
       "    (rnn): GRU(100, 125, bidirectional=True)\n",
       "    (fc): Linear(in_features=250, out_features=125, bias=True)\n",
       "    (vis_fc): Linear(in_features=1000, out_features=125, bias=True)\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "  )\n",
       "  (decoder): Decoder(\n",
       "    (attention): Attention(\n",
       "      (attn): Linear(in_features=375, out_features=125, bias=True)\n",
       "    )\n",
       "    (embedding): Embedding(4025, 100)\n",
       "    (rnn): GRU(600, 125)\n",
       "    (fc_out): Linear(in_features=725, out_features=4025, bias=True)\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def init_weights(m):\n",
    "    for name, param in m.named_parameters():\n",
    "        if 'weight' in name:\n",
    "            nn.init.normal_(param.data, mean=0, std=0.01)\n",
    "        else:\n",
    "            nn.init.constant_(param.data, 0)\n",
    "            \n",
    "model.apply(init_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 5,821,150 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "def count_parameters(model): \n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "print(f'The model has {count_parameters(model):,} trainable parameters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T3_voiowcb_D"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s1zvROSzceIL"
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss(ignore_index = TRG_PAD_IDX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PmqnnM62cg6C"
   },
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion, clip):#, log_writer_train):\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    \n",
    "    for i, batch in enumerate(iterator):\n",
    "        \n",
    "        src,src_len = batch.src\n",
    "        trg = batch.trg\n",
    "        img_id=batch.img_id\n",
    "        \n",
    "        #Log with Tensorboard: Loss & PPL for train & val \n",
    "#        log_writer_train.add_text('Text', str(src), epoch+1)\n",
    "#        log_writer_train.add_text('Comment', str(trg), epoch+1)\n",
    "##       log_writer_train.add_image('Image', img_id, epoch+1)    \n",
    "        \n",
    "        \n",
    "        \n",
    "        x=img_id.cpu().numpy()\n",
    "        y=len(x)\n",
    "        #visual_features=torch.empty(y,4096).cuda()\n",
    "        visual_features=torch.empty(y,NF).cuda()\n",
    "        df=pd.read_csv(r\"visual_features_resnet.csv\")\n",
    "        for i in range(y):\n",
    "            q=df[str(x[i])].to_numpy()\n",
    "            a=torch.from_numpy(q).unsqueeze(0)\n",
    "            visual_features[i]=a\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        #print(visual_features.shape) \n",
    "        #print(type(visual_features))   \n",
    "        output = model(src,visual_features, src_len, trg)\n",
    "        \n",
    "        #trg = [output_dim, batch_size]\n",
    "        #output = [output_dim, batch size, output dim]\n",
    "        \n",
    "        output_dim = output.shape[-1]\n",
    "        \n",
    "        output = output[1:].view(-1, output_dim)\n",
    "        trg = trg[1:].view(-1)\n",
    "        \n",
    "        #trg = [(output_dim - 1) * batch size]\n",
    "        #output = [(output_dim - 1) * batch size, output dim]\n",
    "        \n",
    "        loss = criterion(output, trg)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Jluau9tpcj1l"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):#, log_writer_val):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for i, batch in enumerate(iterator):\n",
    "\n",
    "            src,src_len = batch.src\n",
    "            trg = batch.trg\n",
    "            img_id=batch.img_id\n",
    "            \n",
    "            x=img_id.cpu().numpy()\n",
    "            y=len(x)\n",
    "            \n",
    "            #visual_features=torch.empty(y,4096).cuda()\n",
    "            visual_features=torch.empty(y,NF).cuda()\n",
    "            df=pd.read_csv(r\"visual_features_resnet.csv\")\n",
    "            for i in range(y):\n",
    "                q=df[str(x[i])].to_numpy()\n",
    "                a=torch.from_numpy(q).unsqueeze(0)\n",
    "                visual_features[i]=a\n",
    "\n",
    "            output = model(src,visual_features, src_len, trg,0) #turn off teacher forcing\n",
    "\n",
    "            #trg = [trg len, batch size]\n",
    "            #output = [trg len, batch size, output dim]\n",
    "\n",
    "            output_dim = output.shape[-1]\n",
    "            \n",
    "            output = output[1:].view(-1, output_dim)\n",
    "            trg = trg[1:].view(-1)\n",
    "\n",
    "            #trg = [(trg len - 1) * batch size]\n",
    "            #output = [(trg len - 1) * batch size, output dim]\n",
    "\n",
    "            loss = criterion(output, trg)\n",
    "            \n",
    "            epoch_loss += loss.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fjSF2x4Scl73"
   },
   "outputs": [],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_checkpt(model, optimizer, chpt_file):\n",
    "    start_epoch = 0\n",
    "    if (os.path.exists(chpt_file)):\n",
    "        print(\"=> loading checkpoint '{}'\".format(chpt_file))\n",
    "        checkpoint = torch.load(chpt_file)\n",
    "        model.load_state_dict(checkpoint['state_dict'])\n",
    "        optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "        start_epoch = checkpoint['epoch']\n",
    "        \n",
    "        print(\"=> loaded checkpoint '{}' (epoch {})\".format(chpt_file, checkpoint['epoch']))\n",
    "        \n",
    "    else:\n",
    "        print(\"=> Checkpoint NOT found '{}'\".format(chpt_file))\n",
    "    return model, optimizer, start_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vv3sdjJdA1PS"
   },
   "outputs": [],
   "source": [
    "def translate_sentence(sentence, src_field, trg_field, img_id,path, model, device, max_len = 50):\n",
    "\n",
    "    model.eval()\n",
    "        \n",
    "    if isinstance(sentence, str):\n",
    "        nlp = spacy.load('en_core_web_sm')\n",
    "        #nlp = spacy.load('en') ##https://www.gitmemory.com/issue/OmkarPathak/pyresparser/46/777568505\n",
    "        tokens = [token.text.lower() for token in nlp(sentence)]\n",
    "    else:\n",
    "        tokens = [token.lower() for token in sentence]\n",
    "\n",
    "    tokens = [src_field.init_token] + tokens + [src_field.eos_token]\n",
    "        \n",
    "    src_indexes = [src_field.vocab.stoi[token] for token in tokens]\n",
    "    \n",
    "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(1).to(device)\n",
    "\n",
    "    src_len = torch.LongTensor([len(src_indexes)]).to(device)\n",
    "    \n",
    "    df=pd.read_csv(path)\n",
    "    visual_features=torch.empty(1,NF).cuda()\n",
    "#    visual_features=torch.empty(1,4096).cuda()\n",
    "    q=df[str(img_id)].to_numpy()\n",
    "    a=torch.from_numpy(q).unsqueeze(0)\n",
    "    visual_features[0]=a\n",
    "\n",
    "    with torch.no_grad():\n",
    "        encoder_outputs, hidden, vis_features = model.encoder(src_tensor,visual_features, src_len)\n",
    "\n",
    "    mask = model.create_mask(src_tensor)\n",
    "        \n",
    "    trg_indexes = [trg_field.vocab.stoi[trg_field.init_token]]\n",
    "\n",
    "    attentions = torch.zeros(max_len, 1, len(src_indexes)).to(device)\n",
    "    \n",
    "    for i in range(max_len):\n",
    "\n",
    "        trg_tensor = torch.LongTensor([trg_indexes[-1]]).to(device)\n",
    "                \n",
    "        with torch.no_grad():\n",
    "            output, hidden, attention,vis_attention = model.decoder(trg_tensor, hidden, encoder_outputs,vis_features, mask)\n",
    "\n",
    "        attentions[i] = attention\n",
    "            \n",
    "        pred_token = output.argmax(1).item()\n",
    "        \n",
    "        trg_indexes.append(pred_token)\n",
    "\n",
    "        if pred_token == trg_field.vocab.stoi[trg_field.eos_token]:\n",
    "            break\n",
    "    \n",
    "    trg_tokens = [trg_field.vocab.itos[i] for i in trg_indexes]\n",
    "    \n",
    "    return trg_tokens[1:], attentions[:len(trg_tokens)-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"def display_attention(sentence, translation, attention, n_heads = 8, n_rows = 4, n_cols = 2):\\n    \\n    assert n_rows * n_cols == n_heads\\n    \\n    fig = plt.figure(figsize=(15,25))\\n    \\n    for i in range(n_heads):\\n        \\n        ax = fig.add_subplot(n_rows, n_cols, i+1)\\n        \\n        _attention = attention.squeeze(0)[i].cpu().detach().numpy()\\n\\n        cax = ax.matshow(_attention, cmap='bone')\\n\\n        ax.tick_params(labelsize=12)\\n        ax.set_xticklabels(['']+['<sos>']+[t.lower() for t in sentence]+['<eos>'], \\n                           rotation=45)\\n        ax.set_yticklabels(['']+translation)\\n\\n        ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\\n        ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\\n\\n    plt.show()\\n    plt.close()\\n\""
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''def display_attention(sentence, translation, attention, n_heads = 8, n_rows = 4, n_cols = 2):\n",
    "    \n",
    "    assert n_rows * n_cols == n_heads\n",
    "    \n",
    "    fig = plt.figure(figsize=(15,25))\n",
    "    \n",
    "    for i in range(n_heads):\n",
    "        \n",
    "        ax = fig.add_subplot(n_rows, n_cols, i+1)\n",
    "        \n",
    "        _attention = attention.squeeze(0)[i].cpu().detach().numpy()\n",
    "\n",
    "        cax = ax.matshow(_attention, cmap='bone')\n",
    "\n",
    "        ax.tick_params(labelsize=12)\n",
    "        ax.set_xticklabels(['']+['<sos>']+[t.lower() for t in sentence]+['<eos>'], \n",
    "                           rotation=45)\n",
    "        ax.set_yticklabels(['']+translation)\n",
    "\n",
    "        ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "        ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()\n",
    "    plt.close()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'text_idx = 26                #renamed from \\'example_idx\\'\\nimage_idx=\"9\"\\n\\npath=r\"visual_features_resnet.csv\"\\n\\nsrc = vars(train_data.examples[text_idx])[\\'src\\']\\ntrg = vars(train_data.examples[text_idx])[\\'trg\\']\\n#img = vars(train_data.examples[text_idx])[\\'img_path\\']\\n\\ntranslation, attention = translate_sentence(src, SRC, TRG, image_idx,path, model, device)\\n\\n#print(f\\'src = {src}\\n\\')\\nprint(f\\'Ground-truth Comment\\')\\nprint( \\'--------------------\\')\\nprint(f\\'trg = {trg}\\')\\n\\nprint(f\\'\\n\\nPredicted Feedback (Feedback)\\')\\nprint( \\'-----------------------------\\')\\n    \\nprint(f\\'{translation}\\')\\n'"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''text_idx = 26                #renamed from 'example_idx'\n",
    "image_idx=\"9\"\n",
    "\n",
    "path=r\"visual_features_resnet.csv\"\n",
    "\n",
    "src = vars(train_data.examples[text_idx])['src']\n",
    "trg = vars(train_data.examples[text_idx])['trg']\n",
    "#img = vars(train_data.examples[text_idx])['img_path']\n",
    "\n",
    "translation, attention = translate_sentence(src, SRC, TRG, image_idx,path, model, device)\n",
    "\n",
    "#print(f'src = {src}\\n')\n",
    "print(f'Ground-truth Comment')\n",
    "print( '--------------------')\n",
    "print(f'trg = {trg}')\n",
    "\n",
    "print(f'\\n\\nPredicted Feedback (Feedback)')\n",
    "print( '-----------------------------')\n",
    "    \n",
    "print(f'{translation}')\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Spice & Meteor\n",
    "#### (Hitting erorr in py3, need to run in py2...) tried switching the kernel to py2 and runnign here but T_EPOCHS is cleared off...\n",
    "#### 1. Go to '/home/puneet/code/EvalMetrics_py2'\n",
    "#### 2. Copy 'nEp_test_comments.csv' & 'nEp_test_feedbacks.csv'\n",
    "#### 3. Rename as 'test_comments.csv' & 'test_feedbacks.csv'\n",
    "#### 4. Run eval.py with 'py2' env\n",
    "#### 5. Rename the generated 'scores.txt' file  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quant_eval(model, optimizer, start_epoch, chpt_file):\n",
    "    from pycocoevalcap.bleu.bleu import Bleu\n",
    "    from pycocoevalcap.cider.cider import Cider\n",
    "    from pycocoevalcap.meteor.meteor import Meteor\n",
    "    from pycocoevalcap.rouge.rouge import Rouge\n",
    "    from pycocoevalcap.spice.spice import Spice\n",
    "    import os, json, csv\n",
    "    \n",
    "    model, optimizer, start_epoch = load_checkpt(model, optimizer, chpt_file)\n",
    "    T_EPOCHS = start_epoch + EP_INT    \n",
    "    \n",
    "\n",
    "    #print('Download Stanford models... Run once!')\n",
    "    os.system(\"sh get_stanford_models.sh\")\n",
    "\n",
    "    with open('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv',\"r\") as f: \n",
    "            reader = csv.reader(f)\n",
    "            gts = {rows[0]:rows[1:] for rows in reader}\n",
    "            #print(mydict) #prints with single quotes\n",
    "            #print (json.dumps(mydict)) #prints with double quotes\n",
    "\n",
    "    with open('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv',\"r\") as g: \n",
    "            reader = csv.reader(g)\n",
    "            res = {rows[0]:rows[1:] for rows in reader}\n",
    "            #print(json.dumps(mydict))\n",
    "\n",
    "    '''with open('temp/test_comments.json', 'r') as file:\n",
    "        gts = json.load(file)\n",
    "    with open('temp/test_feedbacks.json', 'r') as file:\n",
    "        res = json.load(file)\n",
    "    '''\n",
    "\n",
    "    def bleu():\n",
    "        scorer = Bleu(n=4)\n",
    "        score, scores = scorer.compute_score(gts, res)\n",
    "        return score\n",
    "\n",
    "\n",
    "    def cider():\n",
    "        scorer = Cider()\n",
    "        (score, scores) = scorer.compute_score(gts, res)\n",
    "        return score\n",
    "\n",
    "    def rouge():\n",
    "        scorer = Rouge()\n",
    "        score, scores = scorer.compute_score(gts, res)\n",
    "        return score\n",
    "\n",
    "    #bgts = gts[0].encode(encoding='UTF-8')\n",
    "    #bres = res[0].encode(encoding='UTF-8')\n",
    "\n",
    "    def spice():\n",
    "        scorer = Spice()\n",
    "        #print(gts, res)\n",
    "        score, scores = scorer.compute_score(gts, res)\n",
    "        return score\n",
    "\n",
    "    def meteor():\n",
    "        scorer = Meteor()\n",
    "        #print(gts, res)\n",
    "        score, scores = scorer.compute_score(bgts, bres)\n",
    "        return score    \n",
    "    s_cider=cider()\n",
    "    s_rouge=rouge()\n",
    "    s_bleu=bleu()\n",
    "    #s_spice=spice()#\n",
    "    #s_meteor=meteor()#\n",
    "    \n",
    "    print('\\n----------------------\\nbleu = %s' %s_bleu )\n",
    "    print('cider = %s' %s_cider )\n",
    "    print('rouge = %s' %s_rouge )\n",
    "    #print('spice = %s' %s_spice )\n",
    "    #print('meteor = %s' %s_meteor )\n",
    "    \n",
    "    b=\" \".join(str(x) for x in s_bleu)\n",
    "    print('\\n----------------------')\n",
    "    f = open('scores.txt', 'w') \n",
    "    f.write(\"\\ncider: %f\" % s_cider)\n",
    "    f.write(\"\\nrouge: %f\" % s_rouge)\n",
    "    #f.write(\"\\nspice: %f\" % s_spice)\n",
    "    #f.write(\"\\nmeteor: %f\" % s_meteor)\n",
    "    f.write(\"\\nbleu :\")\n",
    "    f.write(b)\n",
    "    f.close()\n",
    "    \n",
    "    #print(str(T_EPOCHS))\n",
    "    #Log with Tensorboard: Eval metrics\n",
    "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/cider', str(s_cider))\n",
    "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/rouge', str(s_rouge))\n",
    "    #log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/spice', str(s_spice))\n",
    "    #log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/meteor', str(s_meteor))\n",
    "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/bleu-1', str(s_bleu[0]))\n",
    "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/bleu-2', str(s_bleu[1]))\n",
    "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/bleu-3', str(s_bleu[2]))\n",
    "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/bleu-4', str(s_bleu[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def qual_eval(model, optimizer, start_epoch, chpt_file):\n",
    "    model, optimizer, start_epoch = load_checkpt(model, optimizer, chpt_file)\n",
    "    T_EPOCHS = start_epoch + EP_INT\n",
    "    \n",
    "    # Save the predicted Feedbacks in CSV file\n",
    "    # Log with Tensorboard: Text, Comment, Image and Feedback\n",
    "    test_pred=[]\n",
    "    #test_df=pd.read_csv(\"temp/test_data.csv\")  \n",
    "    test_df=pd.read_csv(\"temp/test_data1.csv\") \n",
    "    path=r\"visual_features_resnet.csv\"      \n",
    "    length=test_df.shape[0]\n",
    "    images = []\n",
    "\n",
    "    #print(length)\n",
    "    for i in range(length):\n",
    "        src=test_df['src'][i]\n",
    "        trg=test_df['trg'][i]\n",
    "        img_path=test_df['img_path'][i]\n",
    "        image_idx=test_df['img_id'][i]\n",
    "        translation, attention = translate_sentence(src, SRC, TRG, image_idx, path, model, device)\n",
    "\n",
    "        if not translation:\n",
    "            translation=\"*empty*\"\n",
    "\n",
    "        #Untokenization    \n",
    "        translation1=translation[0:(len(translation)-1)]    \n",
    "        translation2 = TreebankWordDetokenizer().detokenize(translation1)\n",
    "        test_pred.append(str(translation2))\n",
    "\n",
    "        image = Image.open(img_path)\n",
    "        image = ToTensor()(image)   \n",
    "        #images.append(image)\n",
    "        \n",
    "        #print(str(T_EPOCHS))\n",
    "        if (i%10==0): \n",
    "            #Log with Tensorboard: Text, Comment, Image and Feedback\n",
    "            log_writer.add_text(str(T_EPOCHS)+'Ep=>Ground-truth Comment of Sample/'+str(i+1), str(trg))\n",
    "            log_writer.add_text(str(T_EPOCHS)+'Ep=>News Text of Sample/'+str(i+1), str(src))#, i+1)\n",
    "            log_writer.add_text(str(T_EPOCHS)+'Ep=>Predicted Feedback of Sample/'+str(i+1), str(translation))\n",
    "            #log_writer.add_image('Image', image, i+1)\n",
    "            #image_grid = torchvision.utils.make_grid(images)\n",
    "            log_writer.add_image(str(T_EPOCHS)+'Ep:Image of Sample/'+str(i+1), image)        \n",
    "\n",
    "    with open('temp/'+str(T_EPOCHS)+'Ep_test_results.csv', 'w'): \n",
    "        pass\n",
    "    with open('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv', 'w'): \n",
    "        pass\n",
    "    with open('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv', 'w'): \n",
    "        pass\n",
    "\n",
    "    test_df[\"pred\"] = test_pred \n",
    "    test_df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_results.csv', index= False)\n",
    "    test_df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv', index= True, columns=[\"trg\"]) #index -> \"key\": value -> [\"trg/pred\"]\n",
    "    test_df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv', index= True, columns=[\"pred\"])\n",
    "\n",
    "    #Re-open and save with new column names\n",
    "    df = pd.read_csv('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv')\n",
    "    df.columns = ['id', 'comment']\n",
    "    df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv', index= False)\n",
    "\n",
    "    df = pd.read_csv('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv')\n",
    "    df.columns = ['id', 'feedback']\n",
    "    df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv', index= False)\n",
    "    \n",
    "    print('tensorboard --logdir \"/home/puneet/code/Multimodal Feedback/TBlogs\"\\n---')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def interval_train(model, optimizer, start_epoch, chpt_file):\n",
    "    model, optimizer, start_epoch = load_checkpt(model, optimizer, chpt_file)\n",
    "    T_EPOCHS = start_epoch + EP_INT\n",
    "\n",
    "    print('Already trained for',start_epoch, 'epochs. Training now for', EP_INT, 'more')\n",
    "\n",
    "    best_valid_loss = float('inf')\n",
    "    cur_best_train_loss = float('inf')\n",
    "\n",
    "    #for epoch in range(EP_INT):\n",
    "    for epoch in range(start_epoch, T_EPOCHS):    \n",
    "        start_time = time.time()\n",
    "\n",
    "        train_loss = train(model, train_iterator, optimizer, criterion, CLIP)#, log_writer_train)\n",
    "        valid_loss = evaluate(model, valid_iterator, criterion)#, log_writer_val)\n",
    "\n",
    "        #Log with Tensorboard: Loss & PPL for train & val \n",
    "        log_writer.add_scalar('Train/Loss',float(train_loss), epoch+1)\n",
    "        log_writer.add_scalar('Train/PPL', float(math.exp(train_loss)), epoch+1)    \n",
    "        log_writer.add_scalar('Val/Loss',float(valid_loss), epoch+1)\n",
    "        log_writer.add_scalar('Val/PPL', float(math.exp(valid_loss)), epoch+1)\n",
    "\n",
    "        end_time = time.time()\n",
    "\n",
    "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "\n",
    "        #if valid_loss < best_valid_loss:\n",
    "        #    best_valid_loss = valid_loss\n",
    "        #if train_loss < cur_best_train_loss:\n",
    "        #    cur_best_train_loss = train_loss\n",
    "\n",
    "        state = {'epoch': T_EPOCHS, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), 'loss': train_loss}\n",
    "        torch.save(state, chpt_file)\n",
    "\n",
    "        print(f'\\nEpoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
    "        print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
    "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')\n",
    "\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=> Checkpoint NOT found '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt'\n",
      "=> Checkpoint NOT found '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt'\n",
      "Already trained for 0 epochs. Training now for 1 more\n",
      "\n",
      "Epoch: 01 | Time: 5m 54s\n",
      "\tTrain Loss: 7.035 | Train PPL: 1135.835\n",
      "\t Val. Loss: 6.966 |  Val. PPL: 1060.089\n",
      "\n",
      "Evaluation\n",
      "----------------------\n",
      "=> loading checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt'\n",
      "=> loaded checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt' (epoch 1)\n",
      "tensorboard --logdir \"/home/puneet/code/Multimodal Feedback/TBlogs\"\n",
      "---\n",
      "=> loading checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt'\n",
      "=> loaded checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt' (epoch 1)\n",
      "{'testlen': 96, 'reflen': 1280, 'guess': [96, 0, 0, 0], 'correct': [38, 0, 0, 0]}\n",
      "ratio: 0.0749999999999414\n",
      "\n",
      "----------------------\n",
      "bleu = [1.7426643764117067e-06, 2.7698584248886856e-09, 3.2325023135205575e-10, 1.1042806971312435e-10]\n",
      "cider = 0.021471952631849256\n",
      "rouge = 0.048130053312853104\n",
      "\n",
      "----------------------\n",
      "=> loading checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt'\n",
      "=> loaded checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt' (epoch 1)\n",
      "Already trained for 1 epochs. Training now for 1 more\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 30.00 MiB (GPU 0; 7.80 GiB total capacity; 4.06 GiB already allocated; 61.12 MiB free; 4.42 GiB reserved in total by PyTorch)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-91-b21a4657a88c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEPOCHS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0mEP_INT\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0minterval_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchpt_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\nEvaluation\\n----------------------'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mqual_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchpt_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-90-62c952931899>\u001b[0m in \u001b[0;36minterval_train\u001b[0;34m(model, optimizer, start_epoch, chpt_file)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_iterator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCLIP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#, log_writer_train)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m         \u001b[0mvalid_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_iterator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#, log_writer_val)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-81-765c1c6d59e3>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, iterator, optimizer, criterion, clip)\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0;31m#print(visual_features.shape)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;31m#print(type(visual_features))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvisual_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msrc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;31m#trg = [output_dim, batch_size]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-73-18b57ed81b6f>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, src, visual_features, src_len, trg, teacher_forcing_ratio)\u001b[0m\n\u001b[1;32m     43\u001b[0m             \u001b[0;31m#  and mask\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m             \u001b[0;31m#receive output tensor (predictions) and new hidden state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m             \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrand_1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrand_2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvis_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m             \u001b[0;31m#place predictions in a tensor holding predictions for each token\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-72-dcdff396bf41>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hidden, encoder_outputs, vis_features, mask)\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;31m#embedded = [1, batch size, emb dim]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m         \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0ma_vis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattention\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvis_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;31m#a = [batch size, src len]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-71-26c6817e53c2>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden, encoder_outputs, vis_features, mask)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;31m#encoder_outputs = [batch size, src len, enc hid dim * 2]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m         \u001b[0menergy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtanh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0mvis_energy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtanh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvis_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 30.00 MiB (GPU 0; 7.80 GiB total capacity; 4.06 GiB already allocated; 61.12 MiB free; 4.42 GiB reserved in total by PyTorch)"
     ]
    }
   ],
   "source": [
    "chpt_file = '/home/puneet/code/Multimodal Feedback/checkpoints/baseline2.pt'\n",
    "model, optimizer, start_epoch = load_checkpt(model, optimizer, chpt_file)\n",
    "T_EPOCHS = start_epoch + EP_INT\n",
    "    \n",
    "for i in range(EPOCHS):\n",
    "    if (i%EP_INT==0):\n",
    "        interval_train(model, optimizer, start_epoch, chpt_file)\n",
    "        print('\\nEvaluation\\n----------------------')\n",
    "        qual_eval(model, optimizer, start_epoch, chpt_file)\n",
    "        quant_eval(model, optimizer, start_epoch, chpt_file)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Seq2Seq_vis_attention_ensembled_model.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}