MMFeed_transformer+bboxFeatures_2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "pytorch",
      "language": "python",
      "name": "pytorch"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "colab": {
      "name": "baseline4.1_transformer+bboxFeatures.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ivu1v9ogsjhN"
      },
      "source": [
        "Ref: https://github.com/bentrevett/pytorch-seq2seq/blob/master/6%20-%20Attention%20is%20All%20You%20Need.ipynb"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Eh-Afo8jsjhO"
      },
      "source": [
        "#### Define 'EPOCHS' (total epochs) and 'EP_INT' (interval epochs) and then run from the starting..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G1tfixgksjhO"
      },
      "source": [
        "import torch, torchvision\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torchtext import  vocab,data\n",
        "from torchtext.datasets import TranslationDataset, Multi30k\n",
        "from torchtext.data import Field, BucketIterator, TabularDataset\n",
        "from torchvision.transforms import ToTensor\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "import os, csv, sys, random, re, time, math, spacy, nltk\n",
        "\n",
        "from PIL import Image\n",
        "from numpy.random import RandomState\n",
        "from tensorboardX import SummaryWriter\n",
        "from nltk.tokenize.treebank import TreebankWordDetokenizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5rqGn_39sjhP"
      },
      "source": [
        "nlp = spacy.load('en_core_web_sm')\n",
        "\n",
        "#Define the logger\n",
        "#log_writer_train = SummaryWriter('TBlogs/train/')\n",
        "#log_writer_val = SummaryWriter('TBlogs/val/')\n",
        "#log_writer_test = SummaryWriter('TBlogs/test/')\n",
        "\n",
        "log_writer = SummaryWriter('TBlogs/')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k9LDhCJLsjhQ"
      },
      "source": [
        "EPOCHS = 10     # Total epochs to train for\n",
        "EP_INT = 2     # In the intervals of 'EP_INT' epochs\n",
        "CLIP = 1\n",
        "\n",
        "NF = 36*2048   # For Faster RCNN Feature Extraction using Object Detection\n",
        "#NF=1000       # For ResNet/VGG Feature Extraction"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DvKnRu6sjhQ"
      },
      "source": [
        "SEED = 1234\n",
        "max_length=102\n",
        "\n",
        "random.seed(SEED)\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.cuda.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kbn765BsjhQ"
      },
      "source": [
        "#dataset.isna().sum()\n",
        "#df_null = dataset.isnull().unstack()\n",
        "#t = df_null[df_null]\n",
        "#t"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3yrbVeRCsjhR"
      },
      "source": [
        "Need to pass fixed length tokens to the transformer. Hence, 1. removing rows with null data and 2. truncating src length to max_length"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZamElBxIsjhR",
        "outputId": "fa2b1389-85ef-4d1b-f81a-9856774f15e8"
      },
      "source": [
        "# Cleaning the Training Data file...\n",
        "dataset=pd.read_csv(r\"train_data.csv\")\n",
        "dataset.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(58342, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dXw1uFBXsjhS",
        "outputId": "50967700-de1a-47b2-a60d-154b45b49717"
      },
      "source": [
        "for i in range (dataset.shape[0]):\n",
        "    if len(dataset['src'][i].split()) < max_length:\n",
        "        dataset = dataset.drop(i)\n",
        "    else:\n",
        "        dataset['src'][i] = \" \".join(dataset['src'][i].split()[0:102])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<ipython-input-9-762ee2878af4>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dataset['src'][i] = \" \".join(dataset['src'][i].split()[0:102])\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z8eruD4FsjhS",
        "outputId": "d52b3b44-250a-41e7-c6ec-28fbdb8124c9"
      },
      "source": [
        "dataset.to_csv('temp/train_data1.csv', index= False) \n",
        "dataset = pd.read_csv('temp/train_data1.csv')\n",
        "dataset.head(-1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>src</th>\n",
              "      <th>trg</th>\n",
              "      <th>img_path</th>\n",
              "      <th>img_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>video a bizarre conspiracy theory has surged i...</td>\n",
              "      <td>i take news with a pinch of salt so should eve...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/781...</td>\n",
              "      <td>7817</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>the pentagon said thursday that the us militar...</td>\n",
              "      <td>great news</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/651...</td>\n",
              "      <td>6514</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>washington pressure is growing among president...</td>\n",
              "      <td>more good news please</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/230...</td>\n",
              "      <td>2309</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>our energy supplier for more than years sse is...</td>\n",
              "      <td>why this video is getting so many dislikes</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/447...</td>\n",
              "      <td>4470</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>michigan gov gretchen whitmer has met with pre...</td>\n",
              "      <td>unfortunately this will anger the mudsharks th...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/823...</td>\n",
              "      <td>8237</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50954</th>\n",
              "      <td>kabul afghanistan the taliban ambushed a peace...</td>\n",
              "      <td>i always for the of reeducation no hope= misle...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/669...</td>\n",
              "      <td>6697</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50955</th>\n",
              "      <td>file photo the olympic rings are illuminated i...</td>\n",
              "      <td>the crazy just never seems to stop in florida ...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/541...</td>\n",
              "      <td>5417</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50956</th>\n",
              "      <td>saudi arabia has sentenced five people to deat...</td>\n",
              "      <td>five lives to save face for the prince</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/627...</td>\n",
              "      <td>6271</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50957</th>\n",
              "      <td>acting manatee county administrator dr scott h...</td>\n",
              "      <td>floridas republican governors have been availa...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/935...</td>\n",
              "      <td>9358</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50958</th>\n",
              "      <td>matamoros mexico under a canopy on the edge of...</td>\n",
              "      <td>stop spreading propagandathere are no doctors ...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/622...</td>\n",
              "      <td>6220</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>50959 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     src  \\\n",
              "0      video a bizarre conspiracy theory has surged i...   \n",
              "1      the pentagon said thursday that the us militar...   \n",
              "2      washington pressure is growing among president...   \n",
              "3      our energy supplier for more than years sse is...   \n",
              "4      michigan gov gretchen whitmer has met with pre...   \n",
              "...                                                  ...   \n",
              "50954  kabul afghanistan the taliban ambushed a peace...   \n",
              "50955  file photo the olympic rings are illuminated i...   \n",
              "50956  saudi arabia has sentenced five people to deat...   \n",
              "50957  acting manatee county administrator dr scott h...   \n",
              "50958  matamoros mexico under a canopy on the edge of...   \n",
              "\n",
              "                                                     trg  \\\n",
              "0      i take news with a pinch of salt so should eve...   \n",
              "1                                             great news   \n",
              "2                                  more good news please   \n",
              "3             why this video is getting so many dislikes   \n",
              "4      unfortunately this will anger the mudsharks th...   \n",
              "...                                                  ...   \n",
              "50954  i always for the of reeducation no hope= misle...   \n",
              "50955  the crazy just never seems to stop in florida ...   \n",
              "50956             five lives to save face for the prince   \n",
              "50957  floridas republican governors have been availa...   \n",
              "50958  stop spreading propagandathere are no doctors ...   \n",
              "\n",
              "                                                img_path  img_id  \n",
              "0      /home/puneet/code/Multimodal Feedback/data/781...    7817  \n",
              "1      /home/puneet/code/Multimodal Feedback/data/651...    6514  \n",
              "2      /home/puneet/code/Multimodal Feedback/data/230...    2309  \n",
              "3      /home/puneet/code/Multimodal Feedback/data/447...    4470  \n",
              "4      /home/puneet/code/Multimodal Feedback/data/823...    8237  \n",
              "...                                                  ...     ...  \n",
              "50954  /home/puneet/code/Multimodal Feedback/data/669...    6697  \n",
              "50955  /home/puneet/code/Multimodal Feedback/data/541...    5417  \n",
              "50956  /home/puneet/code/Multimodal Feedback/data/627...    6271  \n",
              "50957  /home/puneet/code/Multimodal Feedback/data/935...    9358  \n",
              "50958  /home/puneet/code/Multimodal Feedback/data/622...    6220  \n",
              "\n",
              "[50959 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IVNuLtlksjhT"
      },
      "source": [
        "# Check 1: on the lengths of the cleaned data\n",
        "#for i in range (dataset1.shape[0]):\n",
        "#    if not len(dataset1['src'][i].split()) == 102:\n",
        "#        print('1')\n",
        "\n",
        "# Check 2: if (kk<max_length), then manually delete the entried from .csv file. \n",
        "# train_data: Print to check whether any samples with length><max_length are remaining...\n",
        "#for i in range (dataset.shape[0]):\n",
        "#     res = re.findall(r'\\w+', dataset[\"src\"][i])\n",
        "#     print(res)\n",
        "#     kk=min(len(res),max_length)\n",
        "#     if (kk<max_length):\n",
        "#            print(i,\" \",len(res),\" \",kk,\" \", dataset[\"img_id\"][i])\n",
        "##      zz=str(res[0])   \n",
        "##      for j in range(kk-1):\n",
        "##          zz=zz+\" \"+str(res[j+1] )\n",
        "##      dataset[\"src\"][i]=zz    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JOL2v4WvsjhT",
        "outputId": "f3f54bfe-c47a-4090-a68a-3b8c28b0a329"
      },
      "source": [
        "# Cleaning the Validation Data file...\n",
        "dataset1=pd.read_csv(r\"val_data.csv\")\n",
        "dataset1.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(15558, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lwyqd8MesjhT",
        "outputId": "98273293-7bae-4f8d-ed22-8de612ad09ba"
      },
      "source": [
        "for i in range (dataset1.shape[0]):\n",
        "    if len(dataset1['src'][i].split()) < max_length:\n",
        "        dataset1 = dataset1.drop(i)\n",
        "    else:\n",
        "        dataset1['src'][i] = \" \".join(dataset1['src'][i].split()[0:102])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<ipython-input-13-d89220fed5a9>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dataset1['src'][i] = \" \".join(dataset1['src'][i].split()[0:102])\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TvOxjcQssjhU",
        "outputId": "d4a7dfbe-1e24-48f1-92d3-a99d975e621e"
      },
      "source": [
        "dataset1.to_csv('temp/val_data1.csv', index= False)  \n",
        "dataset1 = pd.read_csv('temp/val_data1.csv')\n",
        "dataset1.head(-1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>src</th>\n",
              "      <th>trg</th>\n",
              "      <th>img_path</th>\n",
              "      <th>img_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>image copyright getty images image caption the...</td>\n",
              "      <td>yin wei zhong guo de fa zhan rang mei di guo g...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/782...</td>\n",
              "      <td>7826</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>washington amid escalating tensions with both ...</td>\n",
              "      <td>well done advisors</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/831...</td>\n",
              "      <td>8310</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>washington president trump directed the federa...</td>\n",
              "      <td>hey come on you gotta see where trump is comin...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/761...</td>\n",
              "      <td>7619</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>an american sign language interpreter who help...</td>\n",
              "      <td>for references who want to find additional inc...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/856...</td>\n",
              "      <td>8563</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>although many consider them to be rivals both ...</td>\n",
              "      <td>thank you very much for your work</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/260...</td>\n",
              "      <td>2600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13595</th>\n",
              "      <td>a threegame series between the st louis cardin...</td>\n",
              "      <td>heres how cnn creates fake news</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/823...</td>\n",
              "      <td>8236</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13596</th>\n",
              "      <td>a view of the evergiven container ship as it r...</td>\n",
              "      <td>any chance the gpu prices will go down huh no ...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/287...</td>\n",
              "      <td>2870</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13597</th>\n",
              "      <td>in the wake of the atlantaarea shootings and t...</td>\n",
              "      <td>can politicians do anything other than condemn</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/176...</td>\n",
              "      <td>1768</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13598</th>\n",
              "      <td>dr anthony fauci says there are five or six th...</td>\n",
              "      <td>selfish</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/828...</td>\n",
              "      <td>8288</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13599</th>\n",
              "      <td>image copyright epa image caption pope francis...</td>\n",
              "      <td>so instead of leading a good life it is ok to ...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/582...</td>\n",
              "      <td>5826</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13600 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     src  \\\n",
              "0      image copyright getty images image caption the...   \n",
              "1      washington amid escalating tensions with both ...   \n",
              "2      washington president trump directed the federa...   \n",
              "3      an american sign language interpreter who help...   \n",
              "4      although many consider them to be rivals both ...   \n",
              "...                                                  ...   \n",
              "13595  a threegame series between the st louis cardin...   \n",
              "13596  a view of the evergiven container ship as it r...   \n",
              "13597  in the wake of the atlantaarea shootings and t...   \n",
              "13598  dr anthony fauci says there are five or six th...   \n",
              "13599  image copyright epa image caption pope francis...   \n",
              "\n",
              "                                                     trg  \\\n",
              "0      yin wei zhong guo de fa zhan rang mei di guo g...   \n",
              "1                                     well done advisors   \n",
              "2      hey come on you gotta see where trump is comin...   \n",
              "3      for references who want to find additional inc...   \n",
              "4                      thank you very much for your work   \n",
              "...                                                  ...   \n",
              "13595                    heres how cnn creates fake news   \n",
              "13596  any chance the gpu prices will go down huh no ...   \n",
              "13597     can politicians do anything other than condemn   \n",
              "13598                                            selfish   \n",
              "13599  so instead of leading a good life it is ok to ...   \n",
              "\n",
              "                                                img_path  img_id  \n",
              "0      /home/puneet/code/Multimodal Feedback/data/782...    7826  \n",
              "1      /home/puneet/code/Multimodal Feedback/data/831...    8310  \n",
              "2      /home/puneet/code/Multimodal Feedback/data/761...    7619  \n",
              "3      /home/puneet/code/Multimodal Feedback/data/856...    8563  \n",
              "4      /home/puneet/code/Multimodal Feedback/data/260...    2600  \n",
              "...                                                  ...     ...  \n",
              "13595  /home/puneet/code/Multimodal Feedback/data/823...    8236  \n",
              "13596  /home/puneet/code/Multimodal Feedback/data/287...    2870  \n",
              "13597  /home/puneet/code/Multimodal Feedback/data/176...    1768  \n",
              "13598  /home/puneet/code/Multimodal Feedback/data/828...    8288  \n",
              "13599  /home/puneet/code/Multimodal Feedback/data/582...    5826  \n",
              "\n",
              "[13600 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AL6SYc76sjhU",
        "outputId": "de5655ec-be70-4232-dca6-01953cd9bd5d"
      },
      "source": [
        "# val_data: Print to check whether any samples with length><max_length are remaining...\n",
        "dataset=dataset1\n",
        "for i in range (dataset.shape[0]):\n",
        "     res = re.findall(r'\\w+', dataset[\"src\"][i])\n",
        " #     print(res)\n",
        "     kk=min(len(res),max_length)\n",
        "     if (kk<max_length):\n",
        "            print(i,\" \",len(res),\" \",kk,\" \", dataset[\"img_id\"][i])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "22   100   100   8005\n",
            "325   101   101   5425\n",
            "778   100   100   8948\n",
            "1720   100   100   4645\n",
            "1903   100   100   8948\n",
            "1920   100   100   4645\n",
            "1970   100   100   8948\n",
            "2055   100   100   8948\n",
            "2517   100   100   9195\n",
            "2551   101   101   660\n",
            "2600   100   100   8948\n",
            "2725   100   100   4645\n",
            "2905   100   100   8948\n",
            "3045   100   100   8948\n",
            "3056   100   100   8948\n",
            "3084   100   100   8948\n",
            "3308   101   101   8505\n",
            "3473   100   100   8948\n",
            "3958   100   100   8005\n",
            "4703   100   100   8948\n",
            "4777   100   100   8948\n",
            "4893   100   100   4645\n",
            "4895   99   99   81\n",
            "5211   100   100   8948\n",
            "5586   100   100   8948\n",
            "5612   98   98   3729\n",
            "5776   100   100   8948\n",
            "5818   100   100   8005\n",
            "6174   100   100   8948\n",
            "6225   100   100   8948\n",
            "6310   100   100   8948\n",
            "6481   100   100   8948\n",
            "6487   100   100   8948\n",
            "6555   100   100   8948\n",
            "6684   101   101   8505\n",
            "7058   100   100   8948\n",
            "7145   100   100   8948\n",
            "7247   100   100   9195\n",
            "7609   100   100   8005\n",
            "8058   99   99   1426\n",
            "8095   101   101   8496\n",
            "8123   100   100   83\n",
            "8486   100   100   3733\n",
            "8606   100   100   4645\n",
            "8624   100   100   4645\n",
            "8646   100   100   4645\n",
            "8699   100   100   3731\n",
            "8825   100   100   3432\n",
            "8934   100   100   4645\n",
            "8973   100   100   8948\n",
            "9337   101   101   7185\n",
            "9729   100   100   3432\n",
            "9835   100   100   8948\n",
            "10142   98   98   3729\n",
            "11824   100   100   8948\n",
            "11988   100   100   8005\n",
            "12075   100   100   8948\n",
            "12106   100   100   8948\n",
            "12111   100   100   8948\n",
            "12134   100   100   8948\n",
            "12227   100   100   8948\n",
            "12334   100   100   8005\n",
            "12576   100   100   4645\n",
            "12917   100   100   8005\n",
            "13110   100   100   8948\n",
            "13354   100   100   3733\n",
            "13560   101   101   45\n",
            "13574   100   100   8948\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wu3XAXc6sjhU"
      },
      "source": [
        "We'll then create our tokenizers as before."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "klGIwFeZsjhV"
      },
      "source": [
        "def tokenize_en(text):\n",
        "    \"\"\"\n",
        "    Tokenizes English text from a string into a list of strings\n",
        "    \"\"\"\n",
        "    return [tok.text for tok in nlp.tokenizer(text)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8S11c2JsjhV"
      },
      "source": [
        "SRC = Field(tokenize = tokenize_en, \n",
        "            init_token = '<sos>', \n",
        "            eos_token = '<eos>',\n",
        "            fix_length = max_length,\n",
        "            lower = True, \n",
        "            batch_first = True)\n",
        "\n",
        "TRG = Field(tokenize = tokenize_en, \n",
        "            init_token = '<sos>', \n",
        "            eos_token = '<eos>', \n",
        "            fix_length = max_length,\n",
        "            lower = True, \n",
        "            batch_first = True)\n",
        "\n",
        "ID = data.Field(sequential=False,use_vocab=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQjQf-DmsjhV",
        "outputId": "aa814327-d05b-45fc-cd78-5413cd6ff3a8"
      },
      "source": [
        "datafields=[('src', SRC), ('trg', TRG),('img_path',None),('img_id',ID)]\n",
        "print('**********************************************************\\ndatafields:', datafields)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "**********************************************************\n",
            "datafields: [('src', <torchtext.data.field.Field object at 0x7f78ee4b9ee0>), ('trg', <torchtext.data.field.Field object at 0x7f78ee4b9850>), ('img_path', None), ('img_id', <torchtext.data.field.Field object at 0x7f78ee4b9f10>)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3HYQxRE4sjhW"
      },
      "source": [
        "#df = pd.read_csv(\"data1.csv\")\n",
        "\n",
        "#rng = RandomState()\n",
        "#train_data = df.sample(frac=0.80, random_state=rng)\n",
        "#val_data = df.loc[~df.index.isin(train_data.index)]\n",
        "\n",
        "##cols= [\"src\", \"trg\", \"img_path\", \"img_id\"]\n",
        "#train_data.to_csv('train_data1.csv', index= False) #columns=cols\n",
        "#val_data.to_csv('val_data1.csv', index= False) #columns=cols"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pJZKfrAjsjhW"
      },
      "source": [
        "train_data, val_data = data.TabularDataset.splits(path=r\"\",train=\"temp/train_data1.csv\", validation=\"temp/val_data1.csv\", format='csv', skip_header=True, fields=datafields)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bDppMAVIsjhW",
        "outputId": "fe089c25-ed4b-4339-cd0d-b372cf2d3edd"
      },
      "source": [
        "print('\\ntrain_data length: ',len(train_data))\n",
        "print('val_data length: ',len(val_data))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "train_data length:  50960\n",
            "val_data length:  13601\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4XpLghW4sjhX"
      },
      "source": [
        "#print(vars(train_data.examples[4]))\n",
        "#print(sys.maxsize)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XVbL_nQ6sjhX"
      },
      "source": [
        "SRC.build_vocab(train_data, vectors=\"glove.6B.100d\")\n",
        "TRG.build_vocab(train_data, vectors=\"glove.6B.100d\")\n",
        "SRC.build_vocab(train_data, min_freq=2)\n",
        "TRG.build_vocab(train_data, min_freq=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E5dcvw3xsjhX",
        "outputId": "c757e64e-ea48-4c2a-efa3-d5c1c4d99e4d"
      },
      "source": [
        "print(f\"\\nUnique tokens in source (en) vocabulary: {len(SRC.vocab)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Unique tokens in source (en) vocabulary: 32349\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wxOqUboVsjhY",
        "outputId": "4e3686f0-40a7-48ff-8550-4138231a7240"
      },
      "source": [
        "print(f\"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique tokens in target (en) vocabulary: 42325\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oQyANbgmsjhY"
      },
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "#device = 'cuda'\n",
        "#torch.cuda.set_device(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fnxawfNosjhY",
        "outputId": "4a35bb22-5c03-4c9a-da4b-e5b5f1e7cfd4"
      },
      "source": [
        "device"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqPYTJHysjhY",
        "outputId": "aa453fc5-b01a-4f4f-fa56-0816d92594d1"
      },
      "source": [
        "BATCH_SIZE = 32\n",
        "\n",
        "train_iterator, x_iterator = BucketIterator.splits(\n",
        "    (train_data,train_data), \n",
        "    batch_size = BATCH_SIZE,\n",
        "    device = device)\n",
        "\n",
        "valid_iterator, y_iterator = BucketIterator.splits(\n",
        "    (val_data,val_data), \n",
        "    batch_size = BATCH_SIZE, \n",
        "    device = device)\n",
        "\n",
        "'''\n",
        "train_iterator, x_iterator = BucketIterator.splits(\n",
        "    (train_data, train_data), \n",
        "    batch_size = BATCH_SIZE,\n",
        "    #sort_within_batch = True,\n",
        "    #sort_key = lambda x : len(x.src), \n",
        "    device = device)\n",
        "\n",
        "valid_iterator, y_iterator = BucketIterator.splits(\n",
        "    (val_data, val_data), \n",
        "    batch_size = BATCH_SIZE, \n",
        "    #sort_within_batch = True,\n",
        "    #sort_key = lambda x : len(x.src),\n",
        "    device = device\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\ntrain_iterator, x_iterator = BucketIterator.splits(\\n    (train_data, train_data), \\n    batch_size = BATCH_SIZE,\\n    #sort_within_batch = True,\\n    #sort_key = lambda x : len(x.src), \\n    device = device)\\n\\nvalid_iterator, y_iterator = BucketIterator.splits(\\n    (val_data, val_data), \\n    batch_size = BATCH_SIZE, \\n    #sort_within_batch = True,\\n    #sort_key = lambda x : len(x.src),\\n    device = device\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bHxxlfaYsjhZ",
        "outputId": "d9e71154-2305-4583-e49c-78eaf920e1fa"
      },
      "source": [
        "print('\\ntrain_iterator length: ',len(train_iterator))\n",
        "print('valid_iterator length: ',len(valid_iterator))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "train_iterator length:  1593\n",
            "valid_iterator length:  426\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9-r1VP6DsjhZ"
      },
      "source": [
        "for i, batch in enumerate(train_iterator):\n",
        "  print(i,batch)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XHyv2ReUsjhZ"
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, \n",
        "                 input_dim, \n",
        "                 hid_dim, \n",
        "                 n_layers, \n",
        "                 n_heads, \n",
        "                 pf_dim,\n",
        "                 dropout, \n",
        "                 device,\n",
        "                 max_length = max_length):\n",
        "        super().__init__()\n",
        "\n",
        "        self.device = device\n",
        "        \n",
        "        self.tok_embedding = nn.Embedding(input_dim, hid_dim)\n",
        "        self.vis_fc=nn.Linear(NF, hid_dim)\n",
        "        self.fc=nn.Linear(2*hid_dim,hid_dim)\n",
        "        self.pos_embedding = nn.Embedding(max_length, hid_dim)\n",
        "        \n",
        "        self.layers = nn.ModuleList([EncoderLayer(hid_dim, \n",
        "                                                  n_heads, \n",
        "                                                  pf_dim,\n",
        "                                                  dropout, \n",
        "                                                  device) \n",
        "                                     for _ in range(n_layers)])\n",
        "        \n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "        self.enc_ff_layer_norm = nn.LayerNorm(hid_dim)\n",
        "        \n",
        "        self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)\n",
        "\n",
        "        self.enc_positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim, \n",
        "                                                                     pf_dim, \n",
        "                                                                     dropout)\n",
        "        #self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self, src,vs102,vs103, src_mask):\n",
        "        \n",
        "        #src = [batch size, src len]\n",
        "        #src_mask = [batch size, src len]\n",
        "        \n",
        "        batch_size = src.shape[0]\n",
        "        src_len = src.shape[1]\n",
        "        \n",
        "        vs102=torch.tanh(self.vis_fc(vs102))\n",
        "        vs103=torch.tanh(self.vis_fc(vs103))\n",
        "        \n",
        "        #print('[vs102.shape]',vs102.shape)\n",
        "        \n",
        "        pos = torch.arange(0, src_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)\n",
        "        \n",
        "        #pos = [batch size, src len]\n",
        "        shape=self.tok_embedding(src).shape\n",
        "        \n",
        "        #print('[self.tok_embedding(src).shape]',self.tok_embedding(src).shape)\n",
        "        \n",
        "        #if(shape[1]==102):\n",
        "        #    src=self.fc(torch.cat([self.tok_embedding(src),vs102],2))\n",
        "        #if(shape[1]==103):\n",
        "        #    src=self.fc(torch.cat([self.tok_embedding(src),vs103],2))    \n",
        "        \n",
        "        #print('src',src)\n",
        "        #print('.......................\\n[src.shape]',src.shape)\n",
        "        #print(pos.shape)\n",
        "        #print('[embedding shape]',self.pos_embedding(pos).shape)\n",
        "        \n",
        "        #src=self.fc(torch.cat([self.tok_embedding(src),visual_features],2))\n",
        "        #src = self.dropout((src * self.scale) + self.pos_embedding(pos))\n",
        "        \n",
        "        src = self.dropout((self.tok_embedding(src) * self.scale) + self.pos_embedding(pos))\n",
        "        \n",
        "        #src = [batch size, src len, hid dim]\n",
        "        \n",
        "        for layer in self.layers:\n",
        "            src = layer(src, src_mask)\n",
        "\n",
        "\n",
        "        if(shape[1]==102):\n",
        "            vis_src=self.fc(torch.cat([src,vs102],2))\n",
        "        if(shape[1]==103):\n",
        "            vis_src=self.fc(torch.cat([src,vs103],2)) \n",
        "\n",
        "        #positionwise feedforward\n",
        "        _vis_src = self.enc_positionwise_feedforward(vis_src)\n",
        "        \n",
        "        #dropout, residual and layer norm\n",
        "        vis_src = self.enc_ff_layer_norm(vis_src + self.dropout(_vis_src))    \n",
        "\n",
        "        #src = [batch size, src len, hid dim]\n",
        "\n",
        "            \n",
        "        return src, vis_src"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PinRWvwjsjha"
      },
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "    def __init__(self, \n",
        "                 hid_dim, \n",
        "                 n_heads, \n",
        "                 pf_dim,  \n",
        "                 dropout, \n",
        "                 device):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.self_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
        "        self.ff_layer_norm = nn.LayerNorm(hid_dim)\n",
        "        self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
        "        self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim, \n",
        "                                                                     pf_dim, \n",
        "                                                                     dropout)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self, src, src_mask):\n",
        "        \n",
        "        #src = [batch size, src len, hid dim]\n",
        "        #src_mask = [batch size, src len]\n",
        "                \n",
        "        #self attention\n",
        "        _src, _ = self.self_attention(src, src, src, src_mask)\n",
        "        \n",
        "        #dropout, residual connection and layer norm\n",
        "        src = self.self_attn_layer_norm(src + self.dropout(_src))\n",
        "        \n",
        "        #src = [batch size, src len, hid dim]\n",
        "        \n",
        "        #positionwise feedforward\n",
        "        _src = self.positionwise_feedforward(src)\n",
        "        \n",
        "        #dropout, residual and layer norm\n",
        "        src = self.ff_layer_norm(src + self.dropout(_src))\n",
        "        \n",
        "        #src = [batch size, src len, hid dim]\n",
        "        \n",
        "        return src"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0SyNBEvVsjha"
      },
      "source": [
        "class MultiHeadAttentionLayer(nn.Module):\n",
        "    def __init__(self, hid_dim, n_heads, dropout, device):\n",
        "        super().__init__()\n",
        "        \n",
        "        assert hid_dim % n_heads == 0\n",
        "        \n",
        "        self.hid_dim = hid_dim\n",
        "        self.n_heads = n_heads\n",
        "        self.head_dim = hid_dim // n_heads\n",
        "        \n",
        "        self.fc_q = nn.Linear(hid_dim, hid_dim)\n",
        "        self.fc_k = nn.Linear(hid_dim, hid_dim)\n",
        "        self.fc_v = nn.Linear(hid_dim, hid_dim)\n",
        "        \n",
        "        self.fc_o = nn.Linear(hid_dim, hid_dim)\n",
        "        \n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "        self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(device)\n",
        "        \n",
        "    def forward(self, query, key, value, mask = None):\n",
        "        \n",
        "        batch_size = query.shape[0]\n",
        "        \n",
        "        #query = [batch size, query len, hid dim]\n",
        "        #key = [batch size, key len, hid dim]\n",
        "        #value = [batch size, value len, hid dim]\n",
        "                \n",
        "        Q = self.fc_q(query)\n",
        "        K = self.fc_k(key)\n",
        "        V = self.fc_v(value)\n",
        "        \n",
        "        #Q = [batch size, query len, hid dim]\n",
        "        #K = [batch size, key len, hid dim]\n",
        "        #V = [batch size, value len, hid dim]\n",
        "                \n",
        "        Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
        "        K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
        "        V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
        "        \n",
        "        #Q = [batch size, n heads, query len, head dim]\n",
        "        #K = [batch size, n heads, key len, head dim]\n",
        "        #V = [batch size, n heads, value len, head dim]\n",
        "                \n",
        "        energy = torch.matmul(Q, K.permute(0, 1, 3, 2)) / self.scale\n",
        "        \n",
        "        #energy = [batch size, n heads, query len, key len]\n",
        "        \n",
        "        if mask is not None:\n",
        "            energy = energy.masked_fill(mask == 0, -1e10)\n",
        "        \n",
        "        attention = torch.softmax(energy, dim = -1)\n",
        "                \n",
        "        #attention = [batch size, n heads, query len, key len]\n",
        "                \n",
        "        x = torch.matmul(self.dropout(attention), V)\n",
        "        \n",
        "        #x = [batch size, n heads, query len, head dim]\n",
        "        \n",
        "        x = x.permute(0, 2, 1, 3).contiguous()\n",
        "        \n",
        "        #x = [batch size, query len, n heads, head dim]\n",
        "        \n",
        "        x = x.view(batch_size, -1, self.hid_dim)\n",
        "        \n",
        "        #x = [batch size, query len, hid dim]\n",
        "        \n",
        "        x = self.fc_o(x)\n",
        "        \n",
        "        #x = [batch size, query len, hid dim]\n",
        "        \n",
        "        return x, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w405yNPXsjhb"
      },
      "source": [
        "class PositionwiseFeedforwardLayer(nn.Module):\n",
        "    def __init__(self, hid_dim, pf_dim, dropout):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.fc_1 = nn.Linear(hid_dim, pf_dim)\n",
        "        self.fc_2 = nn.Linear(pf_dim, hid_dim)\n",
        "        \n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        \n",
        "        #x = [batch size, seq len, hid dim]\n",
        "        \n",
        "        x = self.dropout(torch.relu(self.fc_1(x)))\n",
        "        \n",
        "        #x = [batch size, seq len, pf dim]\n",
        "        \n",
        "        x = self.fc_2(x)\n",
        "        \n",
        "        #x = [batch size, seq len, hid dim]\n",
        "        \n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oTnYhiJtsjhb"
      },
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self, \n",
        "                 output_dim, \n",
        "                 hid_dim, \n",
        "                 n_layers, \n",
        "                 n_heads, \n",
        "                 pf_dim, \n",
        "                 dropout, \n",
        "                 device,\n",
        "                 max_length = max_length):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.device = device\n",
        "        \n",
        "        self.tok_embedding = nn.Embedding(output_dim, hid_dim)\n",
        "        self.pos_embedding = nn.Embedding(max_length, hid_dim)\n",
        "        \n",
        "        self.layers = nn.ModuleList([DecoderLayer(hid_dim, \n",
        "                                                  n_heads, \n",
        "                                                  pf_dim, \n",
        "                                                  dropout, \n",
        "                                                  device)\n",
        "                                     for _ in range(n_layers)])\n",
        "        \n",
        "        self.fc_out = nn.Linear(hid_dim, output_dim)\n",
        "        \n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "        self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)\n",
        "        \n",
        "    def forward(self, trg, enc_src, trg_mask, src_mask, vs102, vs103, enc_vis_src):\n",
        "        \n",
        "        #trg = [batch size, trg len]\n",
        "        #enc_src = [batch size, src len, hid dim]\n",
        "        #trg_mask = [batch size, trg len]\n",
        "        #src_mask = [batch size, src len]\n",
        "                \n",
        "        batch_size = trg.shape[0]\n",
        "        trg_len = trg.shape[1]\n",
        "        \n",
        "        pos = torch.arange(0, trg_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)\n",
        "                            \n",
        "        #pos = [batch size, trg len]\n",
        "            \n",
        "        trg = self.dropout((self.tok_embedding(trg) * self.scale) + self.pos_embedding(pos))\n",
        "                \n",
        "        #trg = [batch size, trg len, hid dim]\n",
        "        \n",
        "        for layer in self.layers:\n",
        "            trg, attention = layer(trg, enc_src, trg_mask, src_mask,  vs102, vs103, enc_vis_src)\n",
        "        \n",
        "        #trg = [batch size, trg len, hid dim]\n",
        "        #attention = [batch size, n heads, trg len, src len]\n",
        "        \n",
        "        output = self.fc_out(trg)\n",
        "        \n",
        "        #output = [batch size, trg len, output dim]\n",
        "            \n",
        "        return output, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PDk-xYXdsjhc"
      },
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "    def __init__(self, \n",
        "                 hid_dim, \n",
        "                 n_heads, \n",
        "                 pf_dim, \n",
        "                 dropout, \n",
        "                 device):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.vis_fc=nn.Linear(NF, hid_dim)\n",
        "\n",
        "        self.self_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
        "        self.enc_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
        "        self.enc_vis_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
        "        self.ff_layer_norm = nn.LayerNorm(hid_dim)\n",
        "        self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
        "        self.encoder_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
        "        self.encoder_vis_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
        "        self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim, \n",
        "                                                                     pf_dim, \n",
        "                                                                     dropout)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self, trg, enc_src, trg_mask, src_mask, vs102, vs103, enc_vis_src):\n",
        "        \n",
        "        #trg = [batch size, trg len, hid dim]\n",
        "        #enc_src = [batch size, src len, hid dim]\n",
        "        #trg_mask = [batch size, trg len]\n",
        "        #src_mask = [batch size, src len]\n",
        "\n",
        "        #vs102=torch.tanh(self.vis_fc(vs102))\n",
        "        #vs103=torch.tanh(self.vis_fc(vs103))\n",
        "\n",
        "\n",
        "        \n",
        "        #self attention\n",
        "        _trg, _ = self.self_attention(trg, trg, trg, trg_mask)\n",
        "        \n",
        "        #dropout, residual connection and layer norm\n",
        "        trg = self.self_attn_layer_norm(trg + self.dropout(_trg))\n",
        "            \n",
        "        #trg = [batch size, trg len, hid dim]\n",
        "            \n",
        "        #encoder attention\n",
        "        _trg, attention = self.encoder_attention(trg, enc_src, enc_src, src_mask)\n",
        "        \n",
        "        #dropout, residual connection and layer norm\n",
        "        trg = self.enc_attn_layer_norm(trg + self.dropout(_trg))\n",
        "\n",
        "        #encoder visual attention\n",
        "        _trg, attention = self.encoder_vis_attention(trg, enc_vis_src, enc_vis_src, src_mask)\n",
        "        \n",
        "        #dropout, residual connection and layer norm\n",
        "        trg = self.enc_vis_attn_layer_norm(trg + self.dropout(_trg))\n",
        "                    \n",
        "        #trg = [batch size, trg len, hid dim]\n",
        "        \n",
        "        #positionwise feedforward\n",
        "        _trg = self.positionwise_feedforward(trg)\n",
        "        \n",
        "        #dropout, residual and layer norm\n",
        "        trg = self.ff_layer_norm(trg + self.dropout(_trg))\n",
        "        \n",
        "        #trg = [batch size, trg len, hid dim]\n",
        "        #attention = [batch size, n heads, trg len, src len]\n",
        "        \n",
        "        return trg, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jHP1fLOIsjhc"
      },
      "source": [
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self, \n",
        "                 encoder, \n",
        "                 decoder, \n",
        "                 src_pad_idx, \n",
        "                 trg_pad_idx, \n",
        "                 device):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.src_pad_idx = src_pad_idx\n",
        "        self.trg_pad_idx = trg_pad_idx\n",
        "        self.device = device\n",
        "        \n",
        "    def make_src_mask(self, src):\n",
        "        \n",
        "        #src = [batch size, src len]\n",
        "        \n",
        "        src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)\n",
        "\n",
        "        #src_mask = [batch size, 1, 1, src len]\n",
        "\n",
        "        return src_mask\n",
        "    \n",
        "    def make_trg_mask(self, trg):\n",
        "        \n",
        "        #trg = [batch size, trg len]\n",
        "        \n",
        "        trg_pad_mask = (trg != self.trg_pad_idx).unsqueeze(1).unsqueeze(2)\n",
        "        \n",
        "        #trg_pad_mask = [batch size, 1, 1, trg len]\n",
        "        \n",
        "        trg_len = trg.shape[1]\n",
        "        \n",
        "        trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len), device = self.device)).bool()\n",
        "        \n",
        "        #trg_sub_mask = [trg len, trg len]\n",
        "            \n",
        "        trg_mask = trg_pad_mask & trg_sub_mask\n",
        "        \n",
        "        #trg_mask = [batch size, 1, trg len, trg len]\n",
        "        \n",
        "        return trg_mask\n",
        "\n",
        "    def forward(self, src,vs102,vs103, trg):\n",
        "        \n",
        "        #src = [batch size, src len]\n",
        "        #trg = [batch size, trg len]\n",
        "                \n",
        "        src_mask = self.make_src_mask(src)\n",
        "        trg_mask = self.make_trg_mask(trg)\n",
        "        \n",
        "        #src_mask = [batch size, 1, 1, src len]\n",
        "        #trg_mask = [batch size, 1, trg len, trg len]\n",
        "        \n",
        "        enc_src, vis_enc_src = self.encoder(src,vs102,vs103, src_mask)\n",
        "        \n",
        "        #enc_src = [batch size, src len, hid dim]\n",
        "                \n",
        "        output, attention = self.decoder(trg, enc_src, trg_mask, src_mask, vs102, vs103, vis_enc_src)\n",
        "        \n",
        "        #output = [batch size, trg len, output dim]\n",
        "        #attention = [batch size, n heads, trg len, src len]\n",
        "        \n",
        "        return output, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wOT_awB_sjhd"
      },
      "source": [
        "INPUT_DIM = len(SRC.vocab)\n",
        "OUTPUT_DIM = len(TRG.vocab)\n",
        "HID_DIM = 64\n",
        "ENC_LAYERS = 3\n",
        "DEC_LAYERS = 3\n",
        "ENC_HEADS = 8\n",
        "DEC_HEADS = 8\n",
        "ENC_PF_DIM = 128\n",
        "DEC_PF_DIM = 128\n",
        "ENC_DROPOUT = 0.1\n",
        "DEC_DROPOUT = 0.1\n",
        "\n",
        "enc = Encoder(INPUT_DIM, \n",
        "              HID_DIM, \n",
        "              ENC_LAYERS, \n",
        "              ENC_HEADS, \n",
        "              ENC_PF_DIM, \n",
        "              ENC_DROPOUT, \n",
        "              device)\n",
        "\n",
        "dec = Decoder(OUTPUT_DIM, \n",
        "              HID_DIM, \n",
        "              DEC_LAYERS, \n",
        "              DEC_HEADS, \n",
        "              DEC_PF_DIM, \n",
        "              DEC_DROPOUT, \n",
        "              device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c2ypjvCHsjhd"
      },
      "source": [
        "SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token]\n",
        "TRG_PAD_IDX = TRG.vocab.stoi[TRG.pad_token]\n",
        "\n",
        "model = Seq2Seq(enc, dec, SRC_PAD_IDX, TRG_PAD_IDX, device).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HJqOQTI7sjhd",
        "outputId": "e0c1ec15-ac87-4f40-cf11-82418a6f27fa"
      },
      "source": [
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'\\nThe model has {count_parameters(model):,} trainable parameters')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "The model has 12,521,365 trainable parameters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RCy2qrbnsjhd"
      },
      "source": [
        "def initialize_weights(m):\n",
        "    if hasattr(m, 'weight') and m.weight.dim() > 1:\n",
        "        nn.init.xavier_uniform_(m.weight.data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a10pWW2Wsjhe"
      },
      "source": [
        "model.apply(initialize_weights);"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dzQf_0URsjhe"
      },
      "source": [
        "LEARNING_RATE = 0.0005\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = TRG_PAD_IDX)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eYSUFNMDsjhe"
      },
      "source": [
        "def train(model, iterator, optimizer, criterion, clip):\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    \n",
        "    for i, batch in enumerate(iterator):\n",
        "        \n",
        "        src = batch.src\n",
        "        trg = batch.trg\n",
        "        img_id=batch.img_id\n",
        "\n",
        "        x=img_id.cpu().numpy()\n",
        "        y=len(x)\n",
        "#         print(y)\n",
        "        visual_features_102=torch.empty(y,max_length,NF).cuda()\n",
        "        visual_features_103=torch.empty(y,max_length,NF).cuda()\n",
        "        #visual_features_102=torch.empty(y,max_length+2,1000).cuda()\n",
        "        #visual_features_103=torch.empty(y,max_length+3,1000).cuda()\n",
        "        \n",
        "        df=pd.read_csv(r\"visual_features_rcnn.csv\")\n",
        "        \n",
        "        for i in range(y):\n",
        "            q=df[str(x[i])].to_numpy()\n",
        "#            print(q)\n",
        "#            print(len(q))\n",
        "            r=np.zeros((max_length,1),dtype=q.dtype) + q\n",
        "            s=np.zeros((max_length,1),dtype=q.dtype) + q\n",
        "            #r=np.zeros((max_length+2,1),dtype=q.dtype) + q\n",
        "            #s=np.zeros((max_length+3,1),dtype=q.dtype) + q\n",
        "            r=torch.from_numpy(r).float()\n",
        "            s=torch.from_numpy(s).float()\n",
        "            visual_features_102[i]=r\n",
        "            visual_features_103[i]=s\n",
        "            \n",
        "        vs102=visual_features_102.float()\n",
        "        vs103=visual_features_103.float()\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "#        print(src.size())\n",
        "#        print(trg.size())\n",
        "        \n",
        "#        print(trg.size()[1])\n",
        "#        print(src.size()[1])\n",
        "        \n",
        "#        if(src.size()[1]==104 or src.size()[1]==105):\n",
        "#            src.size()[1] = 103\n",
        "#            print('Delete the samples with image id:', img_id)\n",
        "        \n",
        "        sh=src.shape[1]\n",
        "        #print('\\n[src.shape in train()]',sh)\n",
        "        output, _ = model(src, vs102,vs103,trg[:,:-1])\n",
        "                \n",
        "        #output = [batch size, trg len - 1, output dim]\n",
        "        #trg = [batch size, trg len]\n",
        "            \n",
        "        output_dim = output.shape[-1]\n",
        "            \n",
        "        output = output.contiguous().view(-1, output_dim)\n",
        "        trg = trg[:,1:].contiguous().view(-1)\n",
        "                \n",
        "        #output = [batch size * trg len - 1, output dim]\n",
        "        #trg = [batch size * trg len - 1]\n",
        "            \n",
        "        loss = criterion(output, trg)\n",
        "        \n",
        "        loss.backward()\n",
        "        \n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
        "        \n",
        "        optimizer.step()\n",
        "        \n",
        "        epoch_loss += loss.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KzCFZ1aMsjhe"
      },
      "source": [
        "def evaluate(model, iterator, criterion):\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    \n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for i, batch in enumerate(iterator):\n",
        "\n",
        "            src = batch.src\n",
        "            trg = batch.trg\n",
        "            img_id=batch.img_id\n",
        "\n",
        "            x=img_id.cpu().numpy()\n",
        "            y=len(x)\n",
        "            visual_features_102=torch.empty(y,max_length,NF).cuda()\n",
        "            visual_features_103=torch.empty(y,max_length,NF).cuda()\n",
        "            #visual_features_102=torch.empty(y,max_length+2,1000).cuda()\n",
        "            #visual_features_103=torch.empty(y,max_length+3,1000).cuda()\n",
        "\n",
        "            df=pd.read_csv(r\"visual_features_rcnn.csv\")\n",
        "\n",
        "            for i in range(y):\n",
        "                q=df[str(x[i])].to_numpy()\n",
        "\n",
        "                r=np.zeros((max_length,1),dtype=q.dtype) + q\n",
        "                s=np.zeros((max_length,1),dtype=q.dtype) + q\n",
        "                #r=np.zeros((max_length+2,1),dtype=q.dtype) + q\n",
        "                #s=np.zeros((max_length+3,1),dtype=q.dtype) + q\n",
        "                r=torch.from_numpy(r).float()\n",
        "                s=torch.from_numpy(s).float()\n",
        "                visual_features_102[i]=r\n",
        "                visual_features_103[i]=s\n",
        "\n",
        "            vs102=visual_features_102.float()\n",
        "            vs103=visual_features_103.float()\n",
        "\n",
        "            output, _ = model(src,vs102,vs103, trg[:,:-1])\n",
        "            \n",
        "            #output = [batch size, trg len - 1, output dim]\n",
        "            #trg = [batch size, trg len]\n",
        "            \n",
        "            output_dim = output.shape[-1]\n",
        "            \n",
        "            output = output.contiguous().view(-1, output_dim)\n",
        "            trg = trg[:,1:].contiguous().view(-1)\n",
        "            \n",
        "            #output = [batch size * trg len - 1, output dim]\n",
        "            #trg = [batch size * trg len - 1]\n",
        "            \n",
        "            loss = criterion(output, trg)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3SOkroWmsjhf"
      },
      "source": [
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h-bPYeqosjhf"
      },
      "source": [
        "def load_checkpt(model, optimizer, chpt_file):\n",
        "    start_epoch = 0\n",
        "    if (os.path.exists(chpt_file)):\n",
        "        print(\"=> loading checkpoint '{}'\".format(chpt_file))\n",
        "        checkpoint = torch.load(chpt_file)\n",
        "        model.load_state_dict(checkpoint['state_dict'])\n",
        "        optimizer.load_state_dict(checkpoint['optimizer'])\n",
        "        start_epoch = checkpoint['epoch']\n",
        "        \n",
        "        print(\"=> loaded checkpoint '{}' (epoch {})\".format(chpt_file, checkpoint['epoch']))\n",
        "        \n",
        "    else:\n",
        "        print(\"=> Checkpoint NOT found '{}'\".format(chpt_file))\n",
        "    return model, optimizer, start_epoch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VerI6Rr1sjhf"
      },
      "source": [
        "def translate_sentence(sentence, src_field, trg_field,img_id,path, model, device, max_len = 50):\n",
        "    \n",
        "    model.eval()\n",
        "        \n",
        "    if isinstance(sentence, str):\n",
        "        nlp = spacy.load('en_core_web_sm')\n",
        "        #nlp = spacy.load('en') ##https://www.gitmemory.com/issue/OmkarPathak/pyresparser/46/777568505\n",
        "        tokens = [token.text.lower() for token in nlp(sentence)]\n",
        "    else:\n",
        "        tokens = [token.lower() for token in sentence]\n",
        "\n",
        "    tokens = [src_field.init_token] + tokens + [src_field.eos_token]\n",
        "        \n",
        "    src_indexes = [src_field.vocab.stoi[token] for token in tokens]\n",
        "\n",
        "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(0).to(device)\n",
        "    \n",
        "    src_mask = model.make_src_mask(src_tensor)\n",
        "    \n",
        "    df=pd.read_csv(path)\n",
        "    visual_features_102=torch.empty(1,max_length,NF).cuda()\n",
        "    visual_features_103=torch.empty(1,max_length,NF).cuda()\n",
        "    #visual_features_102=torch.empty(1,max_length+2,1000).cuda()\n",
        "    #visual_features_103=torch.empty(1,max_length+3,1000).cuda()\n",
        "    q=df[str(img_id)].to_numpy()\n",
        "    r=np.zeros((max_length,1),dtype=q.dtype) + q\n",
        "    s=np.zeros((max_length,1),dtype=q.dtype) + q\n",
        "    #r=np.zeros((max_length+2,1),dtype=q.dtype) + q\n",
        "    #s=np.zeros((max_length+3,1),dtype=q.dtype) + q\n",
        "    r=torch.from_numpy(r).float()\n",
        "    s=torch.from_numpy(s).float()\n",
        "    visual_features_102[0]=r\n",
        "    visual_features_103[0]=s\n",
        "    \n",
        "    \n",
        "    #print('\\n[src_tensor.shape]',src_tensor.shape)\n",
        "    #print('\\n[src_tensor.shape[1]]',src_tensor.shape[1])\n",
        "    #print('[visual_features_102.shape]',visual_features_102.shape)\n",
        "    #print('[visual_features_103.shape]',visual_features_103.shape)\n",
        "    #print('[src_mask.shape]',src_mask.shape)\n",
        "    \n",
        "    with torch.no_grad():\n",
        "        enc_src, enc_vis_src = model.encoder(src_tensor,visual_features_102,visual_features_103, src_mask)\n",
        "\n",
        "    trg_indexes = [trg_field.vocab.stoi[trg_field.init_token]]\n",
        "\n",
        "    for i in range(max_len):\n",
        "\n",
        "        trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)\n",
        "\n",
        "        trg_mask = model.make_trg_mask(trg_tensor)\n",
        "        \n",
        "        with torch.no_grad():\n",
        "            output, attention = model.decoder(trg_tensor, enc_src, trg_mask, src_mask, visual_features_102, visual_features_103, enc_vis_src)\n",
        "        \n",
        "        pred_token = output.argmax(2)[:,-1].item()\n",
        "        \n",
        "        trg_indexes.append(pred_token)\n",
        "\n",
        "        if pred_token == trg_field.vocab.stoi[trg_field.eos_token]:\n",
        "            break\n",
        "    \n",
        "    trg_tokens = [trg_field.vocab.itos[i] for i in trg_indexes]\n",
        "    \n",
        "    return trg_tokens[1:], attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w4z-I4F7sjhf",
        "outputId": "ce432b58-d9ca-4172-823e-c99a33e66e65"
      },
      "source": [
        "'''def display_attention(sentence, translation, attention, n_heads = 8, n_rows = 4, n_cols = 2):\n",
        "    \n",
        "    assert n_rows * n_cols == n_heads\n",
        "    \n",
        "    fig = plt.figure(figsize=(15,25))\n",
        "    \n",
        "    for i in range(n_heads):\n",
        "        \n",
        "        ax = fig.add_subplot(n_rows, n_cols, i+1)\n",
        "        \n",
        "        _attention = attention.squeeze(0)[i].cpu().detach().numpy()\n",
        "\n",
        "        cax = ax.matshow(_attention, cmap='bone')\n",
        "\n",
        "        ax.tick_params(labelsize=12)\n",
        "        ax.set_xticklabels(['']+['<sos>']+[t.lower() for t in sentence]+['<eos>'], \n",
        "                           rotation=45)\n",
        "        ax.set_yticklabels(['']+translation)\n",
        "\n",
        "        ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "        ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "\n",
        "    plt.show()\n",
        "    plt.close()\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"def display_attention(sentence, translation, attention, n_heads = 8, n_rows = 4, n_cols = 2):\\n    \\n    assert n_rows * n_cols == n_heads\\n    \\n    fig = plt.figure(figsize=(15,25))\\n    \\n    for i in range(n_heads):\\n        \\n        ax = fig.add_subplot(n_rows, n_cols, i+1)\\n        \\n        _attention = attention.squeeze(0)[i].cpu().detach().numpy()\\n\\n        cax = ax.matshow(_attention, cmap='bone')\\n\\n        ax.tick_params(labelsize=12)\\n        ax.set_xticklabels(['']+['<sos>']+[t.lower() for t in sentence]+['<eos>'], \\n                           rotation=45)\\n        ax.set_yticklabels(['']+translation)\\n\\n        ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\\n        ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\\n\\n    plt.show()\\n    plt.close()\\n\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9WGukDNZsjhg",
        "outputId": "773b3f15-ced5-40f9-fa87-d682957ee2e3"
      },
      "source": [
        "'''text_idx = 25                #renamed from 'example_idx'\n",
        "image_idx=\"9\"\n",
        "\n",
        "path=r\"visual_features_rcnn.csv\"\n",
        "\n",
        "src = vars(train_data.examples[text_idx])['src']\n",
        "trg = vars(train_data.examples[text_idx])['trg']\n",
        "#img_path = vars(train_data.examples[text_idx])['img_path']\n",
        "\n",
        "\n",
        "translation, attention = translate_sentence(src, SRC, TRG, image_idx, path, model, device)\n",
        "\n",
        "#print(f'src = {src}\\n')\n",
        "print(f'Ground-truth Comment')\n",
        "print( '--------------------')\n",
        "print(f'trg = {trg}')\n",
        "\n",
        "print(f'\\n\\nPredicted Feedback (Feedback)')\n",
        "print( '-----------------------------')\n",
        "    \n",
        "print(f'{translation}')\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'text_idx = 25                #renamed from \\'example_idx\\'\\nimage_idx=\"9\"\\n\\npath=r\"visual_features_rcnn.csv\"\\n\\nsrc = vars(train_data.examples[text_idx])[\\'src\\']\\ntrg = vars(train_data.examples[text_idx])[\\'trg\\']\\n#img_path = vars(train_data.examples[text_idx])[\\'img_path\\']\\n\\n\\ntranslation, attention = translate_sentence(src, SRC, TRG, image_idx, path, model, device)\\n\\n#print(f\\'src = {src}\\n\\')\\nprint(f\\'Ground-truth Comment\\')\\nprint( \\'--------------------\\')\\nprint(f\\'trg = {trg}\\')\\n\\nprint(f\\'\\n\\nPredicted Feedback (Feedback)\\')\\nprint( \\'-----------------------------\\')\\n    \\nprint(f\\'{translation}\\')\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bHtEWrDBsjhg"
      },
      "source": [
        "#display_attention(src, translation, attention)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uLIITQXGsjhg",
        "outputId": "991f11d7-c0eb-49ab-e0fb-b03fbfbc431d"
      },
      "source": [
        "dataset=pd.read_csv(r\"test_data.csv\")\n",
        "dataset.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(100, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3xNX3Ygxsjhg",
        "outputId": "f648d5af-451b-489b-cda6-89ce90ed3e60"
      },
      "source": [
        "for i in range (dataset.shape[0]):\n",
        "    if len(dataset['src'][i].split()) < 100:\n",
        "        dataset = dataset.drop(i)\n",
        "    else:\n",
        "        dataset['src'][i] = \" \".join(dataset['src'][i].split()[0:100]) #102?"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<ipython-input-55-56139961cbb3>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dataset['src'][i] = \" \".join(dataset['src'][i].split()[0:100]) #102?\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hig4ogqAsjhg"
      },
      "source": [
        "dataset.to_csv('temp/test_data2.csv', index= False) \n",
        "dataset = pd.read_csv('temp/test_data2.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zNQfoxNMsjhh"
      },
      "source": [
        "for i in range (dataset.shape[0]):\n",
        "    sentence = dataset['src'][i]\n",
        "    \n",
        "    if isinstance(sentence, str):\n",
        "        nlp = spacy.load('en_core_web_sm')\n",
        "        #nlp = spacy.load('en') #https://www.gitmemory.com/issue/OmkarPathak/pyresparser/46/777568505\n",
        "        tokens = [token.text.lower() for token in nlp(sentence)]\n",
        "    else:\n",
        "        tokens = [token.lower() for token in sentence]\n",
        "\n",
        "    tokens = [SRC.init_token] + tokens + [SRC.eos_token]\n",
        "        \n",
        "    src_indexes = [SRC.vocab.stoi[token] for token in tokens]\n",
        "\n",
        "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(0).to(device)\n",
        "\n",
        "    if (src_tensor.shape[1] != 102):\n",
        "        dataset = dataset.drop(i)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j3U3aE7Nsjhh",
        "outputId": "5fbac2f5-f2ec-4ec2-ef79-d037805f1cf5"
      },
      "source": [
        "dataset.to_csv('temp/test_data1.csv', index= False) \n",
        "dataset = pd.read_csv('temp/test_data1.csv')\n",
        "dataset.head(-1) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>src</th>\n",
              "      <th>trg</th>\n",
              "      <th>img_path</th>\n",
              "      <th>img_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>there can be few people who have not at some s...</td>\n",
              "      <td>most people want to be fooled</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/133...</td>\n",
              "      <td>133</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>federal and local law enforcement sources told...</td>\n",
              "      <td>a police officer has died too</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/174...</td>\n",
              "      <td>174</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>item one on the agenda according to the kremli...</td>\n",
              "      <td>from a german erspective the longing for sputn...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/227...</td>\n",
              "      <td>227</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>new delhi union health and family welfare mini...</td>\n",
              "      <td>need to learn to take care of your countrymen ...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/714...</td>\n",
              "      <td>714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>the tone of joe bidens first official press co...</td>\n",
              "      <td>yesterdays press conference was a complete emb...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/820...</td>\n",
              "      <td>820</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77</th>\n",
              "      <td>listen and subscribe to the daily apple podcas...</td>\n",
              "      <td>you have learned absolutely nothing about cent...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/894...</td>\n",
              "      <td>8948</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78</th>\n",
              "      <td>file photo feb bradenton florida usa brooks ko...</td>\n",
              "      <td>georgia all who play will be remembered as tra...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/916...</td>\n",
              "      <td>9161</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79</th>\n",
              "      <td>former presidents barack obama and donald trum...</td>\n",
              "      <td>mlb forgot to take the team with them the brav...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/932...</td>\n",
              "      <td>9327</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>for new yorkers a pandemic years fight for the...</td>\n",
              "      <td>for football betting fans search for the chann...</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/935...</td>\n",
              "      <td>9353</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>ian haydon helped test modernas coronavirus va...</td>\n",
              "      <td>more than likely it will be</td>\n",
              "      <td>/home/puneet/code/Multimodal Feedback/data/939...</td>\n",
              "      <td>9399</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>82 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                  src  \\\n",
              "0   there can be few people who have not at some s...   \n",
              "1   federal and local law enforcement sources told...   \n",
              "2   item one on the agenda according to the kremli...   \n",
              "3   new delhi union health and family welfare mini...   \n",
              "4   the tone of joe bidens first official press co...   \n",
              "..                                                ...   \n",
              "77  listen and subscribe to the daily apple podcas...   \n",
              "78  file photo feb bradenton florida usa brooks ko...   \n",
              "79  former presidents barack obama and donald trum...   \n",
              "80  for new yorkers a pandemic years fight for the...   \n",
              "81  ian haydon helped test modernas coronavirus va...   \n",
              "\n",
              "                                                  trg  \\\n",
              "0                       most people want to be fooled   \n",
              "1                       a police officer has died too   \n",
              "2   from a german erspective the longing for sputn...   \n",
              "3   need to learn to take care of your countrymen ...   \n",
              "4   yesterdays press conference was a complete emb...   \n",
              "..                                                ...   \n",
              "77  you have learned absolutely nothing about cent...   \n",
              "78  georgia all who play will be remembered as tra...   \n",
              "79  mlb forgot to take the team with them the brav...   \n",
              "80  for football betting fans search for the chann...   \n",
              "81                        more than likely it will be   \n",
              "\n",
              "                                             img_path  img_id  \n",
              "0   /home/puneet/code/Multimodal Feedback/data/133...     133  \n",
              "1   /home/puneet/code/Multimodal Feedback/data/174...     174  \n",
              "2   /home/puneet/code/Multimodal Feedback/data/227...     227  \n",
              "3   /home/puneet/code/Multimodal Feedback/data/714...     714  \n",
              "4   /home/puneet/code/Multimodal Feedback/data/820...     820  \n",
              "..                                                ...     ...  \n",
              "77  /home/puneet/code/Multimodal Feedback/data/894...    8948  \n",
              "78  /home/puneet/code/Multimodal Feedback/data/916...    9161  \n",
              "79  /home/puneet/code/Multimodal Feedback/data/932...    9327  \n",
              "80  /home/puneet/code/Multimodal Feedback/data/935...    9353  \n",
              "81  /home/puneet/code/Multimodal Feedback/data/939...    9399  \n",
              "\n",
              "[82 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dj4WMEvKsjhh",
        "outputId": "1a4a4f6d-e63d-46b6-9a61-483efa9383a5"
      },
      "source": [
        "for i in range (dataset.shape[0]):\n",
        "     res = re.findall(r'\\w+', dataset[\"src\"][i])\n",
        "     #print(res)\n",
        "     kk=min(len(res),100)\n",
        "     if (kk<100):\n",
        "            print(i,\" \",len(res),\" \",kk,\" \", dataset[\"img_id\"][i])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "57   98   98   8005\n",
            "77   98   98   8948\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6CcoxVPmsjhh"
      },
      "source": [
        "### Spice & Meteor\n",
        "#### (Hitting erorr in py3, need to run in py2...) tried switching the kernel to py2 and runnign here but T_EPOCHS is cleared off...\n",
        "#### 1. Go to '/home/puneet/code/EvalMetrics_py2'\n",
        "#### 2. Copy 'nEp_test_comments.csv' & 'nEp_test_feedbacks.csv'\n",
        "#### 3. Rename as 'test_comments.csv' & 'test_feedbacks.csv'\n",
        "#### 4. Run eval.py with 'py2' env\n",
        "#### 5. Rename the generated 'scores.txt' file  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UmkOOJjfsjhi"
      },
      "source": [
        "def quant_eval(model, optimizer, start_epoch, chpt_file):\n",
        "    from pycocoevalcap.bleu.bleu import Bleu\n",
        "    from pycocoevalcap.cider.cider import Cider\n",
        "    from pycocoevalcap.meteor.meteor import Meteor\n",
        "    from pycocoevalcap.rouge.rouge import Rouge\n",
        "    from pycocoevalcap.spice.spice import Spice\n",
        "    import os, json, csv\n",
        "    \n",
        "    model, optimizer, start_epoch = load_checkpt(model, optimizer, chpt_file)\n",
        "    T_EPOCHS = start_epoch + EP_INT    \n",
        "    \n",
        "\n",
        "    #print('Download Stanford models... Run once!')\n",
        "    os.system(\"sh get_stanford_models.sh\")\n",
        "\n",
        "    with open('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv',\"r\") as f: \n",
        "            reader = csv.reader(f)\n",
        "            gts = {rows[0]:rows[1:] for rows in reader}\n",
        "            #print(mydict) #prints with single quotes\n",
        "            #print (json.dumps(mydict)) #prints with double quotes\n",
        "\n",
        "    with open('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv',\"r\") as g: \n",
        "            reader = csv.reader(g)\n",
        "            res = {rows[0]:rows[1:] for rows in reader}\n",
        "            #print(json.dumps(mydict))\n",
        "\n",
        "    '''with open('temp/test_comments.json', 'r') as file:\n",
        "        gts = json.load(file)\n",
        "    with open('temp/test_feedbacks.json', 'r') as file:\n",
        "        res = json.load(file)\n",
        "    '''\n",
        "\n",
        "    def bleu():\n",
        "        scorer = Bleu(n=4)\n",
        "        score, scores = scorer.compute_score(gts, res)\n",
        "        return score\n",
        "\n",
        "\n",
        "    def cider():\n",
        "        scorer = Cider()\n",
        "        (score, scores) = scorer.compute_score(gts, res)\n",
        "        return score\n",
        "\n",
        "    def rouge():\n",
        "        scorer = Rouge()\n",
        "        score, scores = scorer.compute_score(gts, res)\n",
        "        return score\n",
        "\n",
        "    #bgts = gts[0].encode(encoding='UTF-8')\n",
        "    #bres = res[0].encode(encoding='UTF-8')\n",
        "\n",
        "    def spice():\n",
        "        scorer = Spice()\n",
        "        #print(gts, res)\n",
        "        score, scores = scorer.compute_score(gts, res)\n",
        "        return score\n",
        "\n",
        "    def meteor():\n",
        "        scorer = Meteor()\n",
        "        #print(gts, res)\n",
        "        score, scores = scorer.compute_score(bgts, bres)\n",
        "        return score    \n",
        "    s_cider=cider()\n",
        "    s_rouge=rouge()\n",
        "    s_bleu=bleu()\n",
        "    #s_spice=spice()#\n",
        "    #s_meteor=meteor()#\n",
        "    \n",
        "    print('\\n----------------------\\nbleu = %s' %s_bleu )\n",
        "    print('cider = %s' %s_cider )\n",
        "    print('rouge = %s' %s_rouge )\n",
        "    #print('spice = %s' %s_spice )\n",
        "    #print('meteor = %s' %s_meteor )\n",
        "    \n",
        "    b=\" \".join(str(x) for x in s_bleu)\n",
        "    print('\\n----------------------')\n",
        "    f = open('scores.txt', 'w') \n",
        "    f.write(\"\\ncider: %f\" % s_cider)\n",
        "    f.write(\"\\nrouge: %f\" % s_rouge)\n",
        "    #f.write(\"\\nspice: %f\" % s_spice)\n",
        "    #f.write(\"\\nmeteor: %f\" % s_meteor)\n",
        "    f.write(\"\\nbleu :\")\n",
        "    f.write(b)\n",
        "    f.close()\n",
        "    \n",
        "    #print(str(T_EPOCHS))\n",
        "    #Log with Tensorboard: Eval metrics\n",
        "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/cider', str(s_cider))\n",
        "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/rouge', str(s_rouge))\n",
        "    #log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/spice', str(s_spice))\n",
        "    #log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/meteor', str(s_meteor))\n",
        "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/bleu-1', str(s_bleu[0]))\n",
        "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/bleu-2', str(s_bleu[1]))\n",
        "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/bleu-3', str(s_bleu[2]))\n",
        "    log_writer.add_text(str(T_EPOCHS)+'Ep=>Metrics/bleu-4', str(s_bleu[3]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6oNnjh68sjhi"
      },
      "source": [
        "def qual_eval(model, optimizer, start_epoch, chpt_file):\n",
        "    model, optimizer, start_epoch = load_checkpt(model, optimizer, chpt_file)\n",
        "    T_EPOCHS = start_epoch + EP_INT\n",
        "    \n",
        "    # Save the predicted Feedbacks in CSV file\n",
        "    # Log with Tensorboard: Text, Comment, Image and Feedback\n",
        "    test_pred=[]\n",
        "    #test_df=pd.read_csv(\"temp/test_data.csv\")  \n",
        "    test_df=pd.read_csv(\"temp/test_data1.csv\") \n",
        "    path=r\"visual_features_rcnn.csv\"      \n",
        "    length=test_df.shape[0]\n",
        "    images = []\n",
        "\n",
        "    #print(length)\n",
        "    for i in range(length):\n",
        "        src=test_df['src'][i]\n",
        "        trg=test_df['trg'][i]\n",
        "        img_path=test_df['img_path'][i]\n",
        "        image_idx=test_df['img_id'][i]\n",
        "        translation, attention = translate_sentence(src, SRC, TRG, image_idx, path, model, device)\n",
        "\n",
        "        if not translation:\n",
        "            translation=\"*empty*\"\n",
        "\n",
        "        #Untokenization    \n",
        "        translation1=translation[0:(len(translation)-1)]    \n",
        "        translation2 = TreebankWordDetokenizer().detokenize(translation1)\n",
        "        test_pred.append(str(translation2))\n",
        "\n",
        "        image = Image.open(img_path)\n",
        "        image = ToTensor()(image)   \n",
        "        #images.append(image)\n",
        "        \n",
        "        #print(str(T_EPOCHS))\n",
        "        if (i%10==0): \n",
        "            #Log with Tensorboard: Text, Comment, Image and Feedback\n",
        "            log_writer.add_text(str(T_EPOCHS)+'Ep=>Ground-truth Comment of Sample/'+str(i+1), str(trg))\n",
        "            log_writer.add_text(str(T_EPOCHS)+'Ep=>News Text of Sample/'+str(i+1), str(src))#, i+1)\n",
        "            log_writer.add_text(str(T_EPOCHS)+'Ep=>Predicted Feedback of Sample/'+str(i+1), str(translation))\n",
        "            #log_writer.add_image('Image', image, i+1)\n",
        "            #image_grid = torchvision.utils.make_grid(images)\n",
        "            log_writer.add_image(str(T_EPOCHS)+'Ep:Image of Sample/'+str(i+1), image)        \n",
        "\n",
        "    with open('temp/'+str(T_EPOCHS)+'Ep_test_results.csv', 'w'): \n",
        "        pass\n",
        "    with open('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv', 'w'): \n",
        "        pass\n",
        "    with open('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv', 'w'): \n",
        "        pass\n",
        "\n",
        "    test_df[\"pred\"] = test_pred \n",
        "    test_df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_results.csv', index= False)\n",
        "    test_df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv', index= True, columns=[\"trg\"]) #index -> \"key\": value -> [\"trg/pred\"]\n",
        "    test_df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv', index= True, columns=[\"pred\"])\n",
        "\n",
        "    #Re-open and save with new column names\n",
        "    df = pd.read_csv('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv')\n",
        "    df.columns = ['id', 'comment']\n",
        "    df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_comments.csv', index= False)\n",
        "\n",
        "    df = pd.read_csv('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv')\n",
        "    df.columns = ['id', 'feedback']\n",
        "    df.to_csv('temp/'+str(T_EPOCHS)+'Ep_test_feedbacks.csv', index= False)\n",
        "    \n",
        "    print('tensorboard --logdir \"/home/puneet/code/Multimodal Feedback/TBlogs\"\\n---')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ysVTH8A1sjhk"
      },
      "source": [
        "def interval_train(model, optimizer, start_epoch, chpt_file):\n",
        "    model, optimizer, start_epoch = load_checkpt(model, optimizer, chpt_file)\n",
        "    T_EPOCHS = start_epoch + EP_INT\n",
        "\n",
        "    print('Already trained for',start_epoch, 'epochs. Training now for', EP_INT, 'more')\n",
        "\n",
        "    best_valid_loss = float('inf')\n",
        "    cur_best_train_loss = float('inf')\n",
        "\n",
        "    #for epoch in range(EP_INT):\n",
        "    for epoch in range(start_epoch, T_EPOCHS):    \n",
        "        start_time = time.time()\n",
        "\n",
        "        train_loss = train(model, train_iterator, optimizer, criterion, CLIP)#, log_writer_train)\n",
        "        valid_loss = evaluate(model, valid_iterator, criterion)#, log_writer_val)\n",
        "\n",
        "        #Log with Tensorboard: Loss & PPL for train & val \n",
        "        log_writer.add_scalar('Train/Loss',float(train_loss), epoch+1)\n",
        "        log_writer.add_scalar('Train/PPL', float(math.exp(train_loss)), epoch+1)    \n",
        "        log_writer.add_scalar('Val/Loss',float(valid_loss), epoch+1)\n",
        "        log_writer.add_scalar('Val/PPL', float(math.exp(valid_loss)), epoch+1)\n",
        "\n",
        "        end_time = time.time()\n",
        "\n",
        "        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "\n",
        "        #if valid_loss < best_valid_loss:\n",
        "        #    best_valid_loss = valid_loss\n",
        "        #if train_loss < cur_best_train_loss:\n",
        "        #    cur_best_train_loss = train_loss\n",
        "            #torch.save(model.state_dict(), '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt'\n",
        "            #torch.save({\n",
        "            #    'epoch': T_EPOCHS,\n",
        "            #    'state_dict': model.state_dict(),\n",
        "            #    'optimizer': optimizer.state_dict(),\n",
        "            #    'loss': train_loss,\n",
        "            #    }, '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt')\n",
        "\n",
        "        state = {'epoch': T_EPOCHS, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), 'loss': train_loss}\n",
        "        torch.save(state, chpt_file)\n",
        "\n",
        "        print(f'\\nEpoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
        "        print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
        "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')\n",
        "\n",
        "    torch.cuda.empty_cache()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fWni6zhksjhl",
        "outputId": "20ab804b-19bf-4662-909c-7bf4bb72005b"
      },
      "source": [
        "chpt_file = '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt'\n",
        "model, optimizer, start_epoch = load_checkpt(model, optimizer, chpt_file)\n",
        "T_EPOCHS = start_epoch + EP_INT\n",
        "    \n",
        "for i in range(EPOCHS):\n",
        "    if (i%EP_INT==0):\n",
        "        interval_train(model, optimizer, start_epoch, chpt_file)\n",
        "        print('\\nEvaluation\\n----------------------')\n",
        "        qual_eval(model, optimizer, start_epoch, chpt_file)\n",
        "        quant_eval(model, optimizer, start_epoch, chpt_file)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "=> Checkpoint NOT found '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt'\n",
            "=> Checkpoint NOT found '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt'\n",
            "Already trained for 0 epochs. Training now for 1 more\n",
            "\n",
            "Epoch: 01 | Time: 8210m 4s\n",
            "\tTrain Loss: 6.794 | Train PPL: 892.376\n",
            "\t Val. Loss: 6.318 |  Val. PPL: 554.620\n",
            "\n",
            "Evaluation\n",
            "----------------------\n",
            "=> loading checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt'\n",
            "=> loaded checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt' (epoch 1)\n",
            "tensorboard --logdir \"/home/puneet/code/Multimodal Feedback/TBlogs\"\n",
            "---\n",
            "=> loading checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt'\n",
            "=> loaded checkpoint '/home/puneet/code/Multimodal Feedback/checkpoints/baseline4.pt' (epoch 1)\n",
            "{'testlen': 416, 'reflen': 1418, 'guess': [416, 332, 249, 166], 'correct': [40, 2, 0, 0]}\n",
            "ratio: 0.2933709449927409\n",
            "\n",
            "----------------------\n",
            "bleu = [0.008647719345295986, 0.002164533003836243, 1.1916663270679596e-08, 3.094373336322772e-11]\n",
            "cider = 0.040676355714557515\n",
            "rouge = 0.045875785569741416\n",
            "\n",
            "----------------------\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}