sentiment140 _biLSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Q1nja3002K6d"
   },
   "source": [
    "# **NLP - Sentiment Analysis of Tweets using biLSTM**\n",
    "A deep learning model built using PyTorch and TorchText to classify sentiments of tweets using a subset of the <a href=\"https://www.kaggle.com/kazanova/sentiment140\">sentiment140 dataset</a>.\n",
    "\n",
    "1. [Dataset Preparation](#section1)\n",
    "2. [Preprocessing](#section2)\n",
    "3. [Model](#section3)\n",
    "4. [Training](#section4)\n",
    "5. [Prediction](#section5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-YaKdyDBXxeK"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import spacy\n",
    "import random\n",
    "from pathlib import Path\n",
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchtext import data \n",
    "import torchtext\n",
    "from nltk.tokenize.treebank import TreebankWordDetokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 125
    },
    "colab_type": "code",
    "id": "OFnqfIft6H-o",
    "outputId": "77aab575-7dd4-4535-b2d1-0bd5cfd46623"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n",
      "\n",
      "Tesla P4\n",
      "Memory Usage:\n",
      "Allocated: 0.0 GB\n",
      "Cached:    0.0 GB\n"
     ]
    }
   ],
   "source": [
    "# Setting device on GPU if available\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print('Using device:', device)\n",
    "print()\n",
    "\n",
    "if device.type == 'cuda':\n",
    "    print(torch.cuda.get_device_name(0))\n",
    "    print('Memory Usage:')\n",
    "    print('Allocated:', round(torch.cuda.memory_allocated(0)/1024**3,1), 'GB')\n",
    "    print('Cached:   ', round(torch.cuda.memory_cached(0)/1024**3,1), 'GB')\n",
    "\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NddQVfQD37dO"
   },
   "source": [
    "<a id='section1'></a>\n",
    "# **1. Dataset Preparation**\n",
    "The first column contains the sentiments and the last column contains the tweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "nFyis9DPYpmg",
    "outputId": "2c58e9c9-034e-48dd-9089-5b65d7950ece"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810369</td>\n",
       "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>_TheSpecialOne_</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810917</td>\n",
       "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>mattycus</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811184</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>ElleCTF</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811193</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>Karoli</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  ...                                                  5\n",
       "0  0  ...  @switchfoot http://twitpic.com/2y1zl - Awww, t...\n",
       "1  0  ...  is upset that he can't update his Facebook by ...\n",
       "2  0  ...  @Kenichan I dived many times for the ball. Man...\n",
       "3  0  ...    my whole body feels itchy and like its on fire \n",
       "4  0  ...  @nationwideclass no, it's not behaving at all....\n",
       "\n",
       "[5 rows x 6 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in data into a dataframe\n",
    "df = pd.read_csv(\"training.1600000.processed.noemoticon.csv\", engine=\"python\", header=None)\n",
    "\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YJ22CrBo4LEm"
   },
   "source": [
    "The dataset consists of two sentiments (0 - negative, 4 - positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "dl0IKQQMdtTR",
    "outputId": "dcfd8399-7026-4345-fd12-6024e00ac17e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    800000\n",
       "0    800000\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count the number of tweets per sentiment\n",
    "df[0].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "6CSfGCIF5Thr",
    "outputId": "b7412833-092d-4990-d7e5-e6d97a2319fa"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    800000\n",
       "0    800000\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Model the sentiments as binary (0 - negative, 1 - positive)\n",
    "df[0]=df[0].replace(to_replace=4,value=1)\n",
    "df[0].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3b728ELHelAt"
   },
   "outputs": [],
   "source": [
    "# Save a subset as a smaller dataset from training\n",
    "df.sample(50000).to_csv(\"sentiment140-small.csv\", header=None, index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Sy5EFz2H8MqH"
   },
   "source": [
    "<a id='section2'></a>\n",
    "# **2. Preprocessing**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "eQT5JZvN8tB0",
    "outputId": "5d88197b-be9e-40c6-ec0a-526693d0c677"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of train data: 40000\n",
      "Number of test data: 5000\n",
      "Number of validation data: 5000\n"
     ]
    }
   ],
   "source": [
    "# Declare fields for tweets and labels\n",
    "# include_lengths tells the RNN how long the actual sequences are\n",
    "TEXT = data.Field(tokenize='spacy', lower=True, include_lengths= True)\n",
    "LABEL = data.LabelField(dtype=torch.float)\n",
    "\n",
    "# Map data to fields\n",
    "fields = [('label', LABEL), ('id',None),('date',None),('query',None),\n",
    "      ('name',None), ('text', TEXT),('category',None)]\n",
    "\n",
    "# Apply field definition to create torch dataset\n",
    "dataset = torchtext.data.TabularDataset(\n",
    "        path=\"sentiment140-small.csv\",\n",
    "        format=\"CSV\",\n",
    "        fields=fields,\n",
    "        skip_header=False)\n",
    "\n",
    "# Split data into train, test, validation sets\n",
    "(train_data, test_data, valid_data) = dataset.split(split_ratio=[0.8,0.1,0.1])\n",
    "\n",
    "print(\"Number of train data: {}\".format(len(train_data)))\n",
    "print(\"Number of test data: {}\".format(len(test_data)))\n",
    "print(\"Number of validation data: {}\".format(len(valid_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "mlZ9-mBFhw8L",
    "outputId": "bab39e21-7ed9-4125-d3a9-31c6bc94c5bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'label': '1', 'text': ['@kellbell68', 'yes', 'kelly', 'i', 'think', 'your', 'nice', 'but', 'come', 'on', 'your', 'from', 'ohio']}\n"
     ]
    }
   ],
   "source": [
    "# An example from the training set\n",
    "print(vars(train_data.examples[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TwbtK5lDlJj2"
   },
   "source": [
    "### **Build Vocabulary**\n",
    "Build the vocabulary for the training set using pre-trained GloVe embeddings.\n",
    "GloVe embeddings were trained on 6 billion tokens and the embeddings are 100-dimensional."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "id": "fz_aN0Zl_Wwh",
    "outputId": "0202546b-fd25-4eb7-c76e-2bdeee054c6d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('i', 24929),\n",
       " ('!', 22715),\n",
       " ('.', 20264),\n",
       " (' ', 14637),\n",
       " ('to', 14019),\n",
       " ('the', 13026),\n",
       " (',', 12245),\n",
       " ('a', 9410),\n",
       " ('my', 7827),\n",
       " ('it', 7645)]"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MAX_VOCAB_SIZE = 25000\n",
    "\n",
    "# unk_init initializes words in the vocab using the Gaussian distribution\n",
    "TEXT.build_vocab(train_data, \n",
    "                 max_size = MAX_VOCAB_SIZE,\n",
    "                 vectors = \"glove.6B.100d\",\n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "\n",
    "# build vocab for training set - convert words into integers\n",
    "LABEL.build_vocab(train_data)\n",
    "\n",
    "# Most frequent tokens\n",
    "TEXT.vocab.freqs.most_common(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3d58JkuABB1d"
   },
   "source": [
    "### **Iterator**\n",
    "Pad each tweet to be the same length to process in batch. \n",
    "The BucketIterator will group tweets of similar lengths together for minimized padding in each batch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HsrRy3dhAlSr"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "\n",
    "# sort_within_batch sorts all the tensors within a batch by their lengths\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data),\n",
    "    device = device,\n",
    "    batch_size = BATCH_SIZE,\n",
    "    sort_key = lambda x: len(x.text),\n",
    "    sort_within_batch = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "braesWjckkJu"
   },
   "source": [
    "<a id='section3'></a>\n",
    "# **3. Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Cst_rAeZaB1G"
   },
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, \n",
    "                 bidirectional, dropout, pad_idx):\n",
    "        \"\"\"\n",
    "        Define the layers of the module.\n",
    "\n",
    "        vocab_size - vocabulary size\n",
    "        embedding_dim - size of the dense word vectors\n",
    "        hidden_dim - size of the hidden states\n",
    "        output_dim - number of classes\n",
    "        n_layers - number of multi-layer RNN\n",
    "        bidirectional - boolean - use both directions of LSTM\n",
    "        dropout - dropout probability\n",
    "        pad_idx -  string representing the pad token\n",
    "        \"\"\"\n",
    "        \n",
    "        super().__init__()\n",
    "\n",
    "        # 1. Feed the tweets in the embedding layer\n",
    "        # padding_idx set to not learn the emedding for the <pad> token - irrelevant to determining sentiment\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)\n",
    "\n",
    "        # 2. LSTM layer\n",
    "        # returns the output and a tuple of the final hidden state and final cell state\n",
    "        self.encoder = nn.LSTM(embedding_dim, \n",
    "                               hidden_dim, \n",
    "                               num_layers=n_layers,\n",
    "                               bidirectional=bidirectional,\n",
    "                               dropout=dropout)\n",
    "        \n",
    "        # 3. Fully-connected layer\n",
    "        # Final hidden state has both a forward and a backward component concatenated together\n",
    "        # The size of the input to the nn.Linear layer is twice that of the hidden dimension size\n",
    "        self.predictor = nn.Linear(hidden_dim*2, output_dim)\n",
    "\n",
    "        # Initialize dropout layer for regularization\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "      \n",
    "    def forward(self, text, text_lengths):\n",
    "        \"\"\"\n",
    "        The forward method is called when data is fed into the model.\n",
    "\n",
    "        text - [tweet length, batch size]\n",
    "        text_lengths - lengths of tweet\n",
    "        \"\"\"\n",
    "\n",
    "        # embedded = [sentence len, batch size, emb dim]\n",
    "        embedded = self.dropout(self.embedding(text))    \n",
    "\n",
    "        # Pack the embeddings - cause RNN to only process non-padded elements\n",
    "        # Speeds up computation\n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths)\n",
    "\n",
    "        # output of encoder\n",
    "        packed_output, (hidden, cell) = self.encoder(packed_embedded)\n",
    "\n",
    "        # unpack sequence - transform packed sequence to a tensor\n",
    "        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output)\n",
    "\n",
    "        # output = [sentence len, batch size, hid dim * num directions]\n",
    "        # output over padding tokens are zero tensors\n",
    "        \n",
    "        # hidden = [num layers * num directions, batch size, hid dim]\n",
    "        # cell = [num layers * num directions, batch size, hid dim]\n",
    "        \n",
    "        # Get the final layer forward and backward hidden states  \n",
    "        # concat the final forward and backward hidden layers and apply dropout\n",
    "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))\n",
    "\n",
    "        # hidden = [batch size, hid dim * num directions]\n",
    "\n",
    "        return self.predictor(hidden)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MAk1scwU_g3d"
   },
   "source": [
    "### **Create Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P6vpUuzkdCah"
   },
   "outputs": [],
   "source": [
    "INPUT_DIM = len(TEXT.vocab)\n",
    "# dim must be equal to the dim of pre-trained GloVe vectors\n",
    "EMBEDDING_DIM = 100\n",
    "HIDDEN_DIM = 256\n",
    "OUTPUT_DIM = 1\n",
    "# 2 layers of biLSTM\n",
    "N_LAYERS = 2\n",
    "BIDIRECTIONAL = True\n",
    "# Dropout probability\n",
    "DROPOUT = 0.5\n",
    "# Get pad token index from vocab\n",
    "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
    "\n",
    "# Create an instance of LSTM class\n",
    "model = LSTM(INPUT_DIM,\n",
    "            EMBEDDING_DIM,\n",
    "            HIDDEN_DIM,\n",
    "            OUTPUT_DIM,\n",
    "            N_LAYERS,\n",
    "            BIDIRECTIONAL,\n",
    "            DROPOUT,\n",
    "            PAD_IDX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "TQ39qc-jv4PA",
    "outputId": "278e9670-ebbe-47bb-ee58-b5b8b0c09a4e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'label': '1', 'text': ['@kellbell68', 'yes', 'kelly', 'i', 'think', 'your', 'nice', 'but', 'come', 'on', 'your', 'from', 'ohio']}\n"
     ]
    }
   ],
   "source": [
    "# Sample from the training set\n",
    "print(vars(train_iterator.dataset[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "2xPD99rLd5Fk",
    "outputId": "fddfa17d-bd08-4bab-d0e2-ff4a11d2ee1c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([25002, 100])\n"
     ]
    }
   ],
   "source": [
    "# Copy the pre-trained word embeddings into the embedding layer\n",
    "pretrained_embeddings = TEXT.vocab.vectors\n",
    "\n",
    "# [vocab size, embedding dim]\n",
    "print(pretrained_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "We1MEJwZd9pp",
    "outputId": "a9e97fc7-d07c-4187-9cb1-97ec98e2639b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.1700, -1.6579, -0.7456,  ..., -0.6745, -0.0594,  0.8549],\n",
       "        [ 0.6072,  1.6403,  0.1663,  ..., -0.5321, -0.5791,  0.5554],\n",
       "        [-0.0465,  0.6197,  0.5665,  ..., -0.3762, -0.0325,  0.8062],\n",
       "        ...,\n",
       "        [-0.3067, -0.7526, -0.5373,  ...,  0.8001, -0.3245, -1.8043],\n",
       "        [ 0.2613,  0.3635, -0.0122,  ..., -0.6022, -0.3825,  0.3033],\n",
       "        [ 0.4742, -0.0096,  0.3001,  ...,  1.1507,  1.0169, -0.6021]])"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace the initial weights of the embedding layer with the pre-trained embeddings\n",
    "model.embedding.weight.data.copy_(pretrained_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "p24pobL2eCqj",
    "outputId": "d48582e6-60c8-4147-c8d3-367d8093ffb1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "        [-0.0465,  0.6197,  0.5665,  ..., -0.3762, -0.0325,  0.8062],\n",
      "        ...,\n",
      "        [-0.3067, -0.7526, -0.5373,  ...,  0.8001, -0.3245, -1.8043],\n",
      "        [ 0.2613,  0.3635, -0.0122,  ..., -0.6022, -0.3825,  0.3033],\n",
      "        [ 0.4742, -0.0096,  0.3001,  ...,  1.1507,  1.0169, -0.6021]])\n"
     ]
    }
   ],
   "source": [
    "# Initialize <unk> and <pad> both to all zeros - irrelevant for sentiment analysis\n",
    "UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]\n",
    "\n",
    "# Setting row in the embedding weights matrix to zero using the token index\n",
    "model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)\n",
    "\n",
    "print(model.embedding.weight.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1FzS3q48u3bA"
   },
   "source": [
    "<a id='section4'></a>\n",
    "# **4. Training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mp-WpuhLk1np"
   },
   "outputs": [],
   "source": [
    "# Adam optimizer used to update the weights\n",
    "optimizer = optim.Adam(model.parameters(), lr=2e-2)\n",
    "\n",
    "# Loss function: binary cross entropy with logits\n",
    "# It restricts the predictions to a number between 0 and 1 using the logit function\n",
    "# then use the bound scarlar to calculate the loss using binary cross entropy\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "# Use GPU\n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mkVIS45Tejg0"
   },
   "outputs": [],
   "source": [
    "# Helper functions\n",
    "\n",
    "def batch_accuracy(predictions, label):\n",
    "    \"\"\"\n",
    "    Returns accuracy per batch.\n",
    "\n",
    "    predictions - float\n",
    "    label - 0 or 1\n",
    "    \"\"\"\n",
    "\n",
    "    # Round predictions to the closest integer using the sigmoid function\n",
    "    preds = torch.round(torch.sigmoid(predictions))\n",
    "    # If prediction is equal to label\n",
    "    correct = (preds == label).float()\n",
    "    # Average correct predictions\n",
    "    accuracy = correct.sum() / len(correct)\n",
    "\n",
    "    return accuracy\n",
    "\n",
    "def timer(start_time, end_time):\n",
    "    \"\"\"\n",
    "    Returns the minutes and seconds.\n",
    "    \"\"\"\n",
    "\n",
    "    time = end_time - start_time\n",
    "    mins = int(time / 60)\n",
    "    secs = int(time - (mins * 60))\n",
    "\n",
    "    return mins, secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "V_0IWDF_fQ-u"
   },
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion):\n",
    "    \"\"\"\n",
    "    Function to evaluate training loss and accuracy.\n",
    "\n",
    "    iterator - train iterator\n",
    "    \"\"\"\n",
    "    \n",
    "    # Cumulated Training loss\n",
    "    training_loss = 0.0\n",
    "    # Cumulated Training accuracy\n",
    "    training_acc = 0.0\n",
    "    \n",
    "    # Set model to training mode\n",
    "    model.train()\n",
    "    \n",
    "    # For each batch in the training iterator\n",
    "    for batch in iterator:\n",
    "        \n",
    "        # 1. Zero the gradients\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # batch.text is a tuple (tensor, len of seq)\n",
    "        text, text_lengths = batch.text\n",
    "        \n",
    "        # 2. Compute the predictions\n",
    "        predictions = model(text, text_lengths).squeeze(1)\n",
    "        \n",
    "        # 3. Compute loss\n",
    "        loss = criterion(predictions, batch.label)\n",
    "        \n",
    "        # Compute accuracy\n",
    "        accuracy = batch_accuracy(predictions, batch.label)\n",
    "        \n",
    "        # 4. Use loss to compute gradients\n",
    "        loss.backward()\n",
    "        \n",
    "        # 5. Use optimizer to take gradient step\n",
    "        optimizer.step()\n",
    "        \n",
    "        training_loss += loss.item()\n",
    "        training_acc += accuracy.item()\n",
    "    \n",
    "    # Return the loss and accuracy, averaged across each epoch\n",
    "    # len of iterator = num of batches in the iterator\n",
    "    return training_loss / len(iterator), training_acc / len(iterator)\n",
    "\n",
    "def evaluate(model, iterator, criterion):\n",
    "    \"\"\"\n",
    "    Function to evaluate the loss and accuracy of validation and test sets.\n",
    "\n",
    "    iterator - validation or test iterator\n",
    "    \"\"\"\n",
    "    \n",
    "    # Cumulated Training loss\n",
    "    eval_loss = 0.0\n",
    "    # Cumulated Training accuracy\n",
    "    eval_acc = 0\n",
    "    \n",
    "    # Set model to evaluation mode\n",
    "    model.eval()\n",
    "    \n",
    "    # Don't calculate the gradients\n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "\n",
    "            text, text_lengths = batch.text\n",
    "            \n",
    "            predictions = model(text, text_lengths).squeeze(1)\n",
    "            \n",
    "            loss = criterion(predictions, batch.label)\n",
    "            \n",
    "            accuracy = batch_accuracy(predictions, batch.label)\n",
    "\n",
    "            eval_loss += loss.item()\n",
    "            eval_acc += accuracy.item()\n",
    "        \n",
    "    return eval_loss / len(iterator), eval_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gsuG6zKbDk8t"
   },
   "source": [
    "### **Train the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 377
    },
    "colab_type": "code",
    "id": "VxmPnZ6khFVP",
    "outputId": "851c8f5a-46c2-415f-a1cc-82484a31e6fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1:\n",
      "\t Total Time: 0m 6s\n",
      "\t Train Loss 0.7 | Train Accuracy: 55.67%\n",
      "\t Validation Loss 0.66 | Validation Accuracy: 61.84%\n",
      "Epoch 2:\n",
      "\t Total Time: 0m 6s\n",
      "\t Train Loss 0.63 | Train Accuracy: 65.17%\n",
      "\t Validation Loss 0.54 | Validation Accuracy: 73.01%\n",
      "Epoch 3:\n",
      "\t Total Time: 0m 6s\n",
      "\t Train Loss 0.51 | Train Accuracy: 75.49%\n",
      "\t Validation Loss 0.52 | Validation Accuracy: 76.54%\n",
      "Epoch 4:\n",
      "\t Total Time: 0m 6s\n",
      "\t Train Loss 0.46 | Train Accuracy: 79.07%\n",
      "\t Validation Loss 0.5 | Validation Accuracy: 76.11%\n",
      "Epoch 5:\n",
      "\t Total Time: 0m 6s\n",
      "\t Train Loss 0.43 | Train Accuracy: 80.55%\n",
      "\t Validation Loss 0.51 | Validation Accuracy: 75.76%\n"
     ]
    }
   ],
   "source": [
    "# Number of epochs\n",
    "NUM_EPOCHS = 5\n",
    "\n",
    "# Lowest validation lost\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(NUM_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    # Evaluate training loss and accuracy\n",
    "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
    "    # Evaluate validation loss and accuracy\n",
    "    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    mins, secs = timer(start_time, end_time)\n",
    "    \n",
    "    # At each epoch, if the validation loss is the best\n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        # Save the parameters of the model\n",
    "        torch.save(model.state_dict(), 'model-small.pt')\n",
    "\n",
    "    print(\"Epoch {}:\".format(epoch+1))\n",
    "    print(\"\\t Total Time: {}m {}s\".format(mins, secs))\n",
    "    print(\"\\t Train Loss {} | Train Accuracy: {}%\".format(round(train_loss, 2), round(train_acc*100, 2)))\n",
    "    print(\"\\t Validation Loss {} | Validation Accuracy: {}%\".format(round(valid_loss, 2), round(valid_acc*100, 2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WRHo-gVszo9H"
   },
   "source": [
    "<a id='section5'></a>\n",
    "# **5. Prediction**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "n40NoCEniiL6",
    "outputId": "5bfdc497-c2de-4f0c-d3fe-b722422fd4ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 0.52 | Test Acc: 74.77%\n"
     ]
    }
   ],
   "source": [
    "# Load the model with the best validation loss\n",
    "model.load_state_dict(torch.load('model-small.pt'))\n",
    "\n",
    "# Evaluate test loss and accuracy\n",
    "test_loss, test_acc = evaluate(model, test_iterator, criterion)\n",
    "\n",
    "print(\"Test Loss: {} | Test Acc: {}%\".format(round(test_loss, 2), round(test_acc*100, 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cEzb31Turmm_"
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load('en')\n",
    "\n",
    "def predict(model, text, tokenized=True):\n",
    "    \"\"\"\n",
    "    Given a tweet, predict the sentiment.\n",
    "\n",
    "    text - a string or a a list of tokens\n",
    "    tokenized - True if text is a list of tokens, False if passing in a string\n",
    "    \"\"\"\n",
    "\n",
    "    # Sets the model to evaluation mode\n",
    "    model.eval()\n",
    "\n",
    "    if tokenized == False:\n",
    "        # Tokenizes the sentence\n",
    "        tokens = [token.text for token in nlp.tokenizer(text)]\n",
    "    else:\n",
    "        tokens = text\n",
    "\n",
    "    # Index the tokens by converting to the integer representation from the vocabulary\n",
    "    indexed_tokens = [TEXT.vocab.stoi[t] for t in tokens]\n",
    "    # Get the length of the text\n",
    "    length = [len(indexed_tokens)]\n",
    "    # Convert the indices to a tensor\n",
    "    tensor = torch.LongTensor(indexed_tokens).to(device)\n",
    "    # Add a batch dimension by unsqueezeing\n",
    "    tensor = tensor.unsqueeze(1)\n",
    "    # Converts the length into a tensor\n",
    "    length_tensor = torch.LongTensor(length)\n",
    "    # Convert prediction to be between 0 and 1 with the sigmoid function\n",
    "    prediction = torch.sigmoid(model(tensor, length_tensor))\n",
    "\n",
    "    # Return a single value from the prediction\n",
    "    return prediction.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "LNVfJSHooMus",
    "outputId": "9a7ccf8a-a352-4bf9-a025-1425d1e3761d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweet: on first break . drinking tea without suger . there is no coffee\n",
      "Prediction: 0.25\n",
      "True Label: 0\n"
     ]
    }
   ],
   "source": [
    "# Single example prediction from the test set\n",
    "print(\"Tweet: {}\".format(TreebankWordDetokenizer().detokenize(test_data[10].text)))\n",
    "\n",
    "print(\"Prediction: {}\".format(round(predict(model, test_data[10].text), 2)))\n",
    "\n",
    "print(\"True Label: {}\".format(test_data[10].label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 359
    },
    "colab_type": "code",
    "id": "8GD0zC4Fugk-",
    "outputId": "6f14da85-f61e-401e-fc07-d33533ec7250"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>Prediction</th>\n",
       "      <th>True Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>morning all! got up way too early, the boy and...</td>\n",
       "      <td>0.701752</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>another 2 hours to my flight .... sooo boring....</td>\n",
       "      <td>0.109462</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@chiacy   please do so, he deserves it</td>\n",
       "      <td>0.856429</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>can't wait to make that call tomorrow . gotta ...</td>\n",
       "      <td>0.520005</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@mfrancella i hope you all return for a tourna...</td>\n",
       "      <td>0.305080</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i want to watch twit live!   dratted mediocre ...</td>\n",
       "      <td>0.400689</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>darn i do nt have guitar tomorrow</td>\n",
       "      <td>0.073840</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@chinkyfab well yes but it is a weekly party h...</td>\n",
       "      <td>0.154806</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>07:35am here . waiting for @garyphayes to make...</td>\n",
       "      <td>0.602223</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@gummibalu don't beat yourself up . i've learn...</td>\n",
       "      <td>0.242678</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Tweet  Prediction True Label\n",
       "0  morning all! got up way too early, the boy and...    0.701752          1\n",
       "1  another 2 hours to my flight .... sooo boring....    0.109462          1\n",
       "2             @chiacy   please do so, he deserves it    0.856429          1\n",
       "3  can't wait to make that call tomorrow . gotta ...    0.520005          1\n",
       "4  @mfrancella i hope you all return for a tourna...    0.305080          0\n",
       "5  i want to watch twit live!   dratted mediocre ...    0.400689          0\n",
       "6                  darn i do nt have guitar tomorrow    0.073840          0\n",
       "7  @chinkyfab well yes but it is a weekly party h...    0.154806          1\n",
       "8  07:35am here . waiting for @garyphayes to make...    0.602223          0\n",
       "9  @gummibalu don't beat yourself up . i've learn...    0.242678          0"
      ]
     },
     "execution_count": 26,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example prediction from the test set\n",
    "\n",
    "# List to append data to\n",
    "d = []\n",
    "\n",
    "\n",
    "for idx in range(10):\n",
    "\n",
    "    # Detokenize the tweets from the test set\n",
    "    tweet = TreebankWordDetokenizer().detokenize(test_data[idx].text)\n",
    "                                                 \n",
    "    # Append tweet, prediction, and true label\n",
    "    d.append({'Tweet': tweet, 'Prediction': predict(model, test_data[idx].text), 'True Label': test_data[idx].label})\n",
    "\n",
    "# Convert list to dataframe\n",
    "pd.DataFrame(d)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "sentiment140",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}