3. Quora - Featurizing Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"text-align:center;font-size:30px;\" > Quora Question Pair Similarity </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 3 : Featurizing Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MRP-fAQedMTd"
   },
   "source": [
    "<h2> Featurizing Text Data with TFIDF weighted Word2Vec </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-3IbomL8dMTi",
    "outputId": "3fa8eb7c-ddf2-4f98-edee-0c49db6502e8"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import time\n",
    "import warnings\n",
    "import numpy as np\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import sys\n",
    "import os \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "# exctract word2vec vectors\n",
    "# https://github.com/explosion/spaCy/issues/1721\n",
    "# http://landinghub.visualstudio.com/visual-cpp-build-tools\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j5XNgVyLdMT7"
   },
   "outputs": [],
   "source": [
    "# avoid decoding problems\n",
    "df = pd.read_csv(\"train.csv\")\n",
    " \n",
    "# encode questions to unicode\n",
    "# https://stackoverflow.com/a/6812069\n",
    "# ----------------- python 2 ---------------------\n",
    "# df['question1'] = df['question1'].apply(lambda x: unicode(str(x),\"utf-8\"))\n",
    "# df['question2'] = df['question2'].apply(lambda x: unicode(str(x),\"utf-8\"))\n",
    "# ----------------- python 3 ---------------------\n",
    "df['question1'] = df['question1'].apply(lambda x: str(x))\n",
    "df['question2'] = df['question2'].apply(lambda x: str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RU3HqJXwdMUj"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# merge texts\n",
    "questions = list(df['question1']) + list(df['question2'])\n",
    "\n",
    "tfidf = TfidfVectorizer(lowercase=False, )\n",
    "tfidf.fit_transform(questions)\n",
    "\n",
    "# dict key:word and value:tf-idf score\n",
    "word2tfidf = dict(zip(tfidf.get_feature_names(), tfidf.idf_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "collapsed": true,
    "id": "2JKI2yT4dMUv"
   },
   "source": [
    "- After we find TF-IDF scores, we convert each question to a weighted average of word2vec vectors by these scores.\n",
    "- Here we use a pre-trained GLOVE model which comes free with \"Spacy\".  https://spacy.io/usage/vectors-similarity\n",
    "- It is trained on Wikipedia and therefore, it is stronger in terms of word semantics. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|▏                                                                          | 742/404290 [00:10<2:14:13, 50.11it/s]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-89aa90b23a49>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mqu1\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'question1'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m     \u001b[0mdoc1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mqu1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m     \u001b[1;31m# 384 is the number of dimensions of vectors\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[0mmean_vec1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m96\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\spacy\\language.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, text, disable, component_cfg)\u001b[0m\n\u001b[0;32m    383\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mproc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"__call__\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    384\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mE003\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mproc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 385\u001b[1;33m             \u001b[0mdoc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mproc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mcomponent_cfg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    386\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mdoc\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    387\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mE005\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpipes.pyx\u001b[0m in \u001b[0;36mspacy.pipeline.pipes.Tagger.__call__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpipes.pyx\u001b[0m in \u001b[0;36mspacy.pipeline.pipes.Tagger.predict\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\model.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    163\u001b[0m             \u001b[0mMust\u001b[0m \u001b[0mmatch\u001b[0m \u001b[0mexpected\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    164\u001b[0m         \"\"\"\n\u001b[1;32m--> 165\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    166\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpipe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m128\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\feed_forward.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m     38\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_layers\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m             \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\model.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    163\u001b[0m             \u001b[0mMust\u001b[0m \u001b[0mmatch\u001b[0m \u001b[0mexpected\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    164\u001b[0m         \"\"\"\n\u001b[1;32m--> 165\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    166\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpipe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m128\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\api.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(seqs_in)\u001b[0m\n\u001b[0;32m    277\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mseqs_in\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    278\u001b[0m         \u001b[0mlengths\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mseq\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mseqs_in\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 279\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlayer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mseqs_in\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpad\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpad\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    280\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munflatten\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlengths\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpad\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpad\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    281\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\model.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    163\u001b[0m             \u001b[0mMust\u001b[0m \u001b[0mmatch\u001b[0m \u001b[0mexpected\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    164\u001b[0m         \"\"\"\n\u001b[1;32m--> 165\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    166\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpipe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m128\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\feed_forward.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m     38\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_layers\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m             \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\model.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    163\u001b[0m             \u001b[0mMust\u001b[0m \u001b[0mmatch\u001b[0m \u001b[0mexpected\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    164\u001b[0m         \"\"\"\n\u001b[1;32m--> 165\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    166\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpipe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m128\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\model.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    127\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    128\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 129\u001b[1;33m         \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbegin_update\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    130\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    131\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\api.py\u001b[0m in \u001b[0;36muniqued_fwd\u001b[1;34m(X, drop)\u001b[0m\n\u001b[0;32m    346\u001b[0m         )\n\u001b[0;32m    347\u001b[0m         \u001b[0mX_uniq\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mxp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mascontiguousarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mind\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 348\u001b[1;33m         \u001b[0mY_uniq\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbp_Y_uniq\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbegin_update\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_uniq\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    349\u001b[0m         \u001b[0mY\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mY_uniq\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0minv\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mY_uniq\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    350\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\feed_forward.py\u001b[0m in \u001b[0;36mbegin_update\u001b[1;34m(self, X, drop)\u001b[0m\n\u001b[0;32m     44\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_layers\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 46\u001b[1;33m             \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minc_layer_grad\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbegin_update\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdrop\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     47\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minc_layer_grad\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\api.py\u001b[0m in \u001b[0;36mbegin_update\u001b[1;34m(X, *a, **k)\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mbegin_update\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    131\u001b[0m         \u001b[0mforward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbackward\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msplit_backward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 132\u001b[1;33m         \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfwd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfwd\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    133\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    134\u001b[0m         \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mxp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\api.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mbegin_update\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    131\u001b[0m         \u001b[0mforward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbackward\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msplit_backward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 132\u001b[1;33m         \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfwd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfwd\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    133\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    134\u001b[0m         \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mxp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\api.py\u001b[0m in \u001b[0;36mwrap\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    223\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    224\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mwrap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 225\u001b[1;33m         \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    226\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0msplitter\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    227\u001b[0m             \u001b[0mto_keep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mto_sink\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\thinc\\neural\\_classes\\hash_embed.py\u001b[0m in \u001b[0;36mbegin_update\u001b[1;34m(self, ids, drop)\u001b[0m\n\u001b[0;32m     54\u001b[0m             \u001b[0mids\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mxp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mascontiguousarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mids\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"uint64\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     55\u001b[0m         \u001b[0mkeys\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhash\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mids\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnV\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 56\u001b[1;33m         \u001b[0mvectors\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvectors\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     57\u001b[0m         \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_dropout_mask\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvectors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmask\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# en_vectors_web_lg, which includes over 1 million unique vectors.\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "vecs1 = []\n",
    "\n",
    "for qu1 in tqdm(list(df['question1'])):\n",
    "    doc1 = nlp(qu1) \n",
    "    # 96 is the number of dimensions of vectors \n",
    "    mean_vec1 = np.zeros([len(doc1), 96])\n",
    "    for word1 in doc1:\n",
    "        # word2vec\n",
    "        vec1 = word1.vector\n",
    "        # fetch df score\n",
    "        try:\n",
    "            idf = word2tfidf[str(word1)]\n",
    "        except:\n",
    "            idf = 0\n",
    "        # compute final vec\n",
    "        mean_vec1 += vec1 * idf\n",
    "    mean_vec1 = mean_vec1.mean(axis=0)\n",
    "    vecs1.append(mean_vec1)\n",
    "df['q1_feats_m'] = list(vecs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "96"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec1 = word1.vector\n",
    "len(vec1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PFS6m8z5dMUz",
    "outputId": "3c4fb6fd-7f86-4955-8b8f-762b5969ecce"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                       | 0/404290 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "operands could not be broadcast together with shapes (15,384) (96,) (15,384) ",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-45-e6212d65c433>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     19\u001b[0m         \u001b[1;31m# compute final vec\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m         \u001b[0mtemp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvec1\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0midf\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m         \u001b[0mmean_vec1\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mtemp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     22\u001b[0m     \u001b[0mmean_vec1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmean_vec1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m     \u001b[0mvecs1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmean_vec1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: operands could not be broadcast together with shapes (15,384) (96,) (15,384) "
     ]
    }
   ],
   "source": [
    "# en_vectors_web_lg, which includes over 1 million unique vectors.\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "vecs1 = []\n",
    "# https://github.com/noamraph/tqdm\n",
    "# tqdm is used to print the progress bar\n",
    "for qu1 in tqdm(list(df['question1'])):\n",
    "    doc1 = nlp(qu1) \n",
    "    # 384 is the number of dimensions of vectors \n",
    "    mean_vec1 = np.zeros([len(doc1), len(doc1[0].vector)])\n",
    "    for word1 in doc1:\n",
    "        # word2vec\n",
    "        vec1 = word1.vector\n",
    "        # fetch df score\n",
    "        try:\n",
    "            idf = word2tfidf[str(word1)]\n",
    "        except:\n",
    "            idf = 0\n",
    "        # compute final vec\n",
    "        temp = vec1*idf\n",
    "        mean_vec1 += temp\n",
    "    mean_vec1 = mean_vec1.mean(axis=0)\n",
    "    vecs1.append(mean_vec1)\n",
    "df['q1_feats_m'] = list(vecs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "62GEF-RbdMVB",
    "outputId": "60a4f5f8-5582-4886-befd-2ab6ed99c753"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 404290/404290 [2:00:16<00:00, 56.81it/s]\n"
     ]
    }
   ],
   "source": [
    "vecs2 = []\n",
    "for qu2 in tqdm(list(df['question2'])):\n",
    "    doc2 = nlp(qu2) \n",
    "    mean_vec2 = np.zeros([len(doc1), len(doc1[0].vector)])\n",
    "    for word2 in doc2:\n",
    "        # word2vec\n",
    "        vec2 = word2.vector\n",
    "        # fetch df score\n",
    "        try:\n",
    "            idf = word2tfidf[str(word2)]\n",
    "        except:\n",
    "            #print word\n",
    "            idf = 0\n",
    "        # compute final vec\n",
    "        temp = vec2 * idf\n",
    "        mean_vec1 += temp\n",
    "    mean_vec2 = mean_vec2.mean(axis=0)\n",
    "    vecs2.append(mean_vec2)\n",
    "df['q2_feats_m'] = list(vecs2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "a38GBlGWdMVQ"
   },
   "outputs": [],
   "source": [
    "#prepro_features_train.csv (Simple Preprocessing Feartures)\n",
    "#nlp_features_train.csv (NLP Features)\n",
    "if os.path.isfile('nlp_features_train.csv'):\n",
    "    dfnlp = pd.read_csv(\"nlp_features_train.csv\",encoding='latin-1')\n",
    "else:\n",
    "    print(\"download nlp_features_train.csv from drive or run previous notebook\")\n",
    "\n",
    "if os.path.isfile('df_fe_without_preprocessing_train.csv'):\n",
    "    dfppro = pd.read_csv(\"df_fe_without_preprocessing_train.csv\",encoding='latin-1')\n",
    "else:\n",
    "    print(\"download df_fe_without_preprocessing_train.csv from drive or run previous notebook\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "apdRa1kndMVb"
   },
   "outputs": [],
   "source": [
    "df1 = dfnlp.drop(['qid1','qid2','question1','question2'],axis=1)\n",
    "df2 = dfppro.drop(['qid1','qid2','question1','question2','is_duplicate'],axis=1)\n",
    "df3 = df.drop(['qid1','qid2','question1','question2','is_duplicate'],axis=1)\n",
    "df3_q1 = pd.DataFrame(df3.q1_feats_m.values.tolist(), index= df3.index)\n",
    "df3_q2 = pd.DataFrame(df3.q2_feats_m.values.tolist(), index= df3.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xzWAqGegdMVp",
    "outputId": "2f88eeda-244f-4bbb-a51c-a8680fe8fb92"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>csc_min</th>\n",
       "      <th>csc_max</th>\n",
       "      <th>ctc_min</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>token_set_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>0.833319</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>0.916659</td>\n",
       "      <td>0.785709</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>100</td>\n",
       "      <td>93</td>\n",
       "      <td>93</td>\n",
       "      <td>100</td>\n",
       "      <td>0.982759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>0.749981</td>\n",
       "      <td>0.599988</td>\n",
       "      <td>0.699993</td>\n",
       "      <td>0.466664</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>12.5</td>\n",
       "      <td>86</td>\n",
       "      <td>63</td>\n",
       "      <td>66</td>\n",
       "      <td>75</td>\n",
       "      <td>0.596154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.249997</td>\n",
       "      <td>0.399996</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>66</td>\n",
       "      <td>66</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>36</td>\n",
       "      <td>36</td>\n",
       "      <td>35</td>\n",
       "      <td>40</td>\n",
       "      <td>0.039216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.199998</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>0.307690</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>67</td>\n",
       "      <td>47</td>\n",
       "      <td>46</td>\n",
       "      <td>56</td>\n",
       "      <td>0.175000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  is_duplicate   cwc_min   cwc_max   csc_min   csc_max   ctc_min  \\\n",
       "0   0             0  0.999980  0.833319  0.999983  0.999983  0.916659   \n",
       "1   1             0  0.799984  0.399996  0.749981  0.599988  0.699993   \n",
       "2   2             0  0.399992  0.333328  0.399992  0.249997  0.399996   \n",
       "3   3             0  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "4   4             0  0.399992  0.199998  0.999950  0.666644  0.571420   \n",
       "\n",
       "    ctc_max  last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "0  0.785709           0.0            1.0           2.0      13.0   \n",
       "1  0.466664           0.0            1.0           5.0      12.5   \n",
       "2  0.285712           0.0            1.0           4.0      12.0   \n",
       "3  0.000000           0.0            0.0           2.0      12.0   \n",
       "4  0.307690           0.0            1.0           6.0      10.0   \n",
       "\n",
       "   token_set_ratio  token_sort_ratio  fuzz_ratio  fuzz_partial_ratio  \\\n",
       "0              100                93          93                 100   \n",
       "1               86                63          66                  75   \n",
       "2               66                66          54                  54   \n",
       "3               36                36          35                  40   \n",
       "4               67                47          46                  56   \n",
       "\n",
       "   longest_substr_ratio  \n",
       "0              0.982759  \n",
       "1              0.596154  \n",
       "2              0.166667  \n",
       "3              0.039216  \n",
       "4              0.175000  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dataframe of nlp features\n",
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "N4DQnDtndMV4",
    "outputId": "2e288eed-e8fa-4ec3-a9b9-4e4daba52fc1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>freq_qid1</th>\n",
       "      <th>freq_qid2</th>\n",
       "      <th>q1len</th>\n",
       "      <th>q2len</th>\n",
       "      <th>q1_n_words</th>\n",
       "      <th>q2_n_words</th>\n",
       "      <th>word_Common</th>\n",
       "      <th>word_Total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>freq_q1+q2</th>\n",
       "      <th>freq_q1-q2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>66</td>\n",
       "      <td>57</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>10.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>88</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>59</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>76</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  freq_qid1  freq_qid2  q1len  q2len  q1_n_words  q2_n_words  \\\n",
       "0   0          1          1     66     57          14          12   \n",
       "1   1          4          1     51     88           8          13   \n",
       "2   2          1          1     73     59          14          10   \n",
       "3   3          1          1     50     65          11           9   \n",
       "4   4          3          1     76     39          13           7   \n",
       "\n",
       "   word_Common  word_Total  word_share  freq_q1+q2  freq_q1-q2  \n",
       "0         10.0        23.0    0.434783           2           0  \n",
       "1          4.0        20.0    0.200000           5           3  \n",
       "2          4.0        24.0    0.166667           2           0  \n",
       "3          0.0        19.0    0.000000           2           0  \n",
       "4          2.0        20.0    0.100000           4           2  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data before preprocessing \n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_1YIPtTwdMWC",
    "outputId": "510f4c73-0706-4633-d706-e0d348ebfa71"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>86</th>\n",
       "      <th>87</th>\n",
       "      <th>88</th>\n",
       "      <th>89</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>211.129864</td>\n",
       "      <td>-144.683059</td>\n",
       "      <td>-68.811247</td>\n",
       "      <td>-153.662141</td>\n",
       "      <td>-89.931593</td>\n",
       "      <td>2.311301</td>\n",
       "      <td>136.743747</td>\n",
       "      <td>50.449112</td>\n",
       "      <td>-64.150964</td>\n",
       "      <td>56.627526</td>\n",
       "      <td>...</td>\n",
       "      <td>33.176593</td>\n",
       "      <td>-102.487002</td>\n",
       "      <td>144.595747</td>\n",
       "      <td>50.614258</td>\n",
       "      <td>-21.385940</td>\n",
       "      <td>36.385601</td>\n",
       "      <td>-70.999820</td>\n",
       "      <td>4.141466</td>\n",
       "      <td>-131.379525</td>\n",
       "      <td>71.884573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>144.124685</td>\n",
       "      <td>-114.012484</td>\n",
       "      <td>-111.716694</td>\n",
       "      <td>-104.885038</td>\n",
       "      <td>-88.238478</td>\n",
       "      <td>16.441834</td>\n",
       "      <td>58.238013</td>\n",
       "      <td>102.095138</td>\n",
       "      <td>6.026966</td>\n",
       "      <td>178.498497</td>\n",
       "      <td>...</td>\n",
       "      <td>67.386291</td>\n",
       "      <td>22.131810</td>\n",
       "      <td>40.210794</td>\n",
       "      <td>0.254035</td>\n",
       "      <td>98.600184</td>\n",
       "      <td>-36.436027</td>\n",
       "      <td>-123.609728</td>\n",
       "      <td>71.788337</td>\n",
       "      <td>-15.374614</td>\n",
       "      <td>-10.503958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>81.757898</td>\n",
       "      <td>-142.184507</td>\n",
       "      <td>0.559867</td>\n",
       "      <td>-104.660084</td>\n",
       "      <td>-84.156631</td>\n",
       "      <td>22.515110</td>\n",
       "      <td>115.521661</td>\n",
       "      <td>50.436953</td>\n",
       "      <td>-111.740923</td>\n",
       "      <td>51.713310</td>\n",
       "      <td>...</td>\n",
       "      <td>61.731855</td>\n",
       "      <td>-49.822495</td>\n",
       "      <td>107.080594</td>\n",
       "      <td>-11.431423</td>\n",
       "      <td>-40.667739</td>\n",
       "      <td>-26.771786</td>\n",
       "      <td>-12.482071</td>\n",
       "      <td>1.764694</td>\n",
       "      <td>-100.330459</td>\n",
       "      <td>148.061206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-126.651922</td>\n",
       "      <td>-59.747160</td>\n",
       "      <td>-67.763201</td>\n",
       "      <td>-138.114731</td>\n",
       "      <td>-101.038699</td>\n",
       "      <td>88.148523</td>\n",
       "      <td>-22.912261</td>\n",
       "      <td>85.941426</td>\n",
       "      <td>27.784233</td>\n",
       "      <td>50.810650</td>\n",
       "      <td>...</td>\n",
       "      <td>117.847457</td>\n",
       "      <td>-13.044505</td>\n",
       "      <td>-1.643167</td>\n",
       "      <td>78.158408</td>\n",
       "      <td>23.748127</td>\n",
       "      <td>12.945961</td>\n",
       "      <td>-8.807293</td>\n",
       "      <td>123.360096</td>\n",
       "      <td>64.057049</td>\n",
       "      <td>-24.943289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>299.444044</td>\n",
       "      <td>-188.632001</td>\n",
       "      <td>-22.946291</td>\n",
       "      <td>-273.683355</td>\n",
       "      <td>-188.480395</td>\n",
       "      <td>107.123044</td>\n",
       "      <td>174.946302</td>\n",
       "      <td>-72.042341</td>\n",
       "      <td>-98.290527</td>\n",
       "      <td>137.439973</td>\n",
       "      <td>...</td>\n",
       "      <td>62.256713</td>\n",
       "      <td>-173.503679</td>\n",
       "      <td>249.320174</td>\n",
       "      <td>27.972507</td>\n",
       "      <td>-26.665078</td>\n",
       "      <td>-28.469215</td>\n",
       "      <td>-62.335434</td>\n",
       "      <td>2.738905</td>\n",
       "      <td>-120.762699</td>\n",
       "      <td>111.779907</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 96 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0           1           2           3           4           5   \\\n",
       "0  211.129864 -144.683059  -68.811247 -153.662141  -89.931593    2.311301   \n",
       "1  144.124685 -114.012484 -111.716694 -104.885038  -88.238478   16.441834   \n",
       "2   81.757898 -142.184507    0.559867 -104.660084  -84.156631   22.515110   \n",
       "3 -126.651922  -59.747160  -67.763201 -138.114731 -101.038699   88.148523   \n",
       "4  299.444044 -188.632001  -22.946291 -273.683355 -188.480395  107.123044   \n",
       "\n",
       "           6           7           8           9      ...              86  \\\n",
       "0  136.743747   50.449112  -64.150964   56.627526     ...       33.176593   \n",
       "1   58.238013  102.095138    6.026966  178.498497     ...       67.386291   \n",
       "2  115.521661   50.436953 -111.740923   51.713310     ...       61.731855   \n",
       "3  -22.912261   85.941426   27.784233   50.810650     ...      117.847457   \n",
       "4  174.946302  -72.042341  -98.290527  137.439973     ...       62.256713   \n",
       "\n",
       "           87          88         89         90         91          92  \\\n",
       "0 -102.487002  144.595747  50.614258 -21.385940  36.385601  -70.999820   \n",
       "1   22.131810   40.210794   0.254035  98.600184 -36.436027 -123.609728   \n",
       "2  -49.822495  107.080594 -11.431423 -40.667739 -26.771786  -12.482071   \n",
       "3  -13.044505   -1.643167  78.158408  23.748127  12.945961   -8.807293   \n",
       "4 -173.503679  249.320174  27.972507 -26.665078 -28.469215  -62.335434   \n",
       "\n",
       "           93          94          95  \n",
       "0    4.141466 -131.379525   71.884573  \n",
       "1   71.788337  -15.374614  -10.503958  \n",
       "2    1.764694 -100.330459  148.061206  \n",
       "3  123.360096   64.057049  -24.943289  \n",
       "4    2.738905 -120.762699  111.779907  \n",
       "\n",
       "[5 rows x 96 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Questions 1 tfidf weighted word2vec\n",
    "df3_q1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wUMdkJTNdMWL",
    "outputId": "69e3e256-cbb8-4fe2-aaf2-9088c3868b29"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>86</th>\n",
       "      <th>87</th>\n",
       "      <th>88</th>\n",
       "      <th>89</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 96 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0    1    2    3    4    5    6    7    8    9  ...    86   87   88   89  \\\n",
       "0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0 ...   0.0  0.0  0.0  0.0   \n",
       "1  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0 ...   0.0  0.0  0.0  0.0   \n",
       "2  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0 ...   0.0  0.0  0.0  0.0   \n",
       "3  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0 ...   0.0  0.0  0.0  0.0   \n",
       "4  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0 ...   0.0  0.0  0.0  0.0   \n",
       "\n",
       "    90   91   92   93   94   95  \n",
       "0  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "1  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "2  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "3  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "4  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "\n",
       "[5 rows x 96 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Questions 2 tfidf weighted word2vec\n",
    "df3_q2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ozz83vh4dMWU",
    "outputId": "e5b30f77-2849-4b08-9949-0912ec0db418"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features in nlp dataframe : 17\n",
      "Number of features in preprocessed dataframe : 12\n",
      "Number of features in question1 w2v  dataframe : 96\n",
      "Number of features in question2 w2v  dataframe : 96\n",
      "Number of features in final dataframe  : 221\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of features in nlp dataframe :\", df1.shape[1])\n",
    "print(\"Number of features in preprocessed dataframe :\", df2.shape[1])\n",
    "print(\"Number of features in question1 w2v  dataframe :\", df3_q1.shape[1])\n",
    "print(\"Number of features in question2 w2v  dataframe :\", df3_q2.shape[1])\n",
    "print(\"Number of features in final dataframe  :\", df1.shape[1]+df2.shape[1]+df3_q1.shape[1]+df3_q2.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HmfZ5Q1zdMWl"
   },
   "outputs": [],
   "source": [
    "# storing the final features to csv file\n",
    "if not os.path.isfile('final_features.csv'):\n",
    "    df3_q1['id']=df1['id']\n",
    "    df3_q2['id']=df1['id']\n",
    "    df1  = df1.merge(df2, on='id',how='left')\n",
    "    df2  = df3_q1.merge(df3_q2, on='id',how='left')\n",
    "    result  = df1.merge(df2, on='id',how='left')\n",
    "    result.to_csv('final_features.csv')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "3.Q_Mean_W2V.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}