Seq2Seq_BILSTM_1124_learning30000_emb_size_150-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural Machine Translation using word level language model and embeddings in Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'D:\\\\Visual_Storytelling_Challenge2018\\\\code\\\\IBM'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwdㅎ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\Anaconda3\\envs\\gpu\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "import pickle\n",
    "from pandas import Series\n",
    "from string import digits\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import re\n",
    "from sklearn.cross_validation import train_test_split\n",
    "# Building a english to french translator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before:  (145116, 4)\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "(30000, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Desc</th>\n",
       "      <th>Story</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>big old tree being photographed on a sunny day</td>\n",
       "      <td>and its magnificent trunk , larger than life i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a old curvy tree in the sun light .</td>\n",
       "      <td>and its magnificent trunk , larger than life i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a person is taking a picture of a large tree a...</td>\n",
       "      <td>and its magnificent trunk , larger than life i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>large tree with many outstretching branches an...</td>\n",
       "      <td>we found this tree when we were walking in a n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a green sign is describing a historic tree and...</td>\n",
       "      <td>it turns out it is a popular attraction here .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a large tree with roots that look like crocodi...</td>\n",
       "      <td>the tree is very unusual , with its roots expo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>big old tree being photographed on a sunny day</td>\n",
       "      <td>the trunk was really wide , as much as 12 feet !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>huge brown tree roots rose above the ground .</td>\n",
       "      <td>you can see how big these roots are - pretty a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>a large tree with many branches coming out</td>\n",
       "      <td>we found this tree when we were walking in a n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>a plaque describes an historical tree and advi...</td>\n",
       "      <td>it turns out it is a popular attraction here .</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Desc  \\\n",
       "0     big old tree being photographed on a sunny day   \n",
       "1                a old curvy tree in the sun light .   \n",
       "2  a person is taking a picture of a large tree a...   \n",
       "3  large tree with many outstretching branches an...   \n",
       "4  a green sign is describing a historic tree and...   \n",
       "5  a large tree with roots that look like crocodi...   \n",
       "6     big old tree being photographed on a sunny day   \n",
       "7      huge brown tree roots rose above the ground .   \n",
       "8         a large tree with many branches coming out   \n",
       "9  a plaque describes an historical tree and advi...   \n",
       "\n",
       "                                               Story  \n",
       "0  and its magnificent trunk , larger than life i...  \n",
       "1  and its magnificent trunk , larger than life i...  \n",
       "2  and its magnificent trunk , larger than life i...  \n",
       "3  we found this tree when we were walking in a n...  \n",
       "4     it turns out it is a popular attraction here .  \n",
       "5  the tree is very unusual , with its roots expo...  \n",
       "6   the trunk was really wide , as much as 12 feet !  \n",
       "7  you can see how big these roots are - pretty a...  \n",
       "8  we found this tree when we were walking in a n...  \n",
       "9     it turns out it is a popular attraction here .  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load\n",
    "with open('df_story_desc_final.pickle', 'rb') as f:\n",
    "    df_story_and_desc_pickle = pickle.load(f)\n",
    "\n",
    "df_story_and_desc = df_story_and_desc_pickle\n",
    "print (\"before: \", df_story_and_desc.shape)\n",
    "df_story_and_desc = df_story_and_desc[:30000]\n",
    "df_story_and_desc_id = df_story_and_desc['Story_Photo_id']\n",
    "df_story_and_desc_text = df_story_and_desc[['Desc', 'Story']]\n",
    "print (type(df_story_and_desc_text))\n",
    "print (df_story_and_desc_text.shape)\n",
    "df_story_and_desc_text.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the people shoot fireworks into the air at night .'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_story_and_desc_text.iloc[4000].Desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'they had some low flying fireworks at the display that evening .'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_story_and_desc_text.iloc[4000].Story"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data cleaning\n",
    "<br />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def re_sub(item):\n",
    "    re_sentence = []\n",
    "    for sentence in item:\n",
    "        sentence = re.sub(\" '\", \"'\", sentence) # 'll, 'm, 's\n",
    "        sentence = re.sub(\" ,\", \",\", sentence)\n",
    "        sentence = re.sub(\" [.]\", \". \", sentence)\n",
    "        sentence = re.sub(\" n't\", \"n't\",sentence)\n",
    "        sentence = re.sub(\" !\", \"! \", sentence)\n",
    "        sentence = re.sub(\"  \", \" \", sentence)\n",
    "        sentence = sentence[:-1]\n",
    "        re_sentence.append(sentence)\n",
    "    return re_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description_text</th>\n",
       "      <th>story_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>big old tree being photographed on a sunny da</td>\n",
       "      <td>and its magnificent trunk, larger than life it...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a old curvy tree in the sun light.</td>\n",
       "      <td>and its magnificent trunk, larger than life it...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a person is taking a picture of a large tree a...</td>\n",
       "      <td>and its magnificent trunk, larger than life it...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>large tree with many outstretching branches an...</td>\n",
       "      <td>we found this tree when we were walking in a n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a green sign is describing a historic tree and...</td>\n",
       "      <td>it turns out it is a popular attraction here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a large tree with roots that look like crocodi...</td>\n",
       "      <td>the tree is very unusual, with its roots exposed.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>big old tree being photographed on a sunny da</td>\n",
       "      <td>the trunk was really wide, as much as 12 feet!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>huge brown tree roots rose above the ground.</td>\n",
       "      <td>you can see how big these roots are - pretty a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>a large tree with many branches coming ou</td>\n",
       "      <td>we found this tree when we were walking in a n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>a plaque describes an historical tree and advi...</td>\n",
       "      <td>it turns out it is a popular attraction here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>a tree has spread its roots high and wide acro...</td>\n",
       "      <td>the tree is very unusual, with its roots exposed.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>a old curvy tree in the sun light.</td>\n",
       "      <td>the trunk was really wide, as much as 12 feet!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>some trees that are surrounded by apartment bu...</td>\n",
       "      <td>you can see how big these roots are - pretty a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>the tree has very long and dated branches.</td>\n",
       "      <td>we found this tree when we were walking in a n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>a plaque on a stand surround by died leaves of...</td>\n",
       "      <td>it turns out it is a popular attraction here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>a huge tree sits outside with several large ro...</td>\n",
       "      <td>the tree is very unusual, with its roots exposed.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>a person is taking a picture of a large tree a...</td>\n",
       "      <td>the trunk was really wide, as much as 12 feet!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>various parts of the tree are growing out from...</td>\n",
       "      <td>you can see how big these roots are - pretty a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>big old tree being photographed on a sunny da</td>\n",
       "      <td>some more different parts of the tree.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>a old curvy tree in the sun light.</td>\n",
       "      <td>some more different parts of the tree.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>a person is taking a picture of a large tree a...</td>\n",
       "      <td>some more different parts of the tree.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>large tree with many outstretching branches an...</td>\n",
       "      <td>they went to the botanic gardens specifically ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>a green sign is describing a historic tree and...</td>\n",
       "      <td>there was an informational sign posted near it.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>a large tree with roots that look like crocodi...</td>\n",
       "      <td>the roots were huge and spread out over the gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>big old tree being photographed on a sunny da</td>\n",
       "      <td>the trunk was incredibly thick and rigid.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>huge brown tree roots rose above the ground.</td>\n",
       "      <td>the large roots were almost as thick as the tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>a large tree with many branches coming ou</td>\n",
       "      <td>they went to the botanic gardens specifically ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>a plaque describes an historical tree and advi...</td>\n",
       "      <td>there was an informational sign posted near it.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>a tree has spread its roots high and wide acro...</td>\n",
       "      <td>the roots were huge and spread out over the gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>a old curvy tree in the sun light.</td>\n",
       "      <td>the trunk was incredibly thick and rigid.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29970</th>\n",
       "      <td>winnings are displayed as ribbons and a sign o...</td>\n",
       "      <td>the fair was in town.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29971</th>\n",
       "      <td>a little girl jumps excitedly as she pets a bl...</td>\n",
       "      <td>[female] was so excited to see animals.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29972</th>\n",
       "      <td>sheep heavy with wool gather inside a pen at t...</td>\n",
       "      <td>sheep were there, but better yet there were. .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29973</th>\n",
       "      <td>the llama is waiting for someone to come pet him.</td>\n",
       "      <td>alpacas! the kids loved them.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29974</th>\n",
       "      <td>a sign on a wood fence describes alpacas.</td>\n",
       "      <td>and after they learned all about them.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29975</th>\n",
       "      <td>a place of business displaying their 4 award</td>\n",
       "      <td>the fair was in town.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29976</th>\n",
       "      <td>a small child pets a big animal in a pen.</td>\n",
       "      <td>[female] was so excited to see animals.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29977</th>\n",
       "      <td>a small group sheeps in the ground with leg</td>\n",
       "      <td>sheep were there, but better yet there were. .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29978</th>\n",
       "      <td>a little brown llama inside of a barn.</td>\n",
       "      <td>alpacas! the kids loved them.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29979</th>\n",
       "      <td>signage on wooden fence presenting information...</td>\n",
       "      <td>and after they learned all about them.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29980</th>\n",
       "      <td>there are ribbons on the horses cage, symboliz...</td>\n",
       "      <td>the fair was in town.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29981</th>\n",
       "      <td>a young child is inside a pen with a black ani...</td>\n",
       "      <td>[female] was so excited to see animals.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29982</th>\n",
       "      <td>sheep standing in a display pen at a county fair.</td>\n",
       "      <td>sheep were there, but better yet there were. .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29983</th>\n",
       "      <td>a brown llama is enclosed in a pen covered in ...</td>\n",
       "      <td>alpacas! the kids loved them.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29984</th>\n",
       "      <td>a sign that gives information to the people in...</td>\n",
       "      <td>and after they learned all about them.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29985</th>\n",
       "      <td>a trip over the summer vacation, it was great.</td>\n",
       "      <td>we went to organization last summer for our va...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29986</th>\n",
       "      <td>father, son and mother sit on boat as it trave...</td>\n",
       "      <td>we got to ride so many different rides.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29987</th>\n",
       "      <td>the children pondered as they looked on the sc...</td>\n",
       "      <td>the kids were having so much fun.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29988</th>\n",
       "      <td>three boys are dressed up as pirates in the [o...</td>\n",
       "      <td>they loved going around to all the different s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29989</th>\n",
       "      <td>[organization] [organization] statue with mick...</td>\n",
       "      <td>we had such a good time and can't wait to come...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29990</th>\n",
       "      <td>tourists are on the courtyard between banner d...</td>\n",
       "      <td>we went to organization last summer for our va...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29991</th>\n",
       "      <td>she was more excited than anyone to see the mo...</td>\n",
       "      <td>we got to ride so many different rides.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29992</th>\n",
       "      <td>three young boys sitting on the stairs in a park.</td>\n",
       "      <td>the kids were having so much fun.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29993</th>\n",
       "      <td>three boys displaying their best pirate faces ...</td>\n",
       "      <td>they loved going around to all the different s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29994</th>\n",
       "      <td>statue of a man and a mouse in front of a castle.</td>\n",
       "      <td>we had such a good time and can't wait to come...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>a crowd of people in a village square. three o...</td>\n",
       "      <td>we went to organization last summer for our va...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>people on a safari truck watching as they expl...</td>\n",
       "      <td>we got to ride so many different rides.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>children pose for a photograph on steps in a p...</td>\n",
       "      <td>the kids were having so much fun.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>kids wearing pirate hats are brandishing toy s...</td>\n",
       "      <td>they loved going around to all the different s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>bronze statue outdoors in foreground, large ca...</td>\n",
       "      <td>we had such a good time and can't wait to come...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        description_text  \\\n",
       "0          big old tree being photographed on a sunny da   \n",
       "1                     a old curvy tree in the sun light.   \n",
       "2      a person is taking a picture of a large tree a...   \n",
       "3      large tree with many outstretching branches an...   \n",
       "4      a green sign is describing a historic tree and...   \n",
       "5      a large tree with roots that look like crocodi...   \n",
       "6          big old tree being photographed on a sunny da   \n",
       "7           huge brown tree roots rose above the ground.   \n",
       "8              a large tree with many branches coming ou   \n",
       "9      a plaque describes an historical tree and advi...   \n",
       "10     a tree has spread its roots high and wide acro...   \n",
       "11                    a old curvy tree in the sun light.   \n",
       "12     some trees that are surrounded by apartment bu...   \n",
       "13            the tree has very long and dated branches.   \n",
       "14     a plaque on a stand surround by died leaves of...   \n",
       "15     a huge tree sits outside with several large ro...   \n",
       "16     a person is taking a picture of a large tree a...   \n",
       "17     various parts of the tree are growing out from...   \n",
       "18         big old tree being photographed on a sunny da   \n",
       "19                    a old curvy tree in the sun light.   \n",
       "20     a person is taking a picture of a large tree a...   \n",
       "21     large tree with many outstretching branches an...   \n",
       "22     a green sign is describing a historic tree and...   \n",
       "23     a large tree with roots that look like crocodi...   \n",
       "24         big old tree being photographed on a sunny da   \n",
       "25          huge brown tree roots rose above the ground.   \n",
       "26             a large tree with many branches coming ou   \n",
       "27     a plaque describes an historical tree and advi...   \n",
       "28     a tree has spread its roots high and wide acro...   \n",
       "29                    a old curvy tree in the sun light.   \n",
       "...                                                  ...   \n",
       "29970  winnings are displayed as ribbons and a sign o...   \n",
       "29971  a little girl jumps excitedly as she pets a bl...   \n",
       "29972  sheep heavy with wool gather inside a pen at t...   \n",
       "29973  the llama is waiting for someone to come pet him.   \n",
       "29974          a sign on a wood fence describes alpacas.   \n",
       "29975       a place of business displaying their 4 award   \n",
       "29976          a small child pets a big animal in a pen.   \n",
       "29977        a small group sheeps in the ground with leg   \n",
       "29978             a little brown llama inside of a barn.   \n",
       "29979  signage on wooden fence presenting information...   \n",
       "29980  there are ribbons on the horses cage, symboliz...   \n",
       "29981  a young child is inside a pen with a black ani...   \n",
       "29982  sheep standing in a display pen at a county fair.   \n",
       "29983  a brown llama is enclosed in a pen covered in ...   \n",
       "29984  a sign that gives information to the people in...   \n",
       "29985     a trip over the summer vacation, it was great.   \n",
       "29986  father, son and mother sit on boat as it trave...   \n",
       "29987  the children pondered as they looked on the sc...   \n",
       "29988  three boys are dressed up as pirates in the [o...   \n",
       "29989  [organization] [organization] statue with mick...   \n",
       "29990  tourists are on the courtyard between banner d...   \n",
       "29991  she was more excited than anyone to see the mo...   \n",
       "29992  three young boys sitting on the stairs in a park.   \n",
       "29993  three boys displaying their best pirate faces ...   \n",
       "29994  statue of a man and a mouse in front of a castle.   \n",
       "29995  a crowd of people in a village square. three o...   \n",
       "29996  people on a safari truck watching as they expl...   \n",
       "29997  children pose for a photograph on steps in a p...   \n",
       "29998  kids wearing pirate hats are brandishing toy s...   \n",
       "29999  bronze statue outdoors in foreground, large ca...   \n",
       "\n",
       "                                              story_text  \n",
       "0      and its magnificent trunk, larger than life it...  \n",
       "1      and its magnificent trunk, larger than life it...  \n",
       "2      and its magnificent trunk, larger than life it...  \n",
       "3      we found this tree when we were walking in a n...  \n",
       "4          it turns out it is a popular attraction here.  \n",
       "5      the tree is very unusual, with its roots exposed.  \n",
       "6         the trunk was really wide, as much as 12 feet!  \n",
       "7      you can see how big these roots are - pretty a...  \n",
       "8      we found this tree when we were walking in a n...  \n",
       "9          it turns out it is a popular attraction here.  \n",
       "10     the tree is very unusual, with its roots exposed.  \n",
       "11        the trunk was really wide, as much as 12 feet!  \n",
       "12     you can see how big these roots are - pretty a...  \n",
       "13     we found this tree when we were walking in a n...  \n",
       "14         it turns out it is a popular attraction here.  \n",
       "15     the tree is very unusual, with its roots exposed.  \n",
       "16        the trunk was really wide, as much as 12 feet!  \n",
       "17     you can see how big these roots are - pretty a...  \n",
       "18                some more different parts of the tree.  \n",
       "19                some more different parts of the tree.  \n",
       "20                some more different parts of the tree.  \n",
       "21     they went to the botanic gardens specifically ...  \n",
       "22       there was an informational sign posted near it.  \n",
       "23     the roots were huge and spread out over the gr...  \n",
       "24             the trunk was incredibly thick and rigid.  \n",
       "25     the large roots were almost as thick as the tr...  \n",
       "26     they went to the botanic gardens specifically ...  \n",
       "27       there was an informational sign posted near it.  \n",
       "28     the roots were huge and spread out over the gr...  \n",
       "29             the trunk was incredibly thick and rigid.  \n",
       "...                                                  ...  \n",
       "29970                              the fair was in town.  \n",
       "29971            [female] was so excited to see animals.  \n",
       "29972     sheep were there, but better yet there were. .  \n",
       "29973                      alpacas! the kids loved them.  \n",
       "29974             and after they learned all about them.  \n",
       "29975                              the fair was in town.  \n",
       "29976            [female] was so excited to see animals.  \n",
       "29977     sheep were there, but better yet there were. .  \n",
       "29978                      alpacas! the kids loved them.  \n",
       "29979             and after they learned all about them.  \n",
       "29980                              the fair was in town.  \n",
       "29981            [female] was so excited to see animals.  \n",
       "29982     sheep were there, but better yet there were. .  \n",
       "29983                      alpacas! the kids loved them.  \n",
       "29984             and after they learned all about them.  \n",
       "29985  we went to organization last summer for our va...  \n",
       "29986            we got to ride so many different rides.  \n",
       "29987                  the kids were having so much fun.  \n",
       "29988  they loved going around to all the different s...  \n",
       "29989  we had such a good time and can't wait to come...  \n",
       "29990  we went to organization last summer for our va...  \n",
       "29991            we got to ride so many different rides.  \n",
       "29992                  the kids were having so much fun.  \n",
       "29993  they loved going around to all the different s...  \n",
       "29994  we had such a good time and can't wait to come...  \n",
       "29995  we went to organization last summer for our va...  \n",
       "29996            we got to ride so many different rides.  \n",
       "29997                  the kids were having so much fun.  \n",
       "29998  they loved going around to all the different s...  \n",
       "29999  we had such a good time and can't wait to come...  \n",
       "\n",
       "[30000 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df_all = df_story_and_desc_text.apply(lambda x: re_sub(x))\n",
    "clean_df_all.columns = ['description_text', 'story_text']\n",
    "clean_df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'getting the kids together to play was long overdue.'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df_all.story_text[2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    and its magnificent trunk, larger than life it...\n",
       "1    and its magnificent trunk, larger than life it...\n",
       "2    and its magnificent trunk, larger than life it...\n",
       "3    we found this tree when we were walking in a n...\n",
       "4        it turns out it is a popular attraction here.\n",
       "Name: story_text, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df_all.story_text.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate synthetic data\n",
    "<br / >"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0        big old tree being photographed on a sunny da\n",
       "1                   a old curvy tree in the sun light.\n",
       "2    a person is taking a picture of a large tree a...\n",
       "3    large tree with many outstretching branches an...\n",
       "4    a green sign is describing a historic tree and...\n",
       "Name: description_text, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (type(clean_df_all.description_text))\n",
    "clean_df_all.description_text[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    START_ and its magnificent trunk, larger than ...\n",
       "1    START_ and its magnificent trunk, larger than ...\n",
       "2    START_ and its magnificent trunk, larger than ...\n",
       "3    START_ we found this tree when we were walking...\n",
       "4    START_ it turns out it is a popular attraction...\n",
       "Name: story_text, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df_all.story_text = clean_df_all.story_text.apply(lambda x : 'START_ '+ x + ' _END')\n",
    "print (type(clean_df_all.story_text))\n",
    "clean_df_all.story_text[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7863, 9940)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_desc_words = set()\n",
    "for desc_paragraph in clean_df_all.description_text:\n",
    "    for word in desc_paragraph.split():\n",
    "        if word not in all_desc_words:\n",
    "            all_desc_words.add(word)\n",
    "            \n",
    "all_story_words = set()\n",
    "for story_parapgraph in clean_df_all.story_text:\n",
    "    for word in story_parapgraph.split():\n",
    "        if word not in all_story_words:\n",
    "            all_story_words.add(word)\n",
    "    \n",
    "len(all_desc_words), len(all_story_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc_lenght_list=[]\n",
    "for desc_paragraph in clean_df_all.description_text:\n",
    "    desc_lenght_list.append(len(desc_paragraph.split(' ')))\n",
    "\n",
    "max_desc_length = np.max(desc_lenght_list)\n",
    "max_desc_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "82"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "story_lenght_list=[]\n",
    "for story_parapgraph in clean_df_all.story_text:\n",
    "    story_lenght_list.append(len(story_parapgraph.split(' ')))\n",
    "\n",
    "max_story_length = np.max(story_lenght_list)\n",
    "max_story_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num input_words: 7863, num target_words: 9940\n"
     ]
    }
   ],
   "source": [
    "input_words = sorted(list(all_desc_words))\n",
    "target_words = sorted(list(all_story_words))\n",
    "num_encoder_tokens = len(all_desc_words)\n",
    "num_decoder_tokens = len(all_story_words)\n",
    "print (\"num input_words: {0}, num target_words: {1}\".format(num_encoder_tokens, num_decoder_tokens))\n",
    "# del all_eng_words, all_french_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_token_index = dict(\n",
    "    [(word, i) for i, word in enumerate(input_words)])\n",
    "target_token_index = dict(\n",
    "    [(word, i) for i, word in enumerate(target_words)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'chef': 1359,\n",
       " 'lapse.': 3862,\n",
       " 'glassed': 3044,\n",
       " 'stormy': 6651,\n",
       " 'frie': 2887,\n",
       " 'bearing': 635,\n",
       " 'button': 1091,\n",
       " 'attire,': 434,\n",
       " 'schoolhouse': 5872,\n",
       " 'uneaten': 7369,\n",
       " 'treehous': 7238,\n",
       " 'intersectio': 3650,\n",
       " 'load.': 4046,\n",
       " 'foregroun': 2827,\n",
       " 'retro': 5607,\n",
       " 'delapidated': 1975,\n",
       " 'bounce.': 889,\n",
       " 'steel': 6587,\n",
       " 'watch.': 7582,\n",
       " 'trip': 7262,\n",
       " 'crop': 1819,\n",
       " 'reluctant': 5558,\n",
       " 'fur.': 2933,\n",
       " 'smoky': 6302,\n",
       " 'gas': 2968,\n",
       " 'circles': 1427,\n",
       " 'parka': 4866,\n",
       " 'shepard.': 6021,\n",
       " 'cellular': 1284,\n",
       " 'wispy': 7746,\n",
       " 'eagerly': 2286,\n",
       " 'basketball': 596,\n",
       " 'illustration': 3564,\n",
       " 'for': 2822,\n",
       " 'shiny.': 6035,\n",
       " 'case.': 1235,\n",
       " 'mug': 4458,\n",
       " 'skylin': 6227,\n",
       " 'modern': 4369,\n",
       " 'sharing': 6000,\n",
       " 'drums.': 2251,\n",
       " 'practice': 5281,\n",
       " 'bottom.': 882,\n",
       " 'deck.|a': 1955,\n",
       " 'handle': 3264,\n",
       " 'water,': 7594,\n",
       " 'base.': 586,\n",
       " 'antenna': 293,\n",
       " 'streaking': 6678,\n",
       " 'attention.': 431,\n",
       " 'swirly': 6857,\n",
       " 'property.': 5359,\n",
       " 'advertises': 149,\n",
       " 'newspaper': 4560,\n",
       " 'ground.': 3194,\n",
       " 'luxury': 4138,\n",
       " '200': 17,\n",
       " 'brighten': 957,\n",
       " 'display': 2105,\n",
       " 'locked': 4062,\n",
       " 'hardware': 3296,\n",
       " 'bottles': 876,\n",
       " 'itself.': 3687,\n",
       " 'same.': 5805,\n",
       " 'description': 2012,\n",
       " 'monitor.': 4382,\n",
       " 'device.': 2041,\n",
       " 'pitch': 5105,\n",
       " 'mad': 4145,\n",
       " 'band.': 541,\n",
       " 'inequality': 3603,\n",
       " 'homework.': 3454,\n",
       " 'brownish': 984,\n",
       " 'operation': 4686,\n",
       " 'da': 1895,\n",
       " 'somethign': 6371,\n",
       " 'protestors.': 5369,\n",
       " 'mushroom': 4481,\n",
       " 'detonating': 2038,\n",
       " 'stir': 6616,\n",
       " 'lock.': 4061,\n",
       " 'alpaca': 221,\n",
       " 'surround': 6820,\n",
       " 'asain': 395,\n",
       " 'fight': 2642,\n",
       " 's': 5776,\n",
       " 'blan': 773,\n",
       " 'dress': 2207,\n",
       " 'backlighting': 490,\n",
       " 'plane': 5125,\n",
       " 'abandoned.': 80,\n",
       " 'beam': 627,\n",
       " '[female].': 56,\n",
       " 'awkwardly': 472,\n",
       " 'coffee.': 1547,\n",
       " 'computers.': 1625,\n",
       " 'stilts': 6612,\n",
       " 'stems.': 6595,\n",
       " 'magnifying': 4156,\n",
       " 'consisting': 1654,\n",
       " 'feature': 2596,\n",
       " 'win.': 7711,\n",
       " 'arid': 349,\n",
       " 'sits.': 6182,\n",
       " 'skis': 6219,\n",
       " 'four': 2864,\n",
       " 'reads': 5491,\n",
       " \"childrens'\": 1386,\n",
       " 'work': 7787,\n",
       " 'color': 1563,\n",
       " 'complex.': 1619,\n",
       " 'pitched': 5106,\n",
       " 'dried': 2217,\n",
       " 'cow,': 1767,\n",
       " 'sack': 5778,\n",
       " 'ones.': 4665,\n",
       " 'sparkler.': 6405,\n",
       " 'brilliantly': 966,\n",
       " 'reception.': 5509,\n",
       " 'heard': 3336,\n",
       " 'press': 5308,\n",
       " 'plate': 5141,\n",
       " 'boy,': 906,\n",
       " 'vegas': 7437,\n",
       " 'camping': 1140,\n",
       " 'lils': 3998,\n",
       " 'bulding': 1031,\n",
       " 'give': 3034,\n",
       " 'hauling': 3310,\n",
       " 'performers': 4984,\n",
       " 'shelf.': 6015,\n",
       " 'book': 858,\n",
       " 'tents': 6988,\n",
       " 'wavy': 7618,\n",
       " 'ceremon': 1300,\n",
       " 'spoons': 6470,\n",
       " 'ships.': 6039,\n",
       " 'wires': 7745,\n",
       " 'blur': 822,\n",
       " 'dust': 2277,\n",
       " 'perfectly': 4977,\n",
       " 'haired': 3242,\n",
       " 'lose': 4105,\n",
       " 'dominate': 2156,\n",
       " 'harness': 3300,\n",
       " 'fairgrounds.': 2542,\n",
       " 'blonde': 797,\n",
       " 'projections': 5344,\n",
       " 'memorials': 4280,\n",
       " 'freshly': 2885,\n",
       " 'riders.': 5631,\n",
       " 'glittering': 3051,\n",
       " 'tiles': 7084,\n",
       " 'close-up': 1483,\n",
       " 'spiky': 6443,\n",
       " 'beside': 698,\n",
       " 'angles': 269,\n",
       " 'man,': 4179,\n",
       " 'admire': 127,\n",
       " 'gear': 2990,\n",
       " 'those': 7036,\n",
       " 'cups.': 1860,\n",
       " 'trees,': 7243,\n",
       " 'store': 6640,\n",
       " 'customers.': 1877,\n",
       " 'meter': 4310,\n",
       " 'sticks': 6608,\n",
       " 'skyscrapers.': 6233,\n",
       " 'cabin': 1101,\n",
       " 'streak': 6677,\n",
       " 'family': 2554,\n",
       " 'commode': 1600,\n",
       " 'bottling': 879,\n",
       " 'sailboa': 5788,\n",
       " 'or': 4692,\n",
       " 'folk': 2803,\n",
       " 'flags.': 2726,\n",
       " 'plates': 5143,\n",
       " 'socialize.': 6344,\n",
       " 'floor.': 2761,\n",
       " 'push': 5402,\n",
       " \"tourist''\": 7161,\n",
       " 'viewed.': 7481,\n",
       " 'drums': 2250,\n",
       " 'tries': 7255,\n",
       " 'cheer.': 1347,\n",
       " 'forefront': 2826,\n",
       " 'quickly.': 5417,\n",
       " 'screw': 5889,\n",
       " 'woodchuck.': 7775,\n",
       " 'cheerleaders,': 1352,\n",
       " 'community': 1602,\n",
       " 'dunking': 2271,\n",
       " 'trails.': 7206,\n",
       " 'traveling': 7222,\n",
       " 'release': 5554,\n",
       " 'drive': 2228,\n",
       " 'dances': 1909,\n",
       " 'function.': 2926,\n",
       " 'tank': 6915,\n",
       " 'bud': 1003,\n",
       " 'park-like': 4863,\n",
       " 'limited': 4001,\n",
       " 'resting.': 5598,\n",
       " 'icon': 3552,\n",
       " 'sips': 6171,\n",
       " '12': 10,\n",
       " 'pointing': 5185,\n",
       " 'mountainside.': 4425,\n",
       " 'batman': 609,\n",
       " 'items': 3682,\n",
       " 'working': 7791,\n",
       " 'volleyball.': 7515,\n",
       " 'upright': 7401,\n",
       " 'smiling,': 6292,\n",
       " 'shops.': 6063,\n",
       " 'delicious': 1979,\n",
       " 'grap': 3129,\n",
       " 'merry-go': 4297,\n",
       " 'law': 3889,\n",
       " 'college': 1560,\n",
       " 'ahead': 175,\n",
       " 'suits,': 6768,\n",
       " 'them': 7011,\n",
       " 'tubular': 7303,\n",
       " 'strin': 6705,\n",
       " 'godiva': 3071,\n",
       " 'cascading': 1233,\n",
       " 'facade': 2519,\n",
       " 'cross': 1821,\n",
       " 'intent': 3641,\n",
       " 'stage,': 6517,\n",
       " 'grilling.': 3177,\n",
       " 'bot': 870,\n",
       " 'riding': 5634,\n",
       " 'carousel.': 1206,\n",
       " 'put': 5406,\n",
       " 'conversing': 1693,\n",
       " 'god': 3070,\n",
       " 'knitted': 3796,\n",
       " 'hait': 3244,\n",
       " 'mic.': 4315,\n",
       " 'sparkly': 6409,\n",
       " 'rings': 5644,\n",
       " 'cheerleaders': 1351,\n",
       " 'mixed': 4359,\n",
       " 'family.': 2557,\n",
       " 'underside': 7364,\n",
       " 'it.': 3678,\n",
       " 'spoon.': 6469,\n",
       " 'danced': 1906,\n",
       " 'topless': 7140,\n",
       " 'run.': 5756,\n",
       " 'harbor': 3291,\n",
       " 'grill.': 3174,\n",
       " 'shallow': 5993,\n",
       " \"name'analyssa'\": 4506,\n",
       " 'shield': 6023,\n",
       " 'mouse': 4428,\n",
       " 'shoreline.': 6068,\n",
       " 'listens': 4026,\n",
       " 'buckets.': 999,\n",
       " 'vivid': 7508,\n",
       " 'facility.': 2528,\n",
       " 'bare': 569,\n",
       " 'visit': 7502,\n",
       " '80th': 48,\n",
       " 'shirtless': 6044,\n",
       " 'bought.': 885,\n",
       " 'lucen': 4125,\n",
       " 'belts': 678,\n",
       " 'bikes.': 735,\n",
       " 'walkers': 7544,\n",
       " 'red,': 5522,\n",
       " 'mountainous,': 4420,\n",
       " 'horn.': 3478,\n",
       " \"[location]''\": 59,\n",
       " 'attractive': 441,\n",
       " 'messing': 4303,\n",
       " 'excited.': 2450,\n",
       " 'squirting': 6504,\n",
       " 'japanese.': 3700,\n",
       " 'faculty': 2533,\n",
       " 'sunday,': 6779,\n",
       " 'industrial': 3602,\n",
       " \"toilet''\": 7120,\n",
       " 'vinyl': 7490,\n",
       " 'intake': 3638,\n",
       " 'brightens': 958,\n",
       " 'bursting,': 1064,\n",
       " 'works': 7793,\n",
       " 'sliced': 6251,\n",
       " 'included.': 3587,\n",
       " 'burgandy': 1049,\n",
       " 'attache': 421,\n",
       " 'edges': 2312,\n",
       " 'besides': 699,\n",
       " 'coconut': 1544,\n",
       " 'nose-to-nose': 4596,\n",
       " 'headscarf': 3333,\n",
       " 'cathedral': 1260,\n",
       " 'escorted': 2414,\n",
       " 'boy.': 907,\n",
       " 'waters.': 7608,\n",
       " 'barbed': 566,\n",
       " 'grins': 3183,\n",
       " 'step': 6596,\n",
       " 'repairs.': 5575,\n",
       " 'wreaks': 7808,\n",
       " 'looking': 4092,\n",
       " '[male': 64,\n",
       " 'walled': 7555,\n",
       " 'honor': 3456,\n",
       " 'once.': 4659,\n",
       " 'crossbones': 1824,\n",
       " 'hunting': 3536,\n",
       " 'wing': 7731,\n",
       " 'sugar': 6761,\n",
       " 'tiaras': 7063,\n",
       " 'erected': 2410,\n",
       " 'desig': 2016,\n",
       " 'erotically': 2412,\n",
       " 'pinata': 5087,\n",
       " 'brush': 988,\n",
       " '[male].': 67,\n",
       " '5': 36,\n",
       " 'rising': 5654,\n",
       " 'chips.': 1397,\n",
       " 'clap': 1442,\n",
       " 'darkness.': 1926,\n",
       " 'depart': 1993,\n",
       " 'metal.': 4307,\n",
       " 'kart': 3752,\n",
       " 'convertible': 1695,\n",
       " 'golden': 3083,\n",
       " 'bathroo': 605,\n",
       " 'distorted': 2120,\n",
       " 'screen': 5884,\n",
       " 'snac': 6306,\n",
       " 'niche': 4566,\n",
       " 'waterfall.': 7598,\n",
       " 'tipoff': 7098,\n",
       " 'poker': 5190,\n",
       " 'colorful': 1568,\n",
       " 'sails': 5794,\n",
       " 'crate': 1792,\n",
       " 'audience.': 445,\n",
       " 'carbonated!': 1186,\n",
       " 're': 5479,\n",
       " 'shopping.': 6061,\n",
       " 'characters.': 1329,\n",
       " 'mysterious': 4496,\n",
       " 'stopping': 6636,\n",
       " 'truck.': 7279,\n",
       " 'voting': 7519,\n",
       " 'crawls': 1796,\n",
       " 'blacked': 768,\n",
       " 'pens.': 4966,\n",
       " 'polo': 5207,\n",
       " 'proximity': 5376,\n",
       " 'toppings': 7143,\n",
       " 'rainy': 5452,\n",
       " 'sidewal': 6124,\n",
       " 'figurine': 2649,\n",
       " 'included': 3586,\n",
       " 'prop/wing': 5355,\n",
       " 'theater': 7007,\n",
       " 'closing': 1492,\n",
       " 'beach': 620,\n",
       " 'shot.': 6076,\n",
       " 'surroundings': 6823,\n",
       " 'blanket': 776,\n",
       " 'converted': 1694,\n",
       " 'closet': 1489,\n",
       " 'surrounds': 6824,\n",
       " 'roo': 5704,\n",
       " 'shared': 5999,\n",
       " 'streaks': 6679,\n",
       " 'embraces': 2349,\n",
       " 'laptops.': 3866,\n",
       " 'black.': 766,\n",
       " 'eagle': 2287,\n",
       " 'bass.': 600,\n",
       " 'faithful': 2544,\n",
       " 'vest,': 7466,\n",
       " 'celebrating': 1279,\n",
       " 'floating': 2756,\n",
       " 'return': 5608,\n",
       " 'flume': 2779,\n",
       " 'spear': 6419,\n",
       " 'smiley': 6289,\n",
       " 'puts': 5407,\n",
       " 'feed': 2604,\n",
       " 'troph': 7272,\n",
       " 'seated.': 5914,\n",
       " 'baby.': 479,\n",
       " 'gates': 2974,\n",
       " 'baseball.': 588,\n",
       " 'earth': 2296,\n",
       " 'gloomy': 3053,\n",
       " 'stitching': 6620,\n",
       " 'grou': 3190,\n",
       " 'huddled': 3514,\n",
       " 'tents.': 6990,\n",
       " 'may': 4248,\n",
       " 'clockwise': 1480,\n",
       " 'lush,': 4135,\n",
       " 'beast': 638,\n",
       " 'waffle': 7524,\n",
       " 'low': 4123,\n",
       " 'washington': 7578,\n",
       " 'interesting': 3646,\n",
       " 'whisk': 7684,\n",
       " 'inscribed': 3622,\n",
       " 'wineglass': 7729,\n",
       " 'round.': 5733,\n",
       " 'gummi': 3220,\n",
       " 'schoo': 5867,\n",
       " 'ladies': 3819,\n",
       " 't-shir': 6868,\n",
       " 'had': 3237,\n",
       " 'non-actors': 4586,\n",
       " 'abandoned,': 79,\n",
       " 'lick': 3968,\n",
       " 'people.': 4971,\n",
       " 'acne': 105,\n",
       " '429-8044,': 33,\n",
       " 'downtown.': 2184,\n",
       " 'imag': 3566,\n",
       " 'houses': 3505,\n",
       " 'toot': 7136,\n",
       " 'missio': 4355,\n",
       " 'spiderman': 6441,\n",
       " 'camped': 1135,\n",
       " 'bulbs.': 1030,\n",
       " 'putting': 5408,\n",
       " \"soldier's\": 6360,\n",
       " 'presses': 5309,\n",
       " 'special': 6420,\n",
       " 'flame': 2729,\n",
       " 'stands,': 6541,\n",
       " 'amber': 235,\n",
       " \"here''\": 3375,\n",
       " 'butterfl': 1086,\n",
       " 'containers.': 1665,\n",
       " 'vin': 7487,\n",
       " 'ditch.': 2122,\n",
       " 'raised': 5454,\n",
       " 'apparently': 305,\n",
       " 'graduation': 3116,\n",
       " 'fish': 2704,\n",
       " 'pretentiously': 5311,\n",
       " 'shirts,': 6046,\n",
       " 'stones.': 6629,\n",
       " 'prepared': 5296,\n",
       " 'pedals': 4950,\n",
       " 'melted': 4272,\n",
       " 'oval': 4750,\n",
       " 'skydiver': 6225,\n",
       " 'comical': 1596,\n",
       " 'twiligh': 7331,\n",
       " 'steeples.': 6591,\n",
       " 'tassels.': 6928,\n",
       " 'wihle': 7701,\n",
       " 'booths': 867,\n",
       " 'rendered,': 5571,\n",
       " 'largest': 3870,\n",
       " 'beautifully': 643,\n",
       " 'top,': 7138,\n",
       " 'stri': 6702,\n",
       " 'artfully': 379,\n",
       " 'opinion.': 4688,\n",
       " 'remarking': 5561,\n",
       " 'placed': 5116,\n",
       " 'recording': 5517,\n",
       " 'gravy': 3145,\n",
       " 'navy': 4521,\n",
       " 'coat': 1534,\n",
       " 'blous': 805,\n",
       " 'tickets': 7065,\n",
       " 'people,': 4970,\n",
       " 'lake.': 3826,\n",
       " 'depicted': 1995,\n",
       " 'none': 4589,\n",
       " 'discussion': 2094,\n",
       " 'characters': 1328,\n",
       " 'shine': 6027,\n",
       " 'cheese.': 1355,\n",
       " 'wipes': 7741,\n",
       " 'security': 5922,\n",
       " 'rain': 5446,\n",
       " 'arise': 350,\n",
       " 'toy': 7181,\n",
       " 'pours': 5273,\n",
       " 'facility': 2527,\n",
       " 'concrete.': 1636,\n",
       " 'magnetized': 4152,\n",
       " 'nature': 4519,\n",
       " 'waterfron': 7600,\n",
       " 'coasters.': 1532,\n",
       " 'nervous': 4544,\n",
       " 'characteristics': 1327,\n",
       " 'pretty': 5312,\n",
       " 'whick': 7680,\n",
       " 'tricks': 7251,\n",
       " 'stand': 6533,\n",
       " 'saxaphon': 5839,\n",
       " 'looks': 4094,\n",
       " 'perfect': 4976,\n",
       " 'i': 3543,\n",
       " 'seed': 5927,\n",
       " 'spinn': 6448,\n",
       " 'contestants.': 1676,\n",
       " 'patches': 4907,\n",
       " 'megaphones': 4270,\n",
       " 'casting': 1244,\n",
       " 'flag': 2723,\n",
       " 'approaching': 318,\n",
       " 'chevrolet': 1368,\n",
       " 'awareness.': 464,\n",
       " 'cannoli,': 1160,\n",
       " 'casina': 1238,\n",
       " 'clored': 1481,\n",
       " 'dinner': 2067,\n",
       " 'bright': 954,\n",
       " 'concrete': 1635,\n",
       " 'rop': 5718,\n",
       " 'taking.': 6897,\n",
       " 'bonnet': 856,\n",
       " 'sand.': 5812,\n",
       " 'focus.': 2789,\n",
       " 'gun.': 3223,\n",
       " 'fire': 2679,\n",
       " 'sewing': 5977,\n",
       " 'globe': 3052,\n",
       " 'coastline': 1533,\n",
       " 'content.': 1671,\n",
       " 'indoor': 3599,\n",
       " 'zones': 7860,\n",
       " 'cave.': 1269,\n",
       " 'stick.': 6606,\n",
       " 'quick': 5416,\n",
       " 'skewer': 6204,\n",
       " 'quote': 5422,\n",
       " 'blades': 772,\n",
       " 'apron': 320,\n",
       " 'consumption.': 1660,\n",
       " 'bands.': 547,\n",
       " 'hopes': 3471,\n",
       " 'itself': 3686,\n",
       " 'underwater': 7365,\n",
       " 'billboard': 740,\n",
       " 'noon.': 4591,\n",
       " 'raiser.': 5456,\n",
       " 'receive': 5503,\n",
       " 'tribute': 7249,\n",
       " 'bag.': 510,\n",
       " 'stained': 6520,\n",
       " 'shell': 6016,\n",
       " 'draped': 2196,\n",
       " 'smokes': 6298,\n",
       " 'surrounded': 6821,\n",
       " 'campu': 1143,\n",
       " 'lifting': 3978,\n",
       " 'bridge': 947,\n",
       " 'realm': 5498,\n",
       " 'shingles': 6031,\n",
       " 'drug': 2244,\n",
       " 'googlie': 3095,\n",
       " 'sail': 5787,\n",
       " 'disco,': 2089,\n",
       " 'hangings.': 3276,\n",
       " 'below': 674,\n",
       " 'someone.': 6369,\n",
       " 'directional': 2077,\n",
       " 'charcoal': 1330,\n",
       " 'skep.': 6202,\n",
       " 'funera': 2928,\n",
       " \"isn't\": 3673,\n",
       " 'surprise': 6817,\n",
       " 'saris': 5824,\n",
       " 'tied': 7073,\n",
       " 'nails.': 4501,\n",
       " '[location': 57,\n",
       " 'dusted': 2278,\n",
       " 'coil': 1548,\n",
       " 'stemming': 6594,\n",
       " 'halfway': 3247,\n",
       " 'congregation': 1648,\n",
       " 'boy': 904,\n",
       " 'cooker.': 1704,\n",
       " 'crowd.': 1835,\n",
       " 'enter.': 2384,\n",
       " 'sponsors.': 6466,\n",
       " 'hey': 3383,\n",
       " 'parade,': 4848,\n",
       " 'blankets': 779,\n",
       " 'member': 4275,\n",
       " 'hippy': 3419,\n",
       " 'streets': 6692,\n",
       " 'photographed.': 5023,\n",
       " 'mans': 4186,\n",
       " 'exploding.': 2480,\n",
       " 'artificial': 381,\n",
       " 'target': 6922,\n",
       " 'nearb': 4524,\n",
       " 'flew': 2748,\n",
       " 'selfie,': 5941,\n",
       " 'fusses': 2936,\n",
       " 'board,': 832,\n",
       " 'overexposed': 4759,\n",
       " 'ornament': 4708,\n",
       " 'sales': 5801,\n",
       " 'tools': 7135,\n",
       " 'throughout': 7051,\n",
       " 'focuses': 2790,\n",
       " 'indoors': 3600,\n",
       " 'fabric': 2516,\n",
       " 'grows': 3203,\n",
       " 'innovative,': 3619,\n",
       " 'litter.': 4034,\n",
       " 'destination.': 2031,\n",
       " 'sand,': 5810,\n",
       " 'cottag': 1734,\n",
       " 'platform': 5145,\n",
       " 'her,': 3369,\n",
       " 'blacvk': 770,\n",
       " 'stream': 6680,\n",
       " 'saturated': 5831,\n",
       " 'males': 4170,\n",
       " 'bursts': 1065,\n",
       " 'tortilla': 7147,\n",
       " 'googly': 3096,\n",
       " 'upstairs': 7403,\n",
       " 'roadway.': 5670,\n",
       " 'banner': 553,\n",
       " 'player': 5152,\n",
       " 'balloon': 529,\n",
       " 'choice.': 1401,\n",
       " 'violin.': 7494,\n",
       " 'mercury': 4291,\n",
       " 'around': 360,\n",
       " 'funny': 2931,\n",
       " 'sno': 6319,\n",
       " 'ca': 1100,\n",
       " 'asphalt.': 404,\n",
       " 'police.': 5199,\n",
       " 'embrace': 2348,\n",
       " 'circling': 1428,\n",
       " 'workers': 7790,\n",
       " 'sushi': 6826,\n",
       " 'midst': 4333,\n",
       " 'marketplace.': 4211,\n",
       " 'desaturated': 2003,\n",
       " 'skewers': 6205,\n",
       " 'crown': 1840,\n",
       " 'necklace': 4533,\n",
       " 'touching,': 7155,\n",
       " 'cards.': 1194,\n",
       " 'chewbacca': 1369,\n",
       " 'everyone': 2437,\n",
       " 'lawn,': 3891,\n",
       " 'twirling': 7336,\n",
       " 'square.': 6494,\n",
       " 'simultaneously': 6156,\n",
       " 'product': 5333,\n",
       " 'patch': 4906,\n",
       " 'cacti': 1104,\n",
       " 'which': 7679,\n",
       " \"what's\": 7662,\n",
       " 'slide.': 6256,\n",
       " 'good': 3089,\n",
       " 'shaded': 5983,\n",
       " 'faucet': 2587,\n",
       " 'convenes': 1685,\n",
       " 'between': 704,\n",
       " 'wear': 7625,\n",
       " 'biscuits': 756,\n",
       " 'pants,': 4837,\n",
       " 'paw': 4937,\n",
       " 'walking': 7546,\n",
       " 'admires': 129,\n",
       " 'marathon': 4196,\n",
       " 'touching': 7154,\n",
       " 'happily': 3283,\n",
       " 'better.': 703,\n",
       " 'stripper': 6715,\n",
       " 'his.': 3421,\n",
       " 'smiles': 6286,\n",
       " 'twenty': 7328,\n",
       " 'breeze': 938,\n",
       " 'reflective': 5536,\n",
       " 'yelling': 7835,\n",
       " 'farm': 2571,\n",
       " 'strapped': 6664,\n",
       " 'cars,': 1218,\n",
       " 'speech': 6430,\n",
       " 'hedge': 3347,\n",
       " 'epcot': 2404,\n",
       " 'concert.': 1631,\n",
       " \"sailor's\": 5792,\n",
       " 'specialties': 6421,\n",
       " 'black': 763,\n",
       " 'barbecue': 563,\n",
       " 'under': 7361,\n",
       " 'sort.': 6384,\n",
       " 'following.': 2809,\n",
       " 'sauces': 5833,\n",
       " 'teens,': 6962,\n",
       " 'dumpster': 2266,\n",
       " 'came': 1120,\n",
       " 'th': 6999,\n",
       " 'boat': 840,\n",
       " 'wal': 7539,\n",
       " 'signage': 6138,\n",
       " 'pots.': 5265,\n",
       " 'complex': 1617,\n",
       " 'common': 1601,\n",
       " 'striking': 6704,\n",
       " 'lawnmower.': 3893,\n",
       " 'trio': 7260,\n",
       " 'hearts': 3339,\n",
       " 'darkness': 1925,\n",
       " 'flaming': 2733,\n",
       " 'storefront.': 6642,\n",
       " 'well.': 7654,\n",
       " 'says': 5845,\n",
       " 'pillars.': 5078,\n",
       " 'ducks': 2261,\n",
       " 'festivities': 2630,\n",
       " 'gathered,': 2978,\n",
       " 'highlights': 3395,\n",
       " 'hoop': 3468,\n",
       " 'yar': 7822,\n",
       " 'multicolored': 4466,\n",
       " 'elves': 2343,\n",
       " 'happ': 3280,\n",
       " 'land.': 3835,\n",
       " 'cars': 1217,\n",
       " 'shadows': 5988,\n",
       " 'ukulele': 7350,\n",
       " 'barges': 574,\n",
       " 'horse-riding': 3484,\n",
       " 'names': 4510,\n",
       " 'drink.': 2220,\n",
       " 'fellow': 2611,\n",
       " 'exterior': 2501,\n",
       " \"'\": 2,\n",
       " 'jackets': 3694,\n",
       " 'brow': 980,\n",
       " 'aircraft.': 184,\n",
       " 'danc': 1904,\n",
       " 'spreading': 6483,\n",
       " 'backpacks': 495,\n",
       " 'ball': 526,\n",
       " 'where': 7676,\n",
       " 'bear': 630,\n",
       " 'hydraulic': 3542,\n",
       " 'filled': 2654,\n",
       " 'triomphe.': 7261,\n",
       " 'amazing': 234,\n",
       " 'ocea': 4630,\n",
       " 'stepping': 6599,\n",
       " 'plate.': 5142,\n",
       " 'him': 3414,\n",
       " 'recognition.': 5515,\n",
       " 'tassles': 6929,\n",
       " 'auditorium': 446,\n",
       " 'clear': 1456,\n",
       " 'rainbo': 5448,\n",
       " 'sight': 6132,\n",
       " 'seaso': 5907,\n",
       " 'scape': 5851,\n",
       " 'scrub': 5891,\n",
       " 'measuring': 4255,\n",
       " 'corndo': 1718,\n",
       " 'outline': 4740,\n",
       " 'directions': 2078,\n",
       " 'stall.': 6530,\n",
       " 'pitcher.': 5108,\n",
       " 'spanish': 6399,\n",
       " 'standing,': 6538,\n",
       " 'descent': 2007,\n",
       " 'visibility.': 7498,\n",
       " 'finishing': 2677,\n",
       " 'ground,': 3193,\n",
       " 'stretch': 6696,\n",
       " 'cameraman': 1126,\n",
       " 'panoramic': 4834,\n",
       " 'counter.': 1743,\n",
       " 'contemporary': 1669,\n",
       " 'announces': 285,\n",
       " 'chines': 1392,\n",
       " 'turtle': 7322,\n",
       " 'grow': 3200,\n",
       " 'sets.': 5970,\n",
       " 'really': 5497,\n",
       " 'group': 3197,\n",
       " 'tshirt': 7295,\n",
       " 'dots': 2172,\n",
       " 'teeter': 6964,\n",
       " 'ends': 2366,\n",
       " 'wings': 7733,\n",
       " 'bring': 967,\n",
       " 'animation': 279,\n",
       " 'concret': 1634,\n",
       " 'revolutionary': 5614,\n",
       " 'onlookers.': 4670,\n",
       " 'seagull': 5901,\n",
       " 'country.': 1750,\n",
       " 'aging': 171,\n",
       " 'dog': 2141,\n",
       " 'backdrop': 483,\n",
       " 'dials': 2046,\n",
       " 'smoke.': 6297,\n",
       " 'affixed': 155,\n",
       " 'bitten': 761,\n",
       " 'assortment': 410,\n",
       " 'fixtures.': 2720,\n",
       " 'chicken': 1373,\n",
       " 'footstand.': 2821,\n",
       " 'firecracker': 2683,\n",
       " 'bottles,': 877,\n",
       " 'was': 7573,\n",
       " 'lottery': 4114,\n",
       " 'robe': 5674,\n",
       " 'father': 2584,\n",
       " 'ruins.': 5753,\n",
       " '[female': 53,\n",
       " 'poetry': 5181,\n",
       " 'christmas': 1411,\n",
       " 'non-english': 4587,\n",
       " 'orange,': 4694,\n",
       " 'bracelets': 911,\n",
       " 'person': 4995,\n",
       " 'barrette': 579,\n",
       " 'such': 6758,\n",
       " 'ghoulish': 3011,\n",
       " 'lives': 4040,\n",
       " 'restored': 5600,\n",
       " 'stuff.': 6739,\n",
       " 'desk.': 2024,\n",
       " 'name': 4505,\n",
       " 'mom': 4372,\n",
       " 'monster': 4387,\n",
       " 'aloft.': 214,\n",
       " 'basket.': 594,\n",
       " 'arched': 331,\n",
       " 'clothe': 1495,\n",
       " 'playground.|siseven': 5162,\n",
       " 'icecream.': 3549,\n",
       " 'streetlight': 6690,\n",
       " 'stared': 6554,\n",
       " 'fighting': 2646,\n",
       " 'trombones': 7268,\n",
       " 'china': 1391,\n",
       " 'spider': 6440,\n",
       " 'rippons': 5650,\n",
       " 'panel.': 4829,\n",
       " 'icy': 3555,\n",
       " 'dial': 2045,\n",
       " 'happy,': 3287,\n",
       " 'beauty': 644,\n",
       " 'traps.': 7219,\n",
       " 'english': 2370,\n",
       " 'catches': 1256,\n",
       " 'child,': 1380,\n",
       " '#': 0,\n",
       " 'plahing': 5119,\n",
       " 'facing': 2529,\n",
       " 'statue.': 6573,\n",
       " 'unique': 7380,\n",
       " 'city.': 1435,\n",
       " 'arrived': 369,\n",
       " 'gorilla': 3098,\n",
       " 'spoken': 6463,\n",
       " 'perform.': 4979,\n",
       " 'spinns': 6450,\n",
       " 'skylight': 6226,\n",
       " 'residence': 5586,\n",
       " 'orga': 4702,\n",
       " 'columns.': 1587,\n",
       " 'conversation.': 1691,\n",
       " 'streetlights': 6691,\n",
       " 'coolers': 1712,\n",
       " 'eating': 2306,\n",
       " 'hedges': 3348,\n",
       " 'taxiing': 6937,\n",
       " 'signing': 6141,\n",
       " 'plaid': 5120,\n",
       " 'went': 7656,\n",
       " 'teammates,': 6952,\n",
       " 'angelic': 264,\n",
       " 'charts': 1335,\n",
       " 'street': 6687,\n",
       " 'frame': 2870,\n",
       " 'walkin': 7545,\n",
       " 'pink': 5092,\n",
       " 'flopping': 2763,\n",
       " 'dead': 1947,\n",
       " 'favorite': 2588,\n",
       " 'stain': 6519,\n",
       " 'sort': 6383,\n",
       " 'swim.': 6841,\n",
       " 'talented': 6899,\n",
       " 'gras': 3135,\n",
       " 'guiding': 3214,\n",
       " 'hills': 3408,\n",
       " 'tugboat': 7305,\n",
       " 'idea': 3556,\n",
       " 'whee': 7663,\n",
       " 'hash': 3302,\n",
       " 'lazer': 3897,\n",
       " 'color.': 1564,\n",
       " 'decorate': 1960,\n",
       " 'posin': 5241,\n",
       " 'loves': 4121,\n",
       " 'galler': 2940,\n",
       " 'behold.': 670,\n",
       " 'activists': 115,\n",
       " 'stores': 6643,\n",
       " 'slowly': 6274,\n",
       " 'cloth.': 1494,\n",
       " \"oasis''\": 4615,\n",
       " 'trail.': 7203,\n",
       " 'backpacker': 494,\n",
       " 'cold': 1553,\n",
       " 'front-end': 2910,\n",
       " 'losing': 4107,\n",
       " 'presents': 5305,\n",
       " 'cane': 1156,\n",
       " 'simultaneously.': 6157,\n",
       " 'capital': 1173,\n",
       " 'rollercoaster': 5699,\n",
       " 'greeting': 3163,\n",
       " 'firebrick': 2682,\n",
       " 'brown,': 982,\n",
       " 'sandwich.': 5818,\n",
       " 'regarded': 5542,\n",
       " \"says'my\": 5846,\n",
       " 'furry': 2935,\n",
       " 'marble': 4199,\n",
       " 'lo': 4044,\n",
       " 'cartoon': 1223,\n",
       " 'beautiful': 640,\n",
       " 'here': 3374,\n",
       " 'domes': 2155,\n",
       " 'pushed': 5403,\n",
       " 'begins.': 665,\n",
       " 'dish.': 2098,\n",
       " 'outdoors': 4729,\n",
       " 'ma': 4140,\n",
       " 'moves': 4438,\n",
       " 'skimming': 6214,\n",
       " 'watche': 7583,\n",
       " 'weathered': 7634,\n",
       " 'owl': 4779,\n",
       " 'homes': 3453,\n",
       " 'pants.': 4838,\n",
       " 'depicting': 1996,\n",
       " 'asking': 400,\n",
       " 'chick': 1372,\n",
       " 'lightyear': 3992,\n",
       " 'guards.': 3209,\n",
       " 'cake.': 1112,\n",
       " 'moments.to': 4377,\n",
       " 'objects': 4619,\n",
       " 'spell': 6433,\n",
       " 'sprinkler.': 6489,\n",
       " 'preparation': 5293,\n",
       " 'african': 156,\n",
       " 'tattoos': 6935,\n",
       " 'affection': 154,\n",
       " 'add': 121,\n",
       " 'intently': 3642,\n",
       " 'interact': 3643,\n",
       " 'antenna.': 294,\n",
       " 'monitors': 4383,\n",
       " 'attempting': 425,\n",
       " 'veterans': 7469,\n",
       " 'frosted': 2912,\n",
       " 'covering': 1764,\n",
       " 'year': 7829,\n",
       " 'mammoth': 4176,\n",
       " 'people': 4969,\n",
       " 'gree': 3154,\n",
       " 'fun': 2923,\n",
       " 'roses.': 5725,\n",
       " 'thousands': 7040,\n",
       " 'gateway': 2975,\n",
       " 'advert': 143,\n",
       " 'year.': 7831,\n",
       " 'title.': 7101,\n",
       " 'steaks,': 6584,\n",
       " 'eiffel': 2322,\n",
       " 'grew': 3165,\n",
       " 'pick-up': 5038,\n",
       " 'performing.': 4986,\n",
       " 'mimicking': 4342,\n",
       " 'foamy': 2786,\n",
       " ...}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_token_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4771200000"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clean_df_all.story_text)*16*num_decoder_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_input_data = np.zeros(\n",
    "    (len(clean_df_all.description_text), max_desc_length),\n",
    "    dtype='float32')\n",
    "decoder_input_data = np.zeros(\n",
    "    (len(clean_df_all.story_text), max_story_length),\n",
    "     dtype='float32')\n",
    "decoder_target_data = np.zeros(\n",
    "    (len(clean_df_all.story_text), max_story_length, num_decoder_tokens),\n",
    "    dtype='float32')\n",
    "# decoder_target_data = np.zeros(\n",
    "#     (len(clean_df_all.story_text), 500),\n",
    "#     dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 54)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (encoder_input_data.shape)\n",
    "encoder_input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 82)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (decoder_input_data.shape)\n",
    "decoder_input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 82, 9940)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]]], dtype=float32)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (decoder_target_data.shape)\n",
    "decoder_target_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder_input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'chef': 1672,\n",
       " 'favorite.': 3356,\n",
       " 'the,': 8814,\n",
       " 'complete,': 1977,\n",
       " 'possibility': 6685,\n",
       " 'out,': 6151,\n",
       " 'admired': 176,\n",
       " 'stormy': 8388,\n",
       " 'prom.': 6829,\n",
       " 'ther': 8834,\n",
       " 'start!': 8282,\n",
       " 'schoolhouse': 7510,\n",
       " 'gasp.': 3785,\n",
       " 'exit.': 3176,\n",
       " 'dousing': 2729,\n",
       " 'standby': 8264,\n",
       " 'highway.': 4338,\n",
       " 'ponds.': 6647,\n",
       " \"didn't\": 2564,\n",
       " 'bangs,': 740,\n",
       " 'swim,': 8633,\n",
       " 'cocktail': 1876,\n",
       " 'knocking': 4956,\n",
       " 'gem': 3813,\n",
       " 'views,': 9435,\n",
       " 'anticipation': 399,\n",
       " 'architecture,': 475,\n",
       " 'reluctant': 7143,\n",
       " 'eat!': 2884,\n",
       " 'upsetting': 9336,\n",
       " 'victory': 9422,\n",
       " 'uh-oh,': 9259,\n",
       " 'gas': 3781,\n",
       " 'cultures': 2300,\n",
       " 'wow': 9850,\n",
       " 'farther': 3334,\n",
       " 'celebtated': 1578,\n",
       " 'gorgeous,': 3962,\n",
       " 'perfect.': 6410,\n",
       " 'basketball': 780,\n",
       " 'await': 628,\n",
       " 'reenactments': 7092,\n",
       " 'intend': 4700,\n",
       " 'for': 3615,\n",
       " 'degree.': 2468,\n",
       " 'mug': 5741,\n",
       " 'contort': 2079,\n",
       " 'attendees': 590,\n",
       " 'characters,': 1634,\n",
       " 'ea': 2854,\n",
       " 'modern': 5634,\n",
       " 'dangling': 2366,\n",
       " 'roofs': 7342,\n",
       " 'help.': 4288,\n",
       " 'practice': 6716,\n",
       " '2nd': 43,\n",
       " 'bottom.': 1131,\n",
       " 'expression': 3238,\n",
       " 'write.': 9866,\n",
       " 'water,': 9577,\n",
       " 'streaking': 8408,\n",
       " 'attention.': 594,\n",
       " 'mosaic.': 5680,\n",
       " 'property.': 6843,\n",
       " 'chiming': 1705,\n",
       " 'newspaper': 5874,\n",
       " 'hell!': 4285,\n",
       " 'dollars.': 2700,\n",
       " 'ground.': 4081,\n",
       " 'brighten': 1209,\n",
       " 'display': 2638,\n",
       " 'imagine': 4584,\n",
       " 'outlining': 6170,\n",
       " 'brothers.': 1232,\n",
       " 'itself.': 4792,\n",
       " 'next,': 5877,\n",
       " 'clicking': 1799,\n",
       " 'citizens.': 1761,\n",
       " 'rescue': 7183,\n",
       " 'device.': 2553,\n",
       " 'changed.': 1622,\n",
       " 'appetite': 437,\n",
       " 'fantasy': 3322,\n",
       " 'voice': 9473,\n",
       " 'mad': 5337,\n",
       " 'approached,': 452,\n",
       " 'nugget.': 5962,\n",
       " 'shores': 7734,\n",
       " 'boardwalk,': 1060,\n",
       " 'traverse.': 9123,\n",
       " 'da': 2341,\n",
       " 'suggest': 8520,\n",
       " 'bored,': 1115,\n",
       " 'starving': 8297,\n",
       " 'after.': 218,\n",
       " 'disbanded.': 2615,\n",
       " 'mushroom': 5762,\n",
       " 'allowed!': 292,\n",
       " 'temple.': 8772,\n",
       " 'cruised': 2282,\n",
       " 'adapted': 161,\n",
       " 'cheese.': 1667,\n",
       " 'cousin': 2177,\n",
       " 'explode.': 3207,\n",
       " 'coasters.': 1867,\n",
       " 'elation': 2925,\n",
       " 'green.': 4049,\n",
       " 'quack,': 6925,\n",
       " 's': 7405,\n",
       " 'dress': 2768,\n",
       " 'harbor.': 4205,\n",
       " 'gears.': 3811,\n",
       " 'color!': 1905,\n",
       " 'abandoned.': 95,\n",
       " 'beam': 819,\n",
       " '[female].': 78,\n",
       " 'pair!': 6251,\n",
       " '20': 41,\n",
       " 'turtle': 9236,\n",
       " 'coffee.': 1878,\n",
       " 'mostly': 5685,\n",
       " 'member': 5522,\n",
       " 'district': 2657,\n",
       " 'scents': 7500,\n",
       " 'casserole,': 1516,\n",
       " 'watch': 9566,\n",
       " 'organization': 6116,\n",
       " 'magnifying': 5352,\n",
       " 'solace': 8045,\n",
       " 'display!': 2639,\n",
       " 'launch,': 5050,\n",
       " 'consisting': 2041,\n",
       " 'feature': 3369,\n",
       " 'seaguls': 7545,\n",
       " 'enthusiast.': 3039,\n",
       " 'air,': 257,\n",
       " 'pregnan': 6734,\n",
       " 'waist': 9494,\n",
       " 'skis': 7902,\n",
       " 'fairly': 3285,\n",
       " 'reads': 7028,\n",
       " 'pelicans.': 6392,\n",
       " 'work': 9821,\n",
       " 'color': 1904,\n",
       " 'complex.': 1984,\n",
       " 'pendulum': 6394,\n",
       " 'sees': 7589,\n",
       " 'revealed': 7231,\n",
       " 'dried': 2778,\n",
       " 'sack': 7408,\n",
       " 'informative.': 4651,\n",
       " 'hey': 4316,\n",
       " 'after,': 217,\n",
       " 'ideas.': 4561,\n",
       " 'bottled.': 1129,\n",
       " 'pitch.': 6536,\n",
       " 'jersey,': 4823,\n",
       " 'ashuman': 546,\n",
       " 'press': 6760,\n",
       " 'believes': 909,\n",
       " 'themselve': 8827,\n",
       " 'lights.': 5157,\n",
       " 'camping': 1401,\n",
       " 'cultures.': 2301,\n",
       " 'maybe': 5477,\n",
       " 'amed': 340,\n",
       " 'follows.': 3601,\n",
       " 'directions': 2599,\n",
       " 'segment': 7590,\n",
       " 'hungry,': 4517,\n",
       " 'area,': 485,\n",
       " 'tradition.': 9076,\n",
       " 'moon.': 5666,\n",
       " 'smores.': 7989,\n",
       " 'hola-hopping': 4389,\n",
       " 'tall': 8703,\n",
       " 'room,': 7347,\n",
       " 'book': 1093,\n",
       " 'success!': 8505,\n",
       " 'minutes': 5602,\n",
       " 'crafting': 2206,\n",
       " 'tickets': 8903,\n",
       " 'ships.': 7702,\n",
       " 'ledg': 5094,\n",
       " 'glor': 3900,\n",
       " 'bean': 821,\n",
       " 'mountainsides': 5708,\n",
       " 'perfectly': 6413,\n",
       " 'flavors.': 3531,\n",
       " 'lose': 5274,\n",
       " 'robot': 7298,\n",
       " 'epic': 3056,\n",
       " 'phenomenal.': 6452,\n",
       " 'exchanging': 3139,\n",
       " 'toys.': 9065,\n",
       " 'brutal,': 1238,\n",
       " 'blonde': 1024,\n",
       " 'there': 8835,\n",
       " 'freshly': 3679,\n",
       " 'riders.': 7252,\n",
       " 'pretentious,': 6765,\n",
       " 'exchanged': 3138,\n",
       " 'entertainer': 3032,\n",
       " 'anticipated': 396,\n",
       " 'with': 9772,\n",
       " 'hairdo': 4148,\n",
       " 'baptism.': 749,\n",
       " 'close-up': 1819,\n",
       " 'symerton': 8659,\n",
       " 'flocks': 3551,\n",
       " 'beside': 928,\n",
       " 'cave,': 1552,\n",
       " 'salty': 7432,\n",
       " 'porch.': 6667,\n",
       " \"i'am\": 4544,\n",
       " 'randomly': 7000,\n",
       " 'slide': 7932,\n",
       " 'admire': 175,\n",
       " 'gear': 3808,\n",
       " 'flight,': 3539,\n",
       " 'yellow.': 9899,\n",
       " 'those': 8871,\n",
       " 'cups.': 2308,\n",
       " 'crowded,': 2273,\n",
       " 'store': 8374,\n",
       " 'vineyards': 9444,\n",
       " 'started': 8285,\n",
       " 'tomorrow': 8984,\n",
       " 'hopefully': 4439,\n",
       " 'blimp.': 1011,\n",
       " 'hahaha.': 4145,\n",
       " 'clea': 1784,\n",
       " 'cards,': 1455,\n",
       " 'wass': 9562,\n",
       " 'hop': 4436,\n",
       " 'ski!': 7886,\n",
       " 'laugh.': 5043,\n",
       " 'listened': 5187,\n",
       " 'breakfast.': 1179,\n",
       " 'awhile.': 656,\n",
       " 'blocked.': 1019,\n",
       " 'uncovered': 9272,\n",
       " 'or': 6105,\n",
       " 'flags.': 3514,\n",
       " 'places,': 6553,\n",
       " 'clearly': 1795,\n",
       " 'floor.': 3556,\n",
       " 'push': 6918,\n",
       " 'adventurers.': 198,\n",
       " 'war': 9537,\n",
       " 'awnings.': 658,\n",
       " 'tries': 9161,\n",
       " 'wall': 9520,\n",
       " 'into': 4738,\n",
       " 'quickly.': 6946,\n",
       " 'weeks,': 9635,\n",
       " 'club': 1843,\n",
       " 'childhood,': 1694,\n",
       " 'advantage.': 193,\n",
       " 'blocking': 1020,\n",
       " 'community': 1951,\n",
       " 'multi-explosive': 5744,\n",
       " 'tour,': 9032,\n",
       " 'fans!': 3317,\n",
       " 'confidence.': 2017,\n",
       " 'release': 7136,\n",
       " 'drive': 2792,\n",
       " 'dances': 2358,\n",
       " 'tank': 8711,\n",
       " 'again!': 232,\n",
       " 'bulls!': 1282,\n",
       " 'awake': 632,\n",
       " 'defense.': 2463,\n",
       " 'pie.': 6505,\n",
       " 'icon': 4557,\n",
       " 'guide': 4113,\n",
       " \"location's\": 5232,\n",
       " '12': 24,\n",
       " 'visits': 9466,\n",
       " 'antiques': 402,\n",
       " 'pointing': 6627,\n",
       " 'particularly': 6315,\n",
       " 'batman': 793,\n",
       " 'cater': 1534,\n",
       " 'winter,': 9755,\n",
       " 'straight': 8393,\n",
       " 'hometown.': 4413,\n",
       " 'colder,': 1890,\n",
       " 'superman.': 8560,\n",
       " 'picture,': 6497,\n",
       " 'explaining': 3204,\n",
       " 'inward,': 4764,\n",
       " 'mention': 5538,\n",
       " 'law': 5056,\n",
       " 'owls.': 6212,\n",
       " 'combine': 1922,\n",
       " 'college': 1899,\n",
       " 'ahead': 249,\n",
       " 'oi,': 6038,\n",
       " 'roll.': 7325,\n",
       " 'yesterday': 9904,\n",
       " 'dramatically': 2752,\n",
       " 'intrigued': 4747,\n",
       " 'them': 8819,\n",
       " 'prettiest': 6767,\n",
       " 'intens': 4703,\n",
       " 'allow': 290,\n",
       " 'tackled': 8675,\n",
       " 'sharing': 7675,\n",
       " 'studies,': 8470,\n",
       " 'cross': 2258,\n",
       " 'stage,': 8248,\n",
       " 'birthday,': 982,\n",
       " 'swimming.': 8639,\n",
       " 'grilling.': 4066,\n",
       " 'mov': 5714,\n",
       " 'feet.': 3383,\n",
       " 'carousel.': 1483,\n",
       " 'finding': 3450,\n",
       " 'produced': 6806,\n",
       " 'organization!': 6117,\n",
       " 'god': 3919,\n",
       " 'yesterday.': 9905,\n",
       " 'inspection': 4681,\n",
       " 'creatures': 2242,\n",
       " 'suspicious': 8610,\n",
       " 'presentations': 6750,\n",
       " 'tail.': 8680,\n",
       " 'outdid': 6154,\n",
       " 'artful': 526,\n",
       " 'carnival.': 1471,\n",
       " 'sparkly': 8125,\n",
       " 'victorious': 9421,\n",
       " 'mainly': 5356,\n",
       " 'swung': 8655,\n",
       " 'anyways.': 416,\n",
       " 'family.': 3307,\n",
       " 'useful': 9350,\n",
       " 'it.': 4785,\n",
       " 'sponsor': 8199,\n",
       " 'order.': 6110,\n",
       " 'white': 9693,\n",
       " 'bet': 935,\n",
       " 'run.': 7389,\n",
       " 'harbor': 4204,\n",
       " 'stunning.': 8484,\n",
       " 'statures.': 8316,\n",
       " 'more,': 5671,\n",
       " 'crazy,': 2227,\n",
       " 'listens': 5190,\n",
       " 'appearance.': 431,\n",
       " 'quartet': 6930,\n",
       " 'skyscrapers': 7912,\n",
       " 'launched,': 5052,\n",
       " 'reunited': 7229,\n",
       " 'serving': 7629,\n",
       " 'baking': 710,\n",
       " 'bikes.': 967,\n",
       " 'weather': 9613,\n",
       " 'exciting,': 3151,\n",
       " 'designs,': 2522,\n",
       " 'red,': 7087,\n",
       " 'consult': 2050,\n",
       " 'mall': 5370,\n",
       " 'enhanced': 2999,\n",
       " 'attractive': 603,\n",
       " 'coworkers': 2192,\n",
       " 'excited.': 3144,\n",
       " 'once.': 6062,\n",
       " 'cuddl': 2292,\n",
       " 'opened.': 6084,\n",
       " 'checking': 1655,\n",
       " 'mom': 5636,\n",
       " 'pavement': 6369,\n",
       " 'job,': 4830,\n",
       " 'dining': 2585,\n",
       " 'speeding': 8157,\n",
       " 'advisors': 206,\n",
       " 'days,': 2408,\n",
       " 'brightens': 1210,\n",
       " 'greek': 4047,\n",
       " 'way': 9597,\n",
       " 'overhead': 6194,\n",
       " 'excited!': 3142,\n",
       " 'nevermind.': 5867,\n",
       " 'ceremonial': 1592,\n",
       " 'returned': 7222,\n",
       " 'anchor.': 358,\n",
       " 'feel.': 3376,\n",
       " 'tarp.': 8718,\n",
       " 'disappoint,': 2610,\n",
       " 'experiences': 3197,\n",
       " 'dreamed': 2761,\n",
       " 'gardening': 3777,\n",
       " 'besides': 929,\n",
       " 'stem.': 8332,\n",
       " 'floored': 3557,\n",
       " 'continously.': 2075,\n",
       " '13': 26,\n",
       " \"tree's\": 9141,\n",
       " 'cathedral': 1537,\n",
       " 'husband': 4529,\n",
       " 'boy.': 1148,\n",
       " 'stretching': 8429,\n",
       " 'colder': 1889,\n",
       " 'barbed': 756,\n",
       " 'round.': 7367,\n",
       " 'fun!': 3733,\n",
       " 'crusing': 2285,\n",
       " 'biggest.': 958,\n",
       " 'tipsy': 8941,\n",
       " 'initially': 4656,\n",
       " 'subdued,': 8495,\n",
       " 'looking': 5259,\n",
       " 'fixings!': 3505,\n",
       " '[male': 79,\n",
       " 'becoming': 852,\n",
       " 'outing,': 6168,\n",
       " 'honor': 4422,\n",
       " 'careful': 1461,\n",
       " 'relative': 7123,\n",
       " 'healthy': 4254,\n",
       " 'beaten': 836,\n",
       " 'calmer': 1374,\n",
       " 'colors.': 1917,\n",
       " 'fortunate': 3646,\n",
       " \"here,''\": 4305,\n",
       " 'passenger': 6335,\n",
       " 'desig': 2516,\n",
       " 'breath.': 1182,\n",
       " 'treat.': 9132,\n",
       " 'proves': 6866,\n",
       " 'giggling.': 3858,\n",
       " 'stands.': 8268,\n",
       " 'improve': 4611,\n",
       " '[male].': 87,\n",
       " '5': 59,\n",
       " 'alright,': 313,\n",
       " 'darkness.': 2385,\n",
       " 'depart': 2495,\n",
       " 'metal.': 5563,\n",
       " 'americans': 344,\n",
       " 'floater,': 3545,\n",
       " 'yum!': 9925,\n",
       " 'tears': 8750,\n",
       " 'bungee': 1289,\n",
       " 'secret': 7565,\n",
       " 'details.': 2542,\n",
       " 'screen': 7532,\n",
       " 'snac': 7990,\n",
       " 'preoccupy': 6736,\n",
       " 'woods': 9809,\n",
       " 'masks': 5451,\n",
       " 'colorful': 1911,\n",
       " 'sails': 7427,\n",
       " 'hikes': 4345,\n",
       " 'crate': 2217,\n",
       " 'audience.': 609,\n",
       " 'camels': 1384,\n",
       " 'appealing.': 428,\n",
       " 'miles,': 5579,\n",
       " 'mela,': 5516,\n",
       " 'horses': 4450,\n",
       " 'shopping.': 7728,\n",
       " 'characters.': 1635,\n",
       " 'mysterious': 5784,\n",
       " 'stopping': 8370,\n",
       " 'thank': 8803,\n",
       " 'photo': 6456,\n",
       " 'crawls': 2221,\n",
       " 'wheel': 9673,\n",
       " 'enthusiastically.': 3041,\n",
       " 'newly': 5872,\n",
       " 'airplanes.': 264,\n",
       " 'reservoir': 7187,\n",
       " 'rainy': 6983,\n",
       " 'detonation': 2546,\n",
       " 'okay': 6043,\n",
       " 'toss,': 9014,\n",
       " 'possible': 6686,\n",
       " 'theater': 8815,\n",
       " 'participant': 6306,\n",
       " 'closing': 1828,\n",
       " 'will': 9723,\n",
       " 'iconic': 4558,\n",
       " 'beach': 811,\n",
       " 'puddles': 6880,\n",
       " 'beer!': 862,\n",
       " 'blanket': 996,\n",
       " 'streaks': 8409,\n",
       " 'over!': 6182,\n",
       " 'black.': 993,\n",
       " 'snows,': 8027,\n",
       " 'arrived!': 516,\n",
       " 'mother.': 5687,\n",
       " 'touchdown': 9024,\n",
       " 'tweaked': 9242,\n",
       " 'celebrating': 1567,\n",
       " 'brain.': 1160,\n",
       " 'knots,': 4957,\n",
       " 'floating': 3547,\n",
       " 'return': 7220,\n",
       " 'uninterested': 9300,\n",
       " 'nightfall': 5900,\n",
       " 'contact': 2054,\n",
       " 'twins,': 9245,\n",
       " 'stop,': 8366,\n",
       " 'smiley': 7972,\n",
       " 'go-karts.': 3913,\n",
       " 'feed': 3373,\n",
       " 'untied': 9319,\n",
       " 'rewards.': 7237,\n",
       " 'baby.': 671,\n",
       " 'tips.': 8940,\n",
       " 'vows.': 9487,\n",
       " 'roads': 7288,\n",
       " 'capitol.': 1436,\n",
       " 'baseball.': 772,\n",
       " 'having.': 4232,\n",
       " 'expensive,': 3189,\n",
       " 'earth': 2873,\n",
       " 'coming,': 1941,\n",
       " 'gloomy': 3898,\n",
       " 'along': 302,\n",
       " 'grateful': 4031,\n",
       " 'snacks': 7994,\n",
       " \"we've\": 9607,\n",
       " 'markings.': 5425,\n",
       " 'afar.': 209,\n",
       " 'beast': 834,\n",
       " 'cliffs.': 1804,\n",
       " 'low': 5304,\n",
       " 'forget': 3633,\n",
       " 'carousals.': 1480,\n",
       " 'interesting': 4721,\n",
       " 'band.': 732,\n",
       " 'sudden': 8513,\n",
       " 'tables.': 8673,\n",
       " 'money': 5647,\n",
       " 'giver': 3880,\n",
       " 'adventure!': 195,\n",
       " 'wa': 9489,\n",
       " 'questions': 6940,\n",
       " 'groo': 4075,\n",
       " 'treasure.': 9129,\n",
       " 'ladies': 4973,\n",
       " 'had': 4139,\n",
       " 'contained.': 2057,\n",
       " 'decision.': 2431,\n",
       " 'posture.': 6695,\n",
       " 'jumbo-tron': 4873,\n",
       " 'people.': 6401,\n",
       " 'shoot.': 7720,\n",
       " 'observatory': 5981,\n",
       " 'corn,': 2129,\n",
       " 'cars!': 1493,\n",
       " 'downtown.': 2738,\n",
       " 'person!': 6439,\n",
       " 'chocolates': 1713,\n",
       " 'that,': 8812,\n",
       " 'houses': 4478,\n",
       " 'scratched': 7528,\n",
       " 'camped': 1397,\n",
       " 'bulbs.': 1279,\n",
       " '[female],': 77,\n",
       " 'said.': 7421,\n",
       " 'memory.': 5533,\n",
       " 'us,': 9344,\n",
       " 'coats': 1869,\n",
       " 'architectur': 473,\n",
       " 'prizes.': 6796,\n",
       " 'begging': 878,\n",
       " 'expo.': 3233,\n",
       " 'accomplishment.': 121,\n",
       " 'rapidly': 7005,\n",
       " 'funny,': 3743,\n",
       " 'immediately': 4590,\n",
       " 'abroad': 105,\n",
       " 'thunder': 8899,\n",
       " 'containers.': 2060,\n",
       " 'rowdy': 7374,\n",
       " 'carved.': 1503,\n",
       " 'raised': 6986,\n",
       " 'swag': 8614,\n",
       " 'lifetime': 5138,\n",
       " 'apparently': 425,\n",
       " 'hoops.': 4434,\n",
       " 'fish': 3488,\n",
       " 'given.': 3879,\n",
       " 'involved': 4762,\n",
       " 'hussle': 4533,\n",
       " 'alert': 274,\n",
       " 'prepared': 6738,\n",
       " 'imitation': 4589,\n",
       " 'melted': 5519,\n",
       " 'gig.': 3854,\n",
       " 'pleasantly': 6616,\n",
       " 'burned.': 1298,\n",
       " 'selection,': 7593,\n",
       " 'shred!': 7772,\n",
       " 'longer!': 5249,\n",
       " 'enjoy!': 3002,\n",
       " 'booths': 1108,\n",
       " 'budd': 1248,\n",
       " 'largest': 5020,\n",
       " 'beautifully': 843,\n",
       " 'greenery': 4050,\n",
       " 'scratch': 7527,\n",
       " 'top,': 9006,\n",
       " 'blast,': 1002,\n",
       " 'enthusiasts': 3042,\n",
       " 'patty': 6364,\n",
       " 'dispenser': 2636,\n",
       " 'opinion.': 6092,\n",
       " 'bridesmade': 1197,\n",
       " 'tooth': 9002,\n",
       " 'horns': 4444,\n",
       " 'receive': 7050,\n",
       " 'go!': 3909,\n",
       " 'unformed': 9291,\n",
       " 'time!': 8921,\n",
       " 'keyboardist': 4906,\n",
       " 'cheesing': 1671,\n",
       " 'actually.': 158,\n",
       " 'eventful.': 3092,\n",
       " 'lake.': 4984,\n",
       " 'include': 4619,\n",
       " 'unfortunately': 9293,\n",
       " 'hovering': 4481,\n",
       " 'pedestrians': 6384,\n",
       " 'shuddering': 7778,\n",
       " 'nap!': 5799,\n",
       " 'characters': 1632,\n",
       " 'much.': 5738,\n",
       " 'contemplate': 2061,\n",
       " 'playtime,': 6612,\n",
       " 'coordinated': 2120,\n",
       " 'simple': 7832,\n",
       " 'forever.': 3632,\n",
       " 'relics': 7140,\n",
       " 'pours': 6707,\n",
       " 'spy!': 8230,\n",
       " \"aren't\": 489,\n",
       " 'nature': 5813,\n",
       " 'rack': 6965,\n",
       " 'nervous': 5862,\n",
       " 'bad!': 693,\n",
       " 'pretty': 6769,\n",
       " 'anywhere': 417,\n",
       " 'nephew.': 5856,\n",
       " 'stand': 8260,\n",
       " 'salon': 7431,\n",
       " 'looks': 5263,\n",
       " '--': 9,\n",
       " 'faux': 3352,\n",
       " 'construction,': 2048,\n",
       " 'excellence.': 3128,\n",
       " 'i': 4543,\n",
       " 'tre': 9124,\n",
       " 'stew': 8339,\n",
       " 'boulders.': 1133,\n",
       " 'acclimated.': 116,\n",
       " 'avenue,': 623,\n",
       " 'tiring': 8949,\n",
       " 'flag': 3511,\n",
       " 'bal': 712,\n",
       " 'legendary': 5103,\n",
       " 'easy': 2882,\n",
       " 'mulling': 5742,\n",
       " 'backyard.': 690,\n",
       " 'saw.': 7466,\n",
       " 'everyon': 3107,\n",
       " 'inlaws': 4659,\n",
       " 'bright': 1205,\n",
       " 'concrete': 2005,\n",
       " 'thanked': 8804,\n",
       " 'woods,': 9811,\n",
       " 'friendships': 3699,\n",
       " 'sand.': 7442,\n",
       " 'focus.': 3589,\n",
       " 'plentiful': 6618,\n",
       " 'fire': 3464,\n",
       " 'painted': 6244,\n",
       " 'rally': 6992,\n",
       " 'mood.': 5665,\n",
       " 'staged': 8250,\n",
       " 'giraffe.': 3863,\n",
       " 'prisons': 6791,\n",
       " 'phones,': 6454,\n",
       " 'globe': 3897,\n",
       " 'allowed': 291,\n",
       " 'content.': 2067,\n",
       " 'greenhouse': 4052,\n",
       " 'indoor': 4638,\n",
       " 'zones': 9932,\n",
       " 'cave.': 1553,\n",
       " 'prom,': 6828,\n",
       " 'outskirts': 6178,\n",
       " 'sky.': 7908,\n",
       " 'farewells.': 3326,\n",
       " 'hone': 4414,\n",
       " 'bearer': 830,\n",
       " '12.': 25,\n",
       " 'wrong': 9869,\n",
       " 'hopes': 4440,\n",
       " 'itself': 4790,\n",
       " 'underwater': 9283,\n",
       " 'hauntingly': 4226,\n",
       " 'wel': 9643,\n",
       " 'lives!': 5206,\n",
       " 'everywhere': 3116,\n",
       " 'sturdy': 8487,\n",
       " 'city!': 1763,\n",
       " 'end!': 2977,\n",
       " 'sequence': 7616,\n",
       " 'rooting': 7353,\n",
       " 'plane': 6561,\n",
       " 'overlapped.': 6198,\n",
       " 'underpass.': 9278,\n",
       " 'smokes': 7986,\n",
       " 'surrounded': 8596,\n",
       " 'found.': 3651,\n",
       " 'brighte': 1208,\n",
       " 'lifting': 5143,\n",
       " 'breath': 1181,\n",
       " 'doorway!': 2720,\n",
       " 'bridge': 1199,\n",
       " 'remembering': 7150,\n",
       " \"40's.\": 52,\n",
       " \"boss's\": 1124,\n",
       " 'drug': 2811,\n",
       " 'pioneer': 6528,\n",
       " 'ride!': 7246,\n",
       " 'puffballs.': 6882,\n",
       " 'sops.': 8086,\n",
       " 'sporadically': 8203,\n",
       " 'below': 914,\n",
       " 'serves': 7625,\n",
       " 'spiked': 8171,\n",
       " 'develop': 2548,\n",
       " \"isn't\": 4774,\n",
       " 'surprise': 8583,\n",
       " 'loaded': 5215,\n",
       " 'excursion.': 3155,\n",
       " 'hypnotized': 4541,\n",
       " 'invigorated': 4756,\n",
       " 'coil': 1880,\n",
       " 'excited,': 3143,\n",
       " 'himself,': 4366,\n",
       " 'wild.': 9718,\n",
       " 'boy': 1146,\n",
       " 'train.': 9091,\n",
       " 'crowd.': 2271,\n",
       " 'enter.': 3024,\n",
       " 'dinner!': 2588,\n",
       " 'hard.': 4209,\n",
       " 'banana': 728,\n",
       " 'blankets': 998,\n",
       " 'mowing': 5728,\n",
       " 'streets': 8423,\n",
       " 'mu': 5735,\n",
       " 'fateful': 3344,\n",
       " 'dwellings,': 2846,\n",
       " 'furiously.': 3745,\n",
       " 'mans': 5388,\n",
       " 'die': 2566,\n",
       " 'skateboarders': 7882,\n",
       " 'guy!': 4130,\n",
       " \"mela''\": 5515,\n",
       " 'thril': 8887,\n",
       " 'flew': 3533,\n",
       " 'wonderland.': 9803,\n",
       " 'stopped.': 8369,\n",
       " 'board,': 1055,\n",
       " 'intricate.': 4745,\n",
       " 'overexposed': 6193,\n",
       " 'appreciative': 449,\n",
       " 'successful': 8508,\n",
       " 'loud!': 5286,\n",
       " 'safely.': 7415,\n",
       " 'sleek': 7921,\n",
       " 'ourselves,': 6147,\n",
       " 'matt.': 5471,\n",
       " 'reality.': 7036,\n",
       " 'minute': 5600,\n",
       " 'ornate.': 6129,\n",
       " 'carried': 1487,\n",
       " 'hangout.': 4180,\n",
       " 'partners': 6318,\n",
       " 'scooby': 7512,\n",
       " 'grows': 4098,\n",
       " 'traveling': 9117,\n",
       " 'rush!': 7396,\n",
       " \"where's\": 9683,\n",
       " 'bath.': 788,\n",
       " 'carnival,': 1470,\n",
       " 'remained': 7145,\n",
       " 'bright.': 1207,\n",
       " 'brighter': 1211,\n",
       " 'her,': 4297,\n",
       " 'buried': 1295,\n",
       " 'sweethearts': 8628,\n",
       " 'prices.': 6778,\n",
       " 'gator.': 3798,\n",
       " 'extreme.': 3247,\n",
       " 'close!': 1818,\n",
       " 'countrie': 2157,\n",
       " 'comfy': 1937,\n",
       " 'bursts': 1304,\n",
       " 'googly': 3958,\n",
       " 'suggestive': 8522,\n",
       " 'these.': 8842,\n",
       " 'banner': 743,\n",
       " 'player': 6592,\n",
       " 'growing.': 4095,\n",
       " 'oaisis.': 5975,\n",
       " 'balloon': 720,\n",
       " 'choice.': 1715,\n",
       " 'sick,': 7789,\n",
       " 'tricycle.': 9159,\n",
       " 'mercury': 5545,\n",
       " 'missed.': 5611,\n",
       " 'anniversary,': 382,\n",
       " 'around': 501,\n",
       " 'funny': 3741,\n",
       " 'sno': 8008,\n",
       " 'lodge.': 5237,\n",
       " \"weren't\": 9663,\n",
       " 'forget,': 3634,\n",
       " 'rallied': 6990,\n",
       " 'pumpking': 6895,\n",
       " 'patrons': 6360,\n",
       " 'discoveries,': 2620,\n",
       " '7th': 64,\n",
       " 'amazed.': 329,\n",
       " 'deserved': 2514,\n",
       " 'tricks.': 9156,\n",
       " 'workers': 9827,\n",
       " 'sushi': 8604,\n",
       " 'dhali': 2558,\n",
       " 'sons': 8079,\n",
       " 'started!': 8286,\n",
       " 'sister.': 7854,\n",
       " 'crown': 2278,\n",
       " 'rocket,': 7306,\n",
       " 'cards.': 1456,\n",
       " \"that't\": 8811,\n",
       " 'everyone': 3108,\n",
       " 'banners': 746,\n",
       " 'afterward,': 226,\n",
       " 'gods': 3921,\n",
       " 'travel.': 9115,\n",
       " 'says,': 7474,\n",
       " 'which': 9685,\n",
       " 'victory.': 9424,\n",
       " \"what's\": 9672,\n",
       " 'club!': 1844,\n",
       " 'hamburger.': 4163,\n",
       " 'roads.': 7289,\n",
       " 'sightseeing,': 7810,\n",
       " 'old': 6046,\n",
       " 'plack': 6556,\n",
       " 'agent': 238,\n",
       " 'between': 938,\n",
       " 'interracial': 4730,\n",
       " 'wear': 9609,\n",
       " 'scary,': 7486,\n",
       " 'sat.': 7452,\n",
       " 'marathon': 5399,\n",
       " 'touching': 9027,\n",
       " 'instead': 4690,\n",
       " 'better.': 937,\n",
       " 'received.': 7053,\n",
       " 'surrounding': 8598,\n",
       " 'land,': 4991,\n",
       " 'brightness': 1213,\n",
       " 'trick.': 9152,\n",
       " 'want.': 9532,\n",
       " 'sleep.': 7923,\n",
       " 'breeze': 1189,\n",
       " 'behind!': 897,\n",
       " 'swing.': 8643,\n",
       " 'succeed,': 8502,\n",
       " 'farm': 3327,\n",
       " 'pitched': 6537,\n",
       " 'cage.': 1355,\n",
       " 'snowboards.': 8018,\n",
       " 'cars,': 1494,\n",
       " 'embarrassing': 2948,\n",
       " 'crafts,': 2208,\n",
       " 'rile': 7266,\n",
       " 'winter!': 9753,\n",
       " 'reconnect': 7073,\n",
       " 'cartoonist': 1497,\n",
       " 'epcot': 3055,\n",
       " 'songs': 8077,\n",
       " 'concert.': 1999,\n",
       " 'contain': 2056,\n",
       " 'black': 992,\n",
       " 'barbecue': 753,\n",
       " 'smit': 7975,\n",
       " 'bra.': 1156,\n",
       " 'beliefs.': 906,\n",
       " 'lives,': 5207,\n",
       " 'tempted': 8775,\n",
       " 'story!': 8390,\n",
       " 'dumpster': 2832,\n",
       " 'confusion,': 2023,\n",
       " 'came': 1379,\n",
       " 'boat': 1064,\n",
       " 'halloween!': 4157,\n",
       " 'easier': 2877,\n",
       " 'ones.': 6072,\n",
       " 'things,': 8855,\n",
       " 'sushi!': 8605,\n",
       " 'euphoric': 3080,\n",
       " 'aura': 613,\n",
       " 'wiring.': 9762,\n",
       " 'insects.': 4672,\n",
       " 'cooked,': 2104,\n",
       " 'drank.': 2754,\n",
       " 'complex': 1982,\n",
       " 'camp,': 1394,\n",
       " 'memory': 5532,\n",
       " 'uhm,': 9260,\n",
       " 'violence.': 9449,\n",
       " 'common': 1947,\n",
       " 'cheeseburgers': 1669,\n",
       " 'animals.we': 380,\n",
       " 'whoa!': 9702,\n",
       " 'her': 4295,\n",
       " 'designs.it': 2524,\n",
       " 'introduced': 4750,\n",
       " 'innocently': 4664,\n",
       " 'hike.': 4342,\n",
       " 'flaming': 3520,\n",
       " 'chickens': 1689,\n",
       " 'well.': 9655,\n",
       " 'says': 7473,\n",
       " 'battled': 800,\n",
       " 'reception.': 7062,\n",
       " 'custard.': 2319,\n",
       " 'ducks': 2827,\n",
       " 'festivities': 3402,\n",
       " 'coworker': 2190,\n",
       " 'app': 424,\n",
       " 'tough.': 9029,\n",
       " 'gathered,': 3792,\n",
       " 'troubles,': 9186,\n",
       " 'oh': 6033,\n",
       " 'honeymoo': 4418,\n",
       " 'hoop': 4429,\n",
       " 'marshal': 5435,\n",
       " 'heard': 4257,\n",
       " 'flow': 3562,\n",
       " 'nerve': 5860,\n",
       " 'inning': 4662,\n",
       " 'elves': 2943,\n",
       " 'stream': 8410,\n",
       " 'happ': 4182,\n",
       " 'land.': 4992,\n",
       " 'goofed': 3954,\n",
       " 'gambling': 3757,\n",
       " 'feat': 3368,\n",
       " 'vip': 9451,\n",
       " 'plate': 6579,\n",
       " 'overnight': 6203,\n",
       " 'barges': 764,\n",
       " 'enough': 3015,\n",
       " 'all.': 288,\n",
       " 'drink.': 2783,\n",
       " 'grandmother.': 4011,\n",
       " 'cleaned.': 1788,\n",
       " 'exterior': 3245,\n",
       " \"'\": 4,\n",
       " 'opponents.': 6094,\n",
       " 'residents': 7188,\n",
       " 'perhaps': 6428,\n",
       " 'yeah': 9881,\n",
       " 'spreading': 8221,\n",
       " 'promising': 6832,\n",
       " 'ball': 717,\n",
       " 'link!': 5179,\n",
       " 'raining.': 6982,\n",
       " 'where': 9682,\n",
       " 'beginning.': 886,\n",
       " 'yard': 9874,\n",
       " 'bear': 823,\n",
       " 'passed,': 6333,\n",
       " 'planet': 6566,\n",
       " 'construction': 2047,\n",
       " 'pun': 6899,\n",
       " 'amazing': 333,\n",
       " 'examine.': 3124,\n",
       " 'stepping': 8336,\n",
       " 'plate.': 6581,\n",
       " 'him': 4359,\n",
       " 'competition,': 1968,\n",
       " 'type': 9254,\n",
       " 'clown.': 1839,\n",
       " ...}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_token_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean_df_all.description_text, clean_df_all.story_text\n",
    "for i, (input_text, target_text) in enumerate(zip(clean_df_all.description_text, clean_df_all.story_text)):\n",
    "    #print (\"input_text: {0}, target_text: {1}\".format(input_text, target_text))\n",
    "    # encoder\n",
    "    for t, word in enumerate(input_text.split()):\n",
    "        encoder_input_data[i, t] = input_token_index[word]\n",
    "    # decoder\n",
    "    try:\n",
    "        for t, word in enumerate(target_text.split()):\n",
    "            # decoder_target_data is ahead of decoder_input_data by one timestep\n",
    "            decoder_input_data[i, t] = target_token_index[word]  # word_index를 decoder_input_data에 넣음.\n",
    "            #print (\"check\")\n",
    "            if t > 0:\n",
    "                # decoder_target_data will be ahead by one timestep\n",
    "                # and will not include the start character.\n",
    "                decoder_target_data[i, t - 1, target_token_index[word]] = 1.\n",
    "    except Exception as e:\n",
    "        print (e)\n",
    "        print (\"t number: \", t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 82)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[  72.,  361., 4789., ...,    0.,    0.,    0.],\n",
       "       [  72.,  361., 4789., ...,    0.,    0.,    0.],\n",
       "       [  72.,  361., 4789., ...,    0.,    0.,    0.],\n",
       "       ...,\n",
       "       [  72., 8813., 4919., ...,    0.,    0.,    0.],\n",
       "       [  72., 8843., 5295., ...,    0.,    0.,    0.],\n",
       "       [  72., 9603., 4139., ...,    0.,    0.,    0.]], dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (decoder_input_data.shape)\n",
    "decoder_input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 82, 9940)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]],\n",
       "\n",
       "       [[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]]], dtype=float32)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(decoder_target_data.shape)\n",
    "decoder_target_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build keras encoder-decoder model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\Anaconda3\\envs\\gpu\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "\n",
    "\n",
    "class AttentionL(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(AttentionL, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
    "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        if mask is not None:\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0],  self.features_dim\n",
    "\n",
    "    def get_config(self):\n",
    "        config={'step_dim':self.step_dim}\n",
    "        base_config = super(AttentionL, self).get_config()\n",
    "        return dict(list(base_config.items()) + list(config.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input, LSTM, Embedding, Dense, Concatenate, Bidirectional, RepeatVector, concatenate, TimeDistributed, dot\n",
    "from keras.models import Model\n",
    "from keras.utils import plot_model\n",
    "from keras import Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 150\n",
    "vocab_size_input = num_encoder_tokens\n",
    "vocab_size_output = num_decoder_tokens\n",
    "MAX_LENGTH_INPUT = max_desc_length\n",
    "MAX_LENGTH_OUTPUT = max_story_length\n",
    "units = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'encoder_inputs: '"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(None), Dimension(None)])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor 'lstm_2/transpose_1:0' shape=(?, ?, 200) dtype=float32>, <tf.Tensor 'lstm_2/while/Exit_2:0' shape=(?, 200) dtype=float32>, <tf.Tensor 'lstm_2/while/Exit_3:0' shape=(?, 200) dtype=float32>]\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 150)    1179450     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) [(None, 200), (None, 200800      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, None, 150)    1491000     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 200)          0           bidirectional_1[0][1]            \n",
      "                                                                 bidirectional_1[0][3]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 200)          0           bidirectional_1[0][2]            \n",
      "                                                                 bidirectional_1[0][4]            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   [(None, None, 200),  280800      embedding_2[0][0]                \n",
      "                                                                 concatenate_1[0][0]              \n",
      "                                                                 concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, None, 9940)   1997940     lstm_2[1][0]                     \n",
      "==================================================================================================\n",
      "Total params: 5,149,990\n",
      "Trainable params: 5,149,990\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 11-02  decoder 1 layer\n",
    "encoder_inputs = Input(shape=(None,))\n",
    "\n",
    "display('encoder_inputs: ', encoder_inputs.shape)\n",
    "\n",
    "en_x=  Embedding(num_encoder_tokens, EMBEDDING_DIM)(encoder_inputs)\n",
    "\n",
    "encoder = Bidirectional(LSTM(units, return_state=True,\n",
    "                             dropout = 0.5, recurrent_dropout = 0.5))\n",
    "\n",
    "encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder(en_x)\n",
    "\n",
    "state_h = Concatenate()([forward_h, backward_h])\n",
    "state_c = Concatenate()([forward_c, backward_c])\n",
    "\n",
    "# We discard `encoder_outputs` and only keep the states.\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "### decoder\n",
    "\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "\n",
    "dex=  Embedding(num_decoder_tokens, EMBEDDING_DIM)\n",
    "\n",
    "final_dex= dex(decoder_inputs)\n",
    "\n",
    "decoder_lstm = LSTM(units * 2, return_sequences=True, return_state=True,\n",
    "                    dropout = 0.5, recurrent_dropout = 0.5)\n",
    "\n",
    "print (decoder_lstm(final_dex, initial_state = encoder_states))\n",
    "decoder_outputs, _, _ = decoder_lstm(final_dex, initial_state = encoder_states)\n",
    "\n",
    "decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
    "\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "\n",
    "model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['acc'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "30000/30000 [==============================] - 423s 14ms/step - loss: 0.9122 - acc: 0.0163\n",
      "Epoch 2/50\n",
      "30000/30000 [==============================] - 455s 15ms/step - loss: 0.8310 - acc: 0.0216\n",
      "Epoch 3/50\n",
      "30000/30000 [==============================] - 409s 14ms/step - loss: 0.7771 - acc: 0.0259\n",
      "Epoch 4/50\n",
      "30000/30000 [==============================] - 403s 13ms/step - loss: 0.7445 - acc: 0.0284\n",
      "Epoch 5/50\n",
      "30000/30000 [==============================] - 411s 14ms/step - loss: 0.7179 - acc: 0.0306\n",
      "Epoch 6/50\n",
      "30000/30000 [==============================] - 406s 14ms/step - loss: 0.6934 - acc: 0.0326\n",
      "Epoch 7/50\n",
      "30000/30000 [==============================] - 426s 14ms/step - loss: 0.6716 - acc: 0.0338\n",
      "Epoch 8/50\n",
      "30000/30000 [==============================] - 407s 14ms/step - loss: 0.6520 - acc: 0.0348\n",
      "Epoch 9/50\n",
      "30000/30000 [==============================] - 415s 14ms/step - loss: 0.6342 - acc: 0.0358\n",
      "Epoch 10/50\n",
      "30000/30000 [==============================] - 424s 14ms/step - loss: 0.6176 - acc: 0.0367\n",
      "Epoch 11/50\n",
      "30000/30000 [==============================] - 422s 14ms/step - loss: 0.6019 - acc: 0.0374\n",
      "Epoch 12/50\n",
      "30000/30000 [==============================] - 410s 14ms/step - loss: 0.5874 - acc: 0.0380\n",
      "Epoch 13/50\n",
      "30000/30000 [==============================] - 397s 13ms/step - loss: 0.5740 - acc: 0.0387\n",
      "Epoch 14/50\n",
      "30000/30000 [==============================] - 421s 14ms/step - loss: 0.5609 - acc: 0.0393\n",
      "Epoch 15/50\n",
      "30000/30000 [==============================] - 387s 13ms/step - loss: 0.5485 - acc: 0.0399\n",
      "Epoch 16/50\n",
      "30000/30000 [==============================] - 401s 13ms/step - loss: 0.5367 - acc: 0.0408\n",
      "Epoch 17/50\n",
      "30000/30000 [==============================] - 420s 14ms/step - loss: 0.5256 - acc: 0.0415\n",
      "Epoch 18/50\n",
      "30000/30000 [==============================] - 423s 14ms/step - loss: 0.5145 - acc: 0.0424\n",
      "Epoch 19/50\n",
      "30000/30000 [==============================] - 399s 13ms/step - loss: 0.5043 - acc: 0.0433\n",
      "Epoch 20/50\n",
      "30000/30000 [==============================] - 420s 14ms/step - loss: 0.4945 - acc: 0.0442\n",
      "Epoch 21/50\n",
      "30000/30000 [==============================] - 419s 14ms/step - loss: 0.4849 - acc: 0.0451\n",
      "Epoch 22/50\n",
      "30000/30000 [==============================] - 413s 14ms/step - loss: 0.4761 - acc: 0.0462\n",
      "Epoch 23/50\n",
      "30000/30000 [==============================] - 418s 14ms/step - loss: 0.4677 - acc: 0.0471\n",
      "Epoch 24/50\n",
      "30000/30000 [==============================] - 423s 14ms/step - loss: 0.4596 - acc: 0.0479\n",
      "Epoch 25/50\n",
      "30000/30000 [==============================] - 414s 14ms/step - loss: 0.4520 - acc: 0.0500\n",
      "Epoch 26/50\n",
      "30000/30000 [==============================] - 418s 14ms/step - loss: 0.4452 - acc: 0.0507\n",
      "Epoch 27/50\n",
      "30000/30000 [==============================] - 387s 13ms/step - loss: 0.4383 - acc: 0.0527\n",
      "Epoch 28/50\n",
      "30000/30000 [==============================] - 424s 14ms/step - loss: 0.4317 - acc: 0.0539\n",
      "Epoch 29/50\n",
      "30000/30000 [==============================] - 404s 13ms/step - loss: 0.4255 - acc: 0.0543\n",
      "Epoch 30/50\n",
      "30000/30000 [==============================] - 410s 14ms/step - loss: 0.4196 - acc: 0.0566\n",
      "Epoch 31/50\n",
      "30000/30000 [==============================] - 410s 14ms/step - loss: 0.4140 - acc: 0.0654\n",
      "Epoch 32/50\n",
      "30000/30000 [==============================] - 424s 14ms/step - loss: 0.4085 - acc: 0.0590\n",
      "Epoch 33/50\n",
      "30000/30000 [==============================] - 418s 14ms/step - loss: 0.4034 - acc: 0.0604\n",
      "Epoch 34/50\n",
      "30000/30000 [==============================] - 421s 14ms/step - loss: 0.3984 - acc: 0.0609\n",
      "Epoch 35/50\n",
      "30000/30000 [==============================] - 410s 14ms/step - loss: 0.3936 - acc: 0.0647\n",
      "Epoch 36/50\n",
      "30000/30000 [==============================] - 404s 13ms/step - loss: 0.3891 - acc: 0.0614\n",
      "Epoch 37/50\n",
      "30000/30000 [==============================] - 426s 14ms/step - loss: 0.3846 - acc: 0.0636\n",
      "Epoch 38/50\n",
      "30000/30000 [==============================] - 417s 14ms/step - loss: 0.3803 - acc: 0.0703\n",
      "Epoch 39/50\n",
      "30000/30000 [==============================] - 393s 13ms/step - loss: 0.3762 - acc: 0.0616\n",
      "Epoch 40/50\n",
      "30000/30000 [==============================] - 423s 14ms/step - loss: 0.3723 - acc: 0.0683\n",
      "Epoch 41/50\n",
      "30000/30000 [==============================] - 433s 14ms/step - loss: 0.3684 - acc: 0.0694\n",
      "Epoch 42/50\n",
      "30000/30000 [==============================] - 397s 13ms/step - loss: 0.3645 - acc: 0.0712\n",
      "Epoch 43/50\n",
      "30000/30000 [==============================] - 410s 14ms/step - loss: 0.3612 - acc: 0.0655\n",
      "Epoch 44/50\n",
      "30000/30000 [==============================] - 426s 14ms/step - loss: 0.3576 - acc: 0.0717\n",
      "Epoch 45/50\n",
      "30000/30000 [==============================] - 397s 13ms/step - loss: 0.3543 - acc: 0.0651\n",
      "Epoch 46/50\n",
      "30000/30000 [==============================] - 405s 14ms/step - loss: 0.3506 - acc: 0.0661\n",
      "Epoch 47/50\n",
      "30000/30000 [==============================] - 393s 13ms/step - loss: 0.3477 - acc: 0.0733\n",
      "Epoch 48/50\n",
      "30000/30000 [==============================] - 414s 14ms/step - loss: 0.3445 - acc: 0.0708\n",
      "Epoch 49/50\n",
      "30000/30000 [==============================] - 414s 14ms/step - loss: 0.3415 - acc: 0.0729\n",
      "Epoch 50/50\n",
      "30000/30000 [==============================] - 413s 14ms/step - loss: 0.3387 - acc: 0.0702\n"
     ]
    }
   ],
   "source": [
    "import keras.backend.tensorflow_backend as K\n",
    "with K.tf.device('/gpu:0'):\n",
    "    model.fit([encoder_input_data, decoder_input_data], decoder_target_data,\n",
    "              batch_size = 128,\n",
    "              epochs = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\Anaconda3\\envs\\gpu\\lib\\site-packages\\keras\\engine\\network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'concatenate_1/concat:0' shape=(?, 200) dtype=float32>, <tf.Tensor 'concatenate_2/concat:0' shape=(?, 200) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n"
     ]
    }
   ],
   "source": [
    "# from keras.models import load_model\n",
    "# model.save(\"1124_bilstm_emb150_model22.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create sampling model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 150)    1179450     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) [(None, 200), (None, 200800      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 200)          0           bidirectional_1[0][1]            \n",
      "                                                                 bidirectional_1[0][3]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 200)          0           bidirectional_1[0][2]            \n",
      "                                                                 bidirectional_1[0][4]            \n",
      "==================================================================================================\n",
      "Total params: 1,380,250\n",
      "Trainable params: 1,380,250\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoder_model = Model(encoder_inputs, encoder_states)\n",
    "encoder_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"embedding_2_1/embedding_lookup:0\", shape=(?, ?, 150), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "decoder_state_input_h = Input(shape=(units * 2,))  # encoder를 bilstm으로 학습했기 때문에 shape이 50이 아니고 100이다.\n",
    "decoder_state_input_c = Input(shape=(units * 2,))\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "\n",
    "final_dex2= dex(decoder_inputs)\n",
    "print (final_dex2)\n",
    "\n",
    "decoder_outputs2, state_h2, state_c2 = decoder_lstm(final_dex2, initial_state=decoder_states_inputs)\n",
    "decoder_states2 = [state_h2, state_c2]\n",
    "decoder_outputs2 = decoder_dense(decoder_outputs2)\n",
    "decoder_model = Model(\n",
    "    [decoder_inputs] + decoder_states_inputs,\n",
    "    [decoder_outputs2] + decoder_states2)\n",
    "\n",
    "# Reverse-lookup token index to decode sequences back to\n",
    "# something readable.\n",
    "reverse_input_char_index = dict(\n",
    "    (i, char) for char, i in input_token_index.items())\n",
    "reverse_target_char_index = dict(\n",
    "    (i, char) for char, i in target_token_index.items())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\Anaconda3\\envs\\gpu\\lib\\site-packages\\keras\\engine\\network.py:877: UserWarning: Layer lstm_2 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'input_3:0' shape=(?, 200) dtype=float32>, <tf.Tensor 'input_4:0' shape=(?, 200) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n"
     ]
    }
   ],
   "source": [
    "# # save\n",
    "# with open('bilstm150_encoder_model_1124.json', 'w', encoding='utf8') as f:\n",
    "#     f.write(encoder_model.to_json())\n",
    "# encoder_model.save_weights('bilstm150_encoder_model_1124.h5')\n",
    "\n",
    "# with open('bilstm150_decoder_model_1124.json', 'w', encoding='utf8') as f:\n",
    "#     f.write(decoder_model.to_json())\n",
    "# decoder_model.save_weights('bilstm150_decoder_model_1124.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model load\n",
    "from keras.models import model_from_json\n",
    "def load_model(model_filename, model_weights_filename):\n",
    "    with open(model_filename, 'r', encoding='utf8') as f:\n",
    "        model = model_from_json(f.read())\n",
    "    model.load_weights(model_weights_filename)\n",
    "    return model\n",
    "\n",
    "encoder = load_model('test_bilstm150_encoder_model.json', 'test_bilstm150_encoder_model.h5')\n",
    "decoder = load_model('test_bilstm150_decoder_model.json', 'test_bilstm150_decoder_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: '#',\n",
       " 1: '&',\n",
       " 2: \"'\",\n",
       " 3: '(',\n",
       " 4: ')',\n",
       " 5: '..',\n",
       " 6: '1',\n",
       " 7: '10',\n",
       " 8: '101',\n",
       " 9: '10:05',\n",
       " 10: '12',\n",
       " 11: '14,',\n",
       " 12: '1942',\n",
       " 13: \"1950's\",\n",
       " 14: '1958',\n",
       " 15: '1998.',\n",
       " 16: '2',\n",
       " 17: '200',\n",
       " 18: '2003.',\n",
       " 19: '2007-2008',\n",
       " 20: '2010',\n",
       " 21: '2012',\n",
       " 22: '2012.',\n",
       " 23: '20rh',\n",
       " 24: '20th',\n",
       " 25: '24',\n",
       " 26: '25',\n",
       " 27: '3',\n",
       " 28: '3-legged',\n",
       " 29: '34',\n",
       " 30: '4',\n",
       " 31: '4-lane',\n",
       " 32: '429',\n",
       " 33: '429-8044,',\n",
       " 34: '4th',\n",
       " 35: '4th.',\n",
       " 36: '5',\n",
       " 37: '50',\n",
       " 38: '51',\n",
       " 39: '5k',\n",
       " 40: '6',\n",
       " 41: '6,',\n",
       " 42: '66',\n",
       " 43: '70',\n",
       " 44: '703',\n",
       " 45: '73',\n",
       " 46: '800',\n",
       " 47: '8044',\n",
       " 48: '80th',\n",
       " 49: ':',\n",
       " 50: ';',\n",
       " 51: '?',\n",
       " 52: '[',\n",
       " 53: '[female',\n",
       " 54: '[female]',\n",
       " 55: \"[female]'s\",\n",
       " 56: '[female].',\n",
       " 57: '[location',\n",
       " 58: '[location]',\n",
       " 59: \"[location]''\",\n",
       " 60: \"[location]''.\",\n",
       " 61: \"[location]'s\",\n",
       " 62: '[location],',\n",
       " 63: '[location].',\n",
       " 64: '[male',\n",
       " 65: '[male]',\n",
       " 66: \"[male]'s\",\n",
       " 67: '[male].',\n",
       " 68: '[organization',\n",
       " 69: '[organization]',\n",
       " 70: \"[organization]''.\",\n",
       " 71: \"[organization]'s\",\n",
       " 72: '[organization],',\n",
       " 73: '[organization].',\n",
       " 74: ']',\n",
       " 75: '``',\n",
       " 76: 'a',\n",
       " 77: 'a,',\n",
       " 78: 'abandoned',\n",
       " 79: 'abandoned,',\n",
       " 80: 'abandoned.',\n",
       " 81: 'ablaze',\n",
       " 82: 'ablaze.',\n",
       " 83: 'able',\n",
       " 84: 'abou',\n",
       " 85: 'about',\n",
       " 86: 'about.',\n",
       " 87: 'abov',\n",
       " 88: 'above',\n",
       " 89: 'above.',\n",
       " 90: 'absolutely',\n",
       " 91: 'abstract',\n",
       " 92: 'abstract,',\n",
       " 93: 'ac',\n",
       " 94: 'accent',\n",
       " 95: 'accent.',\n",
       " 96: 'accented',\n",
       " 97: 'accept',\n",
       " 98: 'access',\n",
       " 99: 'accessories',\n",
       " 100: 'accessories.',\n",
       " 101: 'accompanied',\n",
       " 102: 'according',\n",
       " 103: 'accurate',\n",
       " 104: 'ache',\n",
       " 105: 'acne',\n",
       " 106: 'acoustic',\n",
       " 107: 'acros',\n",
       " 108: 'across',\n",
       " 109: 'across.',\n",
       " 110: 'act',\n",
       " 111: 'acting',\n",
       " 112: 'action',\n",
       " 113: 'action.',\n",
       " 114: 'actively',\n",
       " 115: 'activists',\n",
       " 116: 'activities',\n",
       " 117: 'activity',\n",
       " 118: 'actors',\n",
       " 119: 'actual',\n",
       " 120: 'ad',\n",
       " 121: 'add',\n",
       " 122: 'added',\n",
       " 123: 'additional',\n",
       " 124: 'addresses',\n",
       " 125: 'adjusting',\n",
       " 126: 'administered.',\n",
       " 127: 'admire',\n",
       " 128: 'admired',\n",
       " 129: 'admires',\n",
       " 130: 'admiring',\n",
       " 131: 'admission',\n",
       " 132: 'adorable',\n",
       " 133: 'adorned',\n",
       " 134: 'adorns',\n",
       " 135: 'ads',\n",
       " 136: 'adul',\n",
       " 137: 'adult',\n",
       " 138: 'adult,',\n",
       " 139: 'adults',\n",
       " 140: 'adults.',\n",
       " 141: 'adventrious',\n",
       " 142: 'adventure.',\n",
       " 143: 'advert',\n",
       " 144: 'advertise',\n",
       " 145: 'advertised',\n",
       " 146: 'advertisement',\n",
       " 147: 'advertisements',\n",
       " 148: 'advertisements.',\n",
       " 149: 'advertises',\n",
       " 150: 'advertising',\n",
       " 151: 'advises',\n",
       " 152: 'aerial',\n",
       " 153: 'aerosol',\n",
       " 154: 'affection',\n",
       " 155: 'affixed',\n",
       " 156: 'african',\n",
       " 157: \"african's\",\n",
       " 158: 'afro',\n",
       " 159: 'aft',\n",
       " 160: 'after',\n",
       " 161: 'afternoon.',\n",
       " 162: 'agai',\n",
       " 163: 'again.',\n",
       " 164: 'against',\n",
       " 165: 'agape',\n",
       " 166: 'age',\n",
       " 167: 'aged',\n",
       " 168: 'ages.',\n",
       " 169: 'aggieland.',\n",
       " 170: 'aggressive.',\n",
       " 171: 'aging',\n",
       " 172: 'ago',\n",
       " 173: 'ago.',\n",
       " 174: 'ahea',\n",
       " 175: 'ahead',\n",
       " 176: 'ahead.',\n",
       " 177: 'ai',\n",
       " 178: 'aid',\n",
       " 179: 'aiming',\n",
       " 180: 'air',\n",
       " 181: 'air.',\n",
       " 182: 'airbrushed',\n",
       " 183: 'aircraft',\n",
       " 184: 'aircraft.',\n",
       " 185: 'airplane',\n",
       " 186: 'airplane.',\n",
       " 187: 'airplanes',\n",
       " 188: 'airpor',\n",
       " 189: 'airport',\n",
       " 190: 'airport.',\n",
       " 191: 'airshow',\n",
       " 192: 'aisle',\n",
       " 193: 'aladdin',\n",
       " 194: 'alarge',\n",
       " 195: 'album',\n",
       " 196: 'album.',\n",
       " 197: 'alcatel',\n",
       " 198: 'alcoho',\n",
       " 199: 'alcohol',\n",
       " 200: 'alcoholic',\n",
       " 201: 'alie',\n",
       " 202: 'alien',\n",
       " 203: 'alien.',\n",
       " 204: 'aliens',\n",
       " 205: 'aliens,',\n",
       " 206: 'align',\n",
       " 207: 'alka',\n",
       " 208: 'all',\n",
       " 209: 'alley',\n",
       " 210: 'alley.',\n",
       " 211: 'alligators',\n",
       " 212: 'allowing',\n",
       " 213: 'almost',\n",
       " 214: 'aloft.',\n",
       " 215: 'alon',\n",
       " 216: 'alone',\n",
       " 217: 'along',\n",
       " 218: 'along.',\n",
       " 219: 'alongside',\n",
       " 220: 'alot',\n",
       " 221: 'alpaca',\n",
       " 222: 'alpacas.',\n",
       " 223: 'already',\n",
       " 224: 'also',\n",
       " 225: 'altar',\n",
       " 226: 'altar.',\n",
       " 227: 'alter.',\n",
       " 228: 'altima',\n",
       " 229: 'altima,',\n",
       " 230: 'alto',\n",
       " 231: 'always',\n",
       " 232: 'am',\n",
       " 233: 'amazin',\n",
       " 234: 'amazing',\n",
       " 235: 'amber',\n",
       " 236: 'ambulance',\n",
       " 237: 'american',\n",
       " 238: 'americana',\n",
       " 239: 'americans',\n",
       " 240: 'amid',\n",
       " 241: 'amidst',\n",
       " 242: 'ammusment',\n",
       " 243: 'among',\n",
       " 244: 'amongst',\n",
       " 245: 'amount',\n",
       " 246: 'amount.',\n",
       " 247: 'amphitheate',\n",
       " 248: 'amphitheater',\n",
       " 249: 'amused.',\n",
       " 250: 'amusement',\n",
       " 251: 'amusment',\n",
       " 252: 'an',\n",
       " 253: 'analyssa',\n",
       " 254: 'ancho',\n",
       " 255: 'anchor',\n",
       " 256: 'anchor,',\n",
       " 257: 'anchored',\n",
       " 258: 'ancient',\n",
       " 259: 'and',\n",
       " 260: 'android',\n",
       " 261: 'anemone',\n",
       " 262: 'angel',\n",
       " 263: 'angeles',\n",
       " 264: 'angelic',\n",
       " 265: 'angels',\n",
       " 266: 'angle',\n",
       " 267: 'angle.',\n",
       " 268: 'angled',\n",
       " 269: 'angles',\n",
       " 270: 'angrily',\n",
       " 271: 'angry',\n",
       " 272: 'anima',\n",
       " 273: 'animal',\n",
       " 274: 'animal.',\n",
       " 275: 'animals',\n",
       " 276: 'animals.',\n",
       " 277: 'animated',\n",
       " 278: 'animatedly',\n",
       " 279: 'animation',\n",
       " 280: 'ankle',\n",
       " 281: 'anniversary',\n",
       " 282: 'announce',\n",
       " 283: 'announced',\n",
       " 284: 'announcemen',\n",
       " 285: 'announces',\n",
       " 286: 'announcing',\n",
       " 287: 'anothe',\n",
       " 288: 'another',\n",
       " 289: 'another,',\n",
       " 290: 'another.',\n",
       " 291: 'ans',\n",
       " 292: 'antelope',\n",
       " 293: 'antenna',\n",
       " 294: 'antenna.',\n",
       " 295: 'anticipating',\n",
       " 296: 'antique',\n",
       " 297: 'antlers',\n",
       " 298: 'anxiously',\n",
       " 299: 'any',\n",
       " 300: 'anymore.',\n",
       " 301: 'anyone',\n",
       " 302: 'apartment',\n",
       " 303: 'apartment.',\n",
       " 304: 'apparent',\n",
       " 305: 'apparently',\n",
       " 306: 'appartment',\n",
       " 307: 'appeal',\n",
       " 308: 'appear',\n",
       " 309: 'appearanc',\n",
       " 310: 'appearance',\n",
       " 311: 'appeared',\n",
       " 312: 'appears',\n",
       " 313: 'applauding',\n",
       " 314: 'applauds',\n",
       " 315: 'apple',\n",
       " 316: 'appointing',\n",
       " 317: 'approaches',\n",
       " 318: 'approaching',\n",
       " 319: 'approval',\n",
       " 320: 'apron',\n",
       " 321: 'aprons',\n",
       " 322: 'apse',\n",
       " 323: 'aquariu',\n",
       " 324: 'aquarium',\n",
       " 325: 'aquarium.',\n",
       " 326: 'ar',\n",
       " 327: 'arc',\n",
       " 328: 'arch',\n",
       " 329: 'arch.',\n",
       " 330: 'arche',\n",
       " 331: 'arched',\n",
       " 332: 'arches',\n",
       " 333: 'architectural',\n",
       " 334: 'architecture',\n",
       " 335: 'architecture.',\n",
       " 336: 'archway',\n",
       " 337: 'archways',\n",
       " 338: 'arcs',\n",
       " 339: 'arctic',\n",
       " 340: 'are',\n",
       " 341: 'are.',\n",
       " 342: 'area',\n",
       " 343: 'area.',\n",
       " 344: 'areas',\n",
       " 345: 'aren',\n",
       " 346: 'arena',\n",
       " 347: 'arena.',\n",
       " 348: 'arial',\n",
       " 349: 'arid',\n",
       " 350: 'arise',\n",
       " 351: 'arm',\n",
       " 352: 'arm.',\n",
       " 353: 'armchair',\n",
       " 354: 'armor',\n",
       " 355: 'arms',\n",
       " 356: 'arms.',\n",
       " 357: 'army',\n",
       " 358: 'army.',\n",
       " 359: 'aroun',\n",
       " 360: 'around',\n",
       " 361: 'around,',\n",
       " 362: 'around.',\n",
       " 363: 'aroung',\n",
       " 364: 'arraignment',\n",
       " 365: 'arranged',\n",
       " 366: 'arranges',\n",
       " 367: 'arranging',\n",
       " 368: 'array',\n",
       " 369: 'arrived',\n",
       " 370: 'arrived.',\n",
       " 371: 'arrives.',\n",
       " 372: 'arro',\n",
       " 373: 'arrow',\n",
       " 374: 'arrows',\n",
       " 375: 'arrows,',\n",
       " 376: 'art',\n",
       " 377: \"art''\",\n",
       " 378: 'art.',\n",
       " 379: 'artfully',\n",
       " 380: 'article',\n",
       " 381: 'artificial',\n",
       " 382: 'artis',\n",
       " 383: 'artist',\n",
       " 384: 'artist.',\n",
       " 385: 'artistic',\n",
       " 386: 'artistic,',\n",
       " 387: 'artistically',\n",
       " 388: 'artists',\n",
       " 389: 'arts',\n",
       " 390: 'artsy',\n",
       " 391: 'artwor',\n",
       " 392: 'artwork',\n",
       " 393: 'artwork.',\n",
       " 394: 'as',\n",
       " 395: 'asain',\n",
       " 396: 'asia',\n",
       " 397: 'asian',\n",
       " 398: 'asians',\n",
       " 399: 'aside',\n",
       " 400: 'asking',\n",
       " 401: 'asleep',\n",
       " 402: 'asleep.',\n",
       " 403: 'asphalt',\n",
       " 404: 'asphalt.',\n",
       " 405: 'aspiring',\n",
       " 406: 'assembly',\n",
       " 407: 'assignments',\n",
       " 408: 'associate.',\n",
       " 409: 'assorted',\n",
       " 410: 'assortment',\n",
       " 411: 'astonishment.',\n",
       " 412: 'at',\n",
       " 413: 'athlete',\n",
       " 414: 'athlete.',\n",
       " 415: 'athletes',\n",
       " 416: 'athleticism.',\n",
       " 417: 'atmosphere',\n",
       " 418: 'atmosphere.',\n",
       " 419: 'atop',\n",
       " 420: 'atrium',\n",
       " 421: 'attache',\n",
       " 422: 'attached',\n",
       " 423: 'attack.',\n",
       " 424: 'attempt',\n",
       " 425: 'attempting',\n",
       " 426: 'attempts',\n",
       " 427: 'attend',\n",
       " 428: 'attended',\n",
       " 429: 'attending',\n",
       " 430: 'attention',\n",
       " 431: 'attention.',\n",
       " 432: 'attentive',\n",
       " 433: 'attire',\n",
       " 434: 'attire,',\n",
       " 435: 'attire.',\n",
       " 436: 'attract',\n",
       " 437: 'attracting',\n",
       " 438: 'attraction',\n",
       " 439: 'attraction.',\n",
       " 440: 'attractions',\n",
       " 441: 'attractive',\n",
       " 442: 'audieance',\n",
       " 443: 'audienc',\n",
       " 444: 'audience',\n",
       " 445: 'audience.',\n",
       " 446: 'auditorium',\n",
       " 447: 'automated',\n",
       " 448: 'autum',\n",
       " 449: 'autumn',\n",
       " 450: 'autumn.',\n",
       " 451: 'autumnal',\n",
       " 452: 'available.',\n",
       " 453: 'avenue',\n",
       " 454: 'aviation',\n",
       " 455: 'avid',\n",
       " 456: 'aw',\n",
       " 457: 'awa',\n",
       " 458: 'awaited',\n",
       " 459: 'awaiting',\n",
       " 460: 'award',\n",
       " 461: 'award,',\n",
       " 462: 'award.',\n",
       " 463: 'awards',\n",
       " 464: 'awareness.',\n",
       " 465: 'away',\n",
       " 466: 'away.',\n",
       " 467: 'awe',\n",
       " 468: 'awesome',\n",
       " 469: 'awesome.',\n",
       " 470: 'awhile',\n",
       " 471: 'awkward',\n",
       " 472: 'awkwardly',\n",
       " 473: 'awning',\n",
       " 474: 'b',\n",
       " 475: 'ba',\n",
       " 476: 'bab',\n",
       " 477: 'baby',\n",
       " 478: 'baby,',\n",
       " 479: 'baby.',\n",
       " 480: 'bac',\n",
       " 481: 'back',\n",
       " 482: 'back.',\n",
       " 483: 'backdrop',\n",
       " 484: 'backdrop.',\n",
       " 485: 'backgrond',\n",
       " 486: 'backgroun',\n",
       " 487: 'background',\n",
       " 488: 'background,',\n",
       " 489: 'background.',\n",
       " 490: 'backlighting',\n",
       " 491: 'backlit',\n",
       " 492: 'backpack',\n",
       " 493: 'backpack.',\n",
       " 494: 'backpacker',\n",
       " 495: 'backpacks',\n",
       " 496: 'backpacks.',\n",
       " 497: 'backround',\n",
       " 498: 'backs',\n",
       " 499: 'backs.',\n",
       " 500: 'backseat',\n",
       " 501: 'backside',\n",
       " 502: 'backward',\n",
       " 503: 'backwards',\n",
       " 504: 'backyard',\n",
       " 505: 'backyard.',\n",
       " 506: 'bacon',\n",
       " 507: 'bacon,',\n",
       " 508: 'badge.',\n",
       " 509: 'bag',\n",
       " 510: 'bag.',\n",
       " 511: 'bagpipes.',\n",
       " 512: 'bags',\n",
       " 513: 'bags.',\n",
       " 514: 'baked',\n",
       " 515: 'bal',\n",
       " 516: 'balance.',\n",
       " 517: 'balanced',\n",
       " 518: 'balcon',\n",
       " 519: 'balconies',\n",
       " 520: 'balcony',\n",
       " 521: 'balcony,',\n",
       " 522: 'balcony.',\n",
       " 523: 'bald',\n",
       " 524: 'balding',\n",
       " 525: 'bale',\n",
       " 526: 'ball',\n",
       " 527: 'ball,',\n",
       " 528: 'ball.',\n",
       " 529: 'balloon',\n",
       " 530: 'balloon,',\n",
       " 531: 'balloon.',\n",
       " 532: 'balloons',\n",
       " 533: 'balloons.',\n",
       " 534: 'balls',\n",
       " 535: 'balls.',\n",
       " 536: 'bamboo.',\n",
       " 537: 'ban',\n",
       " 538: 'banana',\n",
       " 539: 'bananas.',\n",
       " 540: 'band',\n",
       " 541: 'band.',\n",
       " 542: 'bandage',\n",
       " 543: 'bandan',\n",
       " 544: 'bandana',\n",
       " 545: 'bandanna',\n",
       " 546: 'bandanna,',\n",
       " 547: 'bands.',\n",
       " 548: 'banjo',\n",
       " 549: 'bank',\n",
       " 550: 'bank.',\n",
       " 551: 'banks',\n",
       " 552: 'banks.',\n",
       " 553: 'banner',\n",
       " 554: 'banner.',\n",
       " 555: 'banners.',\n",
       " 556: 'bannister',\n",
       " 557: 'banquet.',\n",
       " 558: 'bar',\n",
       " 559: \"bar''.\",\n",
       " 560: 'bar,',\n",
       " 561: 'bar.',\n",
       " 562: 'barbecu',\n",
       " 563: 'barbecue',\n",
       " 564: 'barbecue.',\n",
       " 565: 'barbecuing',\n",
       " 566: 'barbed',\n",
       " 567: 'barbequ',\n",
       " 568: 'barbeque',\n",
       " 569: 'bare',\n",
       " 570: 'barely',\n",
       " 571: 'barg',\n",
       " 572: 'barge',\n",
       " 573: 'barge.',\n",
       " 574: 'barges',\n",
       " 575: 'barking',\n",
       " 576: 'barn',\n",
       " 577: 'barn.',\n",
       " 578: 'barren',\n",
       " 579: 'barrette',\n",
       " 580: 'barricades',\n",
       " 581: 'barrier',\n",
       " 582: 'barriers',\n",
       " 583: 'bars',\n",
       " 584: 'bas',\n",
       " 585: 'base',\n",
       " 586: 'base.',\n",
       " 587: 'baseball',\n",
       " 588: 'baseball.',\n",
       " 589: 'based',\n",
       " 590: 'basement',\n",
       " 591: 'basement.',\n",
       " 592: 'basin.',\n",
       " 593: 'basket',\n",
       " 594: 'basket.',\n",
       " 595: 'basketbal',\n",
       " 596: 'basketball',\n",
       " 597: 'basketball.',\n",
       " 598: 'baskets',\n",
       " 599: 'bass',\n",
       " 600: 'bass.',\n",
       " 601: 'bat',\n",
       " 602: 'bathed',\n",
       " 603: 'bathes',\n",
       " 604: 'bathing',\n",
       " 605: 'bathroo',\n",
       " 606: 'bathroom',\n",
       " 607: 'bathroom.',\n",
       " 608: 'bathrooms',\n",
       " 609: 'batman',\n",
       " 610: 'batter',\n",
       " 611: 'battl',\n",
       " 612: 'battle.',\n",
       " 613: 'bay',\n",
       " 614: 'bay.',\n",
       " 615: 'bayonet',\n",
       " 616: 'bbq',\n",
       " 617: 'be',\n",
       " 618: 'bea',\n",
       " 619: 'beac',\n",
       " 620: 'beach',\n",
       " 621: 'beach,',\n",
       " 622: 'beach.',\n",
       " 623: 'beads.',\n",
       " 624: 'beadstead',\n",
       " 625: 'beagle',\n",
       " 626: 'beaked',\n",
       " 627: 'beam',\n",
       " 628: 'beams',\n",
       " 629: 'beams.',\n",
       " 630: 'bear',\n",
       " 631: 'bear.',\n",
       " 632: 'beard',\n",
       " 633: 'bearded',\n",
       " 634: 'bearers',\n",
       " 635: 'bearing',\n",
       " 636: 'bears',\n",
       " 637: 'bears.',\n",
       " 638: 'beast',\n",
       " 639: 'beautifu',\n",
       " 640: 'beautiful',\n",
       " 641: 'beautiful,',\n",
       " 642: 'beautiful.',\n",
       " 643: 'beautifully',\n",
       " 644: 'beauty',\n",
       " 645: 'because',\n",
       " 646: 'beckons',\n",
       " 647: 'becomes',\n",
       " 648: 'bed',\n",
       " 649: 'bed,',\n",
       " 650: 'bed.',\n",
       " 651: 'bedding.',\n",
       " 652: 'bedroom',\n",
       " 653: 'bedroom.',\n",
       " 654: 'bee',\n",
       " 655: 'beef,',\n",
       " 656: 'been',\n",
       " 657: 'beer',\n",
       " 658: 'beer,',\n",
       " 659: 'beer.',\n",
       " 660: 'beers',\n",
       " 661: 'beetle.',\n",
       " 662: 'before',\n",
       " 663: 'begin',\n",
       " 664: 'begin.',\n",
       " 665: 'begins.',\n",
       " 666: 'behin',\n",
       " 667: 'behind',\n",
       " 668: 'behind,',\n",
       " 669: 'behind.',\n",
       " 670: 'behold.',\n",
       " 671: 'beige',\n",
       " 672: 'beige,',\n",
       " 673: 'being',\n",
       " 674: 'below',\n",
       " 675: 'below,',\n",
       " 676: 'below.',\n",
       " 677: 'belted',\n",
       " 678: 'belts',\n",
       " 679: 'ben',\n",
       " 680: 'benc',\n",
       " 681: 'bench',\n",
       " 682: 'bench.',\n",
       " 683: 'benches',\n",
       " 684: 'bend',\n",
       " 685: 'bending',\n",
       " 686: 'bends',\n",
       " 687: 'beneat',\n",
       " 688: 'beneath',\n",
       " 689: 'bento',\n",
       " 690: 'berets',\n",
       " 691: 'berkeley',\n",
       " 692: \"berkeley''\",\n",
       " 693: 'bernard',\n",
       " 694: 'berrie',\n",
       " 695: 'berries',\n",
       " 696: 'berry',\n",
       " 697: 'berserk',\n",
       " 698: 'beside',\n",
       " 699: 'besides',\n",
       " 700: 'bespectacled',\n",
       " 701: 'best',\n",
       " 702: 'better',\n",
       " 703: 'better.',\n",
       " 704: 'between',\n",
       " 705: 'beutiful',\n",
       " 706: 'beverage',\n",
       " 707: 'beverage.',\n",
       " 708: 'beverages',\n",
       " 709: 'beverages.',\n",
       " 710: 'beyond',\n",
       " 711: 'bi',\n",
       " 712: 'bib',\n",
       " 713: 'bible.',\n",
       " 714: 'bicep',\n",
       " 715: 'bicycl',\n",
       " 716: 'bicycle',\n",
       " 717: 'bicycle,',\n",
       " 718: 'bicycle.',\n",
       " 719: 'bicycles',\n",
       " 720: 'bicycles.',\n",
       " 721: 'bicycling',\n",
       " 722: 'bicyclist',\n",
       " 723: 'bicyclists',\n",
       " 724: 'bide',\n",
       " 725: 'big',\n",
       " 726: 'big,',\n",
       " 727: 'bigger',\n",
       " 728: 'bight',\n",
       " 729: 'bik',\n",
       " 730: 'bike',\n",
       " 731: 'bike,',\n",
       " 732: 'bike.',\n",
       " 733: 'bikes',\n",
       " 734: 'bikes,',\n",
       " 735: 'bikes.',\n",
       " 736: 'biking',\n",
       " 737: 'biking,',\n",
       " 738: 'bikini',\n",
       " 739: 'bikrs',\n",
       " 740: 'billboard',\n",
       " 741: 'billboards',\n",
       " 742: 'billiards',\n",
       " 743: 'billowing',\n",
       " 744: 'bin',\n",
       " 745: 'binary',\n",
       " 746: 'birch',\n",
       " 747: 'bird',\n",
       " 748: 'bird.',\n",
       " 749: 'birds',\n",
       " 750: 'birds!',\n",
       " 751: 'birds.',\n",
       " 752: 'birthda',\n",
       " 753: 'birthday',\n",
       " 754: 'birthday.',\n",
       " 755: 'biscuit',\n",
       " 756: 'biscuits',\n",
       " 757: 'bit',\n",
       " 758: 'bite',\n",
       " 759: 'bites',\n",
       " 760: 'biting',\n",
       " 761: 'bitten',\n",
       " 762: 'blac',\n",
       " 763: 'black',\n",
       " 764: 'black,',\n",
       " 765: 'black-and-brown-furred',\n",
       " 766: 'black.',\n",
       " 767: 'blackberries',\n",
       " 768: 'blacked',\n",
       " 769: 'blacks',\n",
       " 770: 'blacvk',\n",
       " 771: 'blade',\n",
       " 772: 'blades',\n",
       " 773: 'blan',\n",
       " 774: 'blank',\n",
       " 775: 'blanke',\n",
       " 776: 'blanket',\n",
       " 777: 'blanket,',\n",
       " 778: 'blanket.',\n",
       " 779: 'blankets',\n",
       " 780: 'blankets.',\n",
       " 781: 'blast',\n",
       " 782: \"blast'\",\n",
       " 783: 'blasting',\n",
       " 784: 'bleacher',\n",
       " 785: 'bleachers',\n",
       " 786: 'bleachers.',\n",
       " 787: 'blend',\n",
       " 788: 'blimp',\n",
       " 789: 'blo',\n",
       " 790: 'bloc',\n",
       " 791: 'block',\n",
       " 792: \"block''\",\n",
       " 793: 'block.',\n",
       " 794: 'blocks',\n",
       " 795: 'blocks.',\n",
       " 796: 'blond',\n",
       " 797: 'blonde',\n",
       " 798: 'bloo',\n",
       " 799: 'bloom',\n",
       " 800: 'bloom.',\n",
       " 801: 'blooming',\n",
       " 802: 'blossom',\n",
       " 803: 'blossoming',\n",
       " 804: 'blossoms.',\n",
       " 805: 'blous',\n",
       " 806: 'blouse',\n",
       " 807: 'blouse.',\n",
       " 808: 'blow',\n",
       " 809: 'blowing',\n",
       " 810: 'blown',\n",
       " 811: 'blows',\n",
       " 812: 'blu',\n",
       " 813: 'blue',\n",
       " 814: 'blue,',\n",
       " 815: 'blue.',\n",
       " 816: 'blueb',\n",
       " 817: 'blueberries',\n",
       " 818: 'blueberry',\n",
       " 819: 'bluejeans',\n",
       " 820: 'blues',\n",
       " 821: 'bluff',\n",
       " 822: 'blur',\n",
       " 823: 'blurred',\n",
       " 824: 'blurry',\n",
       " 825: 'blurry.',\n",
       " 826: 'blurs',\n",
       " 827: 'bo',\n",
       " 828: 'boa',\n",
       " 829: 'boad',\n",
       " 830: 'boar',\n",
       " 831: 'board',\n",
       " 832: 'board,',\n",
       " 833: 'board.',\n",
       " 834: 'boarded',\n",
       " 835: 'boarding',\n",
       " 836: 'boards',\n",
       " 837: 'boards.',\n",
       " 838: 'boardwalk',\n",
       " 839: 'boardwalk.',\n",
       " 840: 'boat',\n",
       " 841: 'boat.',\n",
       " 842: 'boaters',\n",
       " 843: 'boats',\n",
       " 844: 'boats.',\n",
       " 845: 'bob',\n",
       " 846: 'bobbing',\n",
       " 847: 'bock',\n",
       " 848: 'bodies',\n",
       " 849: 'bodies.',\n",
       " 850: 'body',\n",
       " 851: 'body.',\n",
       " 852: 'boiled',\n",
       " 853: 'bomb',\n",
       " 854: 'bonfire',\n",
       " 855: 'bonfire.',\n",
       " 856: 'bonnet',\n",
       " 857: 'bonnets.',\n",
       " 858: 'book',\n",
       " 859: 'book.',\n",
       " 860: 'books',\n",
       " 861: 'books,',\n",
       " 862: 'booming',\n",
       " 863: 'boot',\n",
       " 864: 'booth',\n",
       " 865: 'booth,',\n",
       " 866: 'booth.',\n",
       " 867: 'booths',\n",
       " 868: 'boots',\n",
       " 869: 'bored',\n",
       " 870: 'bot',\n",
       " 871: 'both',\n",
       " 872: 'bottl',\n",
       " 873: 'bottle',\n",
       " 874: 'bottle.',\n",
       " 875: 'bottled',\n",
       " 876: 'bottles',\n",
       " 877: 'bottles,',\n",
       " 878: 'bottles.',\n",
       " 879: 'bottling',\n",
       " 880: 'botto',\n",
       " 881: 'bottom',\n",
       " 882: 'bottom.',\n",
       " 883: 'bough',\n",
       " 884: 'bought',\n",
       " 885: 'bought.',\n",
       " 886: 'boulder',\n",
       " 887: 'boulders',\n",
       " 888: 'bounce',\n",
       " 889: 'bounce.',\n",
       " 890: 'bounced',\n",
       " 891: 'bouncing',\n",
       " 892: 'bouncy',\n",
       " 893: 'bouquet',\n",
       " 894: 'bouquets',\n",
       " 895: 'bow',\n",
       " 896: 'bowl',\n",
       " 897: 'bowl.',\n",
       " 898: 'bowling',\n",
       " 899: 'bowls',\n",
       " 900: 'box',\n",
       " 901: 'box.',\n",
       " 902: 'boxes',\n",
       " 903: 'boxes,',\n",
       " 904: 'boy',\n",
       " 905: \"boy's\",\n",
       " 906: 'boy,',\n",
       " 907: 'boy.',\n",
       " 908: 'boyfriend',\n",
       " 909: 'boys',\n",
       " 910: 'boys,',\n",
       " 911: 'bracelets',\n",
       " 912: 'braided,',\n",
       " 913: 'branch',\n",
       " 914: 'branch.',\n",
       " 915: 'branche',\n",
       " 916: 'branches',\n",
       " 917: 'branches.',\n",
       " 918: 'brand.',\n",
       " 919: 'brandishing',\n",
       " 920: 'brass',\n",
       " 921: 'brave',\n",
       " 922: 'braving',\n",
       " 923: 'bread',\n",
       " 924: 'bread,',\n",
       " 925: 'bread.',\n",
       " 926: 'breaded',\n",
       " 927: 'break',\n",
       " 928: 'break.',\n",
       " 929: 'breakdanc',\n",
       " 930: 'breakers',\n",
       " 931: 'breakfast',\n",
       " 932: 'breakfast!',\n",
       " 933: 'breaks',\n",
       " 934: 'breather',\n",
       " 935: 'breathing',\n",
       " 936: 'breathtaking',\n",
       " 937: 'breed',\n",
       " 938: 'breeze',\n",
       " 939: 'brew',\n",
       " 940: 'brick',\n",
       " 941: 'bricked',\n",
       " 942: 'bricks',\n",
       " 943: 'bricks.',\n",
       " 944: 'bridal',\n",
       " 945: 'bride',\n",
       " 946: 'bridg',\n",
       " 947: 'bridge',\n",
       " 948: 'bridge.',\n",
       " 949: 'bridges',\n",
       " 950: 'briefcase',\n",
       " 951: 'briefcase.',\n",
       " 952: 'briefs',\n",
       " 953: 'brigh',\n",
       " 954: 'bright',\n",
       " 955: 'bright,',\n",
       " 956: 'bright.',\n",
       " 957: 'brighten',\n",
       " 958: 'brightens',\n",
       " 959: 'brighting',\n",
       " 960: 'brightly',\n",
       " 961: 'brightly-colored',\n",
       " 962: 'brightly-lit',\n",
       " 963: 'brightly.',\n",
       " 964: 'brilliance',\n",
       " 965: 'brilliant',\n",
       " 966: 'brilliantly',\n",
       " 967: 'bring',\n",
       " 968: 'brings',\n",
       " 969: 'british',\n",
       " 970: 'broad',\n",
       " 971: 'broken',\n",
       " 972: 'bronco',\n",
       " 973: 'bronze',\n",
       " 974: 'bronzed',\n",
       " 975: 'broom',\n",
       " 976: \"brother'',\",\n",
       " 977: 'brother,',\n",
       " 978: 'brother.',\n",
       " 979: 'brothers',\n",
       " 980: 'brow',\n",
       " 981: 'brown',\n",
       " 982: 'brown,',\n",
       " 983: 'brown.',\n",
       " 984: 'brownish',\n",
       " 985: 'browns',\n",
       " 986: 'browsing',\n",
       " 987: 'brunette',\n",
       " 988: 'brush',\n",
       " 989: 'brush.',\n",
       " 990: 'bu',\n",
       " 991: 'bubble',\n",
       " 992: 'bubbles',\n",
       " 993: 'bubbles.',\n",
       " 994: 'buch',\n",
       " 995: 'bucked',\n",
       " 996: 'bucket',\n",
       " 997: 'bucket.',\n",
       " 998: 'buckets',\n",
       " 999: 'buckets.',\n",
       " ...}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reverse_input_char_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: '!',\n",
       " 1: '#',\n",
       " 2: '$',\n",
       " 3: '&',\n",
       " 4: \"'\",\n",
       " 5: \"''\",\n",
       " 6: '(',\n",
       " 7: ')',\n",
       " 8: '-',\n",
       " 9: '--',\n",
       " 10: '-free',\n",
       " 11: '.',\n",
       " 12: '..',\n",
       " 13: '...',\n",
       " 14: '.finally!',\n",
       " 15: '0.0',\n",
       " 16: '000',\n",
       " 17: '1',\n",
       " 18: '10',\n",
       " 19: '100',\n",
       " 20: \"100's\",\n",
       " 21: '100,000',\n",
       " 22: '1000',\n",
       " 23: '11:30pm,',\n",
       " 24: '12',\n",
       " 25: '12.',\n",
       " 26: '13',\n",
       " 27: '1300',\n",
       " 28: '1500',\n",
       " 29: '1700',\n",
       " 30: \"1700's.\",\n",
       " 31: \"1800's.\",\n",
       " 32: '1900',\n",
       " 33: '1940',\n",
       " 34: '1940s',\n",
       " 35: '1942',\n",
       " 36: '1994.',\n",
       " 37: '1995.',\n",
       " 38: '1998',\n",
       " 39: '1st',\n",
       " 40: '2',\n",
       " 41: '20',\n",
       " 42: '20,000',\n",
       " 43: '2nd',\n",
       " 44: '3',\n",
       " 45: '30',\n",
       " 46: '30,',\n",
       " 47: '30th',\n",
       " 48: '37',\n",
       " 49: '3d',\n",
       " 50: '3rd',\n",
       " 51: '4',\n",
       " 52: \"40's.\",\n",
       " 53: '45',\n",
       " 54: '4h',\n",
       " 55: '4t',\n",
       " 56: '4th',\n",
       " 57: '4th!',\n",
       " 58: '4th.',\n",
       " 59: '5',\n",
       " 60: '5.',\n",
       " 61: '56',\n",
       " 62: '5th',\n",
       " 63: '66',\n",
       " 64: '7th',\n",
       " 65: '80',\n",
       " 66: \"80's\",\n",
       " 67: '9:20',\n",
       " 68: ':',\n",
       " 69: ';',\n",
       " 70: '?',\n",
       " 71: \"?'\",\n",
       " 72: 'START_',\n",
       " 73: '[female',\n",
       " 74: '[female]',\n",
       " 75: '[female]!',\n",
       " 76: \"[female]'s\",\n",
       " 77: '[female],',\n",
       " 78: '[female].',\n",
       " 79: '[male',\n",
       " 80: '[male]',\n",
       " 81: '[male]!',\n",
       " 82: \"[male]'s\",\n",
       " 83: \"[male]'s''.\",\n",
       " 84: \"[male]'s,\",\n",
       " 85: \"[male]'s.\",\n",
       " 86: '[male],',\n",
       " 87: '[male].',\n",
       " 88: '_END',\n",
       " 89: '``',\n",
       " 90: 'a',\n",
       " 91: 'a+',\n",
       " 92: 'aa',\n",
       " 93: 'aardvark.',\n",
       " 94: 'abandoned',\n",
       " 95: 'abandoned.',\n",
       " 96: 'abilities.',\n",
       " 97: 'ablaze',\n",
       " 98: 'able',\n",
       " 99: 'aboard',\n",
       " 100: 'abou',\n",
       " 101: 'about',\n",
       " 102: 'about.',\n",
       " 103: 'above',\n",
       " 104: 'above.',\n",
       " 105: 'abroad',\n",
       " 106: 'absolutely',\n",
       " 107: 'abstract',\n",
       " 108: 'abyss.',\n",
       " 109: 'academic',\n",
       " 110: 'academy,',\n",
       " 111: 'accelerate',\n",
       " 112: 'accented',\n",
       " 113: 'accept',\n",
       " 114: 'access',\n",
       " 115: 'accident.',\n",
       " 116: 'acclimated.',\n",
       " 117: 'accolades.',\n",
       " 118: 'accommodating',\n",
       " 119: 'accomodations.',\n",
       " 120: 'accomplished',\n",
       " 121: 'accomplishment.',\n",
       " 122: 'accomplishments',\n",
       " 123: 'account,',\n",
       " 124: 'accounted',\n",
       " 125: 'accounting',\n",
       " 126: 'accounts',\n",
       " 127: 'achievement.',\n",
       " 128: 'achievements',\n",
       " 129: 'achievements.',\n",
       " 130: 'aching.',\n",
       " 131: 'acres',\n",
       " 132: 'acrobatics',\n",
       " 133: 'across',\n",
       " 134: 'across!',\n",
       " 135: 'act',\n",
       " 136: 'act!',\n",
       " 137: 'act,',\n",
       " 138: 'act.',\n",
       " 139: 'acted',\n",
       " 140: 'acting',\n",
       " 141: 'actio',\n",
       " 142: 'action',\n",
       " 143: 'action!',\n",
       " 144: 'action.',\n",
       " 145: 'active',\n",
       " 146: 'activities',\n",
       " 147: 'activities!',\n",
       " 148: 'activities,',\n",
       " 149: 'activities.',\n",
       " 150: 'activity',\n",
       " 151: 'activity.',\n",
       " 152: 'actor',\n",
       " 153: 'actors',\n",
       " 154: 'actress',\n",
       " 155: 'acts',\n",
       " 156: 'acts.',\n",
       " 157: 'actually',\n",
       " 158: 'actually.',\n",
       " 159: \"ad's\",\n",
       " 160: 'adage',\n",
       " 161: 'adapted',\n",
       " 162: 'add',\n",
       " 163: 'added',\n",
       " 164: 'addicted',\n",
       " 165: 'addiction',\n",
       " 166: 'addition',\n",
       " 167: 'addressing',\n",
       " 168: 'adds',\n",
       " 169: 'adjacent',\n",
       " 170: 'adjacent.',\n",
       " 171: 'adjoining',\n",
       " 172: 'adjustments,',\n",
       " 173: 'adjusts',\n",
       " 174: 'admiration,',\n",
       " 175: 'admire',\n",
       " 176: 'admired',\n",
       " 177: 'admirer',\n",
       " 178: 'admiring',\n",
       " 179: 'adopt',\n",
       " 180: 'adopted',\n",
       " 181: 'adorable',\n",
       " 182: 'adorable!',\n",
       " 183: 'adorable,',\n",
       " 184: 'adorned',\n",
       " 185: 'adrenaline',\n",
       " 186: 'adult',\n",
       " 187: 'adults',\n",
       " 188: 'adults.',\n",
       " 189: 'advance',\n",
       " 190: 'advance.',\n",
       " 191: 'advancements',\n",
       " 192: 'advantage',\n",
       " 193: 'advantage.',\n",
       " 194: 'adventure',\n",
       " 195: 'adventure!',\n",
       " 196: 'adventure,',\n",
       " 197: 'adventure.',\n",
       " 198: 'adventurers.',\n",
       " 199: 'adventures',\n",
       " 200: 'adventures.',\n",
       " 201: 'adventuring',\n",
       " 202: 'adventurous',\n",
       " 203: 'adventurous.',\n",
       " 204: 'advertising',\n",
       " 205: 'advertising.',\n",
       " 206: 'advisors',\n",
       " 207: 'aerial',\n",
       " 208: 'afar',\n",
       " 209: 'afar.',\n",
       " 210: 'affair.',\n",
       " 211: 'affect',\n",
       " 212: 'affect.',\n",
       " 213: 'affections',\n",
       " 214: 'afraid',\n",
       " 215: 'afraid,',\n",
       " 216: 'after',\n",
       " 217: 'after,',\n",
       " 218: 'after.',\n",
       " 219: 'afterall.',\n",
       " 220: 'aftermath',\n",
       " 221: 'afternoo',\n",
       " 222: 'afternoon',\n",
       " 223: 'afternoon!',\n",
       " 224: 'afternoon.',\n",
       " 225: 'afterward',\n",
       " 226: 'afterward,',\n",
       " 227: 'afterward.',\n",
       " 228: 'afterwards',\n",
       " 229: 'afterwards,',\n",
       " 230: 'afterwards.',\n",
       " 231: 'again',\n",
       " 232: 'again!',\n",
       " 233: 'again,',\n",
       " 234: 'again.',\n",
       " 235: 'against',\n",
       " 236: 'agenda',\n",
       " 237: 'agenda.',\n",
       " 238: 'agent',\n",
       " 239: 'ages',\n",
       " 240: 'ages.',\n",
       " 241: 'aggieland',\n",
       " 242: 'aging',\n",
       " 243: 'ago.',\n",
       " 244: 'agree',\n",
       " 245: 'agreed',\n",
       " 246: 'agreed!',\n",
       " 247: 'agreements.',\n",
       " 248: 'agricultural',\n",
       " 249: 'ahead',\n",
       " 250: 'ahead.',\n",
       " 251: 'ahhd',\n",
       " 252: 'ai',\n",
       " 253: 'aid',\n",
       " 254: 'aim',\n",
       " 255: 'aimed',\n",
       " 256: 'air',\n",
       " 257: 'air,',\n",
       " 258: 'air.',\n",
       " 259: 'airforce',\n",
       " 260: 'airline',\n",
       " 261: 'airplac',\n",
       " 262: 'airplane',\n",
       " 263: 'airplane.',\n",
       " 264: 'airplanes.',\n",
       " 265: 'airport.',\n",
       " 266: 'airshow,',\n",
       " 267: 'aisle.',\n",
       " 268: 'aisles',\n",
       " 269: 'aladdin',\n",
       " 270: 'alarms.',\n",
       " 271: 'albums.',\n",
       " 272: 'alcohol',\n",
       " 273: 'alerady.',\n",
       " 274: 'alert',\n",
       " 275: 'algae',\n",
       " 276: 'alien',\n",
       " 277: 'aliens',\n",
       " 278: 'alike',\n",
       " 279: 'alike,',\n",
       " 280: 'alike.',\n",
       " 281: 'aliv',\n",
       " 282: 'alive',\n",
       " 283: 'alive.',\n",
       " 284: 'alka',\n",
       " 285: 'all',\n",
       " 286: 'all!',\n",
       " 287: 'all,',\n",
       " 288: 'all.',\n",
       " 289: 'alley.',\n",
       " 290: 'allow',\n",
       " 291: 'allowed',\n",
       " 292: 'allowed!',\n",
       " 293: 'allowing',\n",
       " 294: 'allows',\n",
       " 295: 'allready',\n",
       " 296: 'almost',\n",
       " 297: 'alo',\n",
       " 298: 'alon',\n",
       " 299: 'alone',\n",
       " 300: 'alone,',\n",
       " 301: 'alone.',\n",
       " 302: 'along',\n",
       " 303: 'along.',\n",
       " 304: 'alongside',\n",
       " 305: 'alot',\n",
       " 306: 'alot.',\n",
       " 307: 'aloud',\n",
       " 308: 'alpaca.',\n",
       " 309: 'alpacas!',\n",
       " 310: 'already',\n",
       " 311: 'already,',\n",
       " 312: 'already.',\n",
       " 313: 'alright,',\n",
       " 314: 'alright.',\n",
       " 315: 'als',\n",
       " 316: 'also',\n",
       " 317: 'also!',\n",
       " 318: 'alte',\n",
       " 319: 'alter,',\n",
       " 320: 'although',\n",
       " 321: 'although,',\n",
       " 322: 'altogether,',\n",
       " 323: 'always',\n",
       " 324: 'am',\n",
       " 325: 'amateur',\n",
       " 326: 'amaze',\n",
       " 327: 'amazed',\n",
       " 328: 'amazed!',\n",
       " 329: 'amazed.',\n",
       " 330: 'amazement',\n",
       " 331: 'amazement.',\n",
       " 332: 'amazin',\n",
       " 333: 'amazing',\n",
       " 334: 'amazing!',\n",
       " 335: 'amazing,',\n",
       " 336: 'amazing.',\n",
       " 337: 'amazingly',\n",
       " 338: 'ambiance',\n",
       " 339: 'amd',\n",
       " 340: 'amed',\n",
       " 341: 'amenities.',\n",
       " 342: 'american',\n",
       " 343: 'american.',\n",
       " 344: 'americans',\n",
       " 345: 'amish',\n",
       " 346: 'among',\n",
       " 347: 'amongst',\n",
       " 348: 'amount',\n",
       " 349: 'amp',\n",
       " 350: 'ample',\n",
       " 351: 'amused.',\n",
       " 352: 'amusement',\n",
       " 353: 'amusements',\n",
       " 354: 'amusing.',\n",
       " 355: 'an',\n",
       " 356: 'analog',\n",
       " 357: 'anchor',\n",
       " 358: 'anchor.',\n",
       " 359: 'ancient',\n",
       " 360: 'ancient.',\n",
       " 361: 'and',\n",
       " 362: 'and,',\n",
       " 363: 'anemone!',\n",
       " 364: 'angel',\n",
       " 365: 'angels',\n",
       " 366: 'angels.',\n",
       " 367: 'anger.',\n",
       " 368: 'angle',\n",
       " 369: 'angle,',\n",
       " 370: 'angle.',\n",
       " 371: 'angry',\n",
       " 372: 'angry.',\n",
       " 373: 'animal',\n",
       " 374: \"animal's\",\n",
       " 375: 'animal,',\n",
       " 376: 'animal.',\n",
       " 377: 'animals',\n",
       " 378: 'animals,',\n",
       " 379: 'animals.',\n",
       " 380: 'animals.we',\n",
       " 381: 'anniversary',\n",
       " 382: 'anniversary,',\n",
       " 383: 'announced',\n",
       " 384: 'announcemne',\n",
       " 385: 'annoy.',\n",
       " 386: 'annoyed,',\n",
       " 387: 'annoying,',\n",
       " 388: 'annual',\n",
       " 389: 'anoint',\n",
       " 390: 'anonymous',\n",
       " 391: 'another',\n",
       " 392: 'another!',\n",
       " 393: 'another.',\n",
       " 394: 'answer.',\n",
       " 395: 'anticipate',\n",
       " 396: 'anticipated',\n",
       " 397: 'anticipates',\n",
       " 398: 'anticipating',\n",
       " 399: 'anticipation',\n",
       " 400: 'anticipation.',\n",
       " 401: 'antique',\n",
       " 402: 'antiques',\n",
       " 403: 'antiques.',\n",
       " 404: 'antlers',\n",
       " 405: 'ants!',\n",
       " 406: 'anual',\n",
       " 407: 'anxious',\n",
       " 408: 'any',\n",
       " 409: 'any,',\n",
       " 410: 'anybody',\n",
       " 411: 'anymore.',\n",
       " 412: 'anyone',\n",
       " 413: 'anything',\n",
       " 414: 'anything.',\n",
       " 415: 'anyway.',\n",
       " 416: 'anyways.',\n",
       " 417: 'anywhere',\n",
       " 418: 'anywhere.',\n",
       " 419: 'apart',\n",
       " 420: 'apart.',\n",
       " 421: 'apartment',\n",
       " 422: 'apartment.',\n",
       " 423: 'apartments',\n",
       " 424: 'app',\n",
       " 425: 'apparently',\n",
       " 426: 'appeal',\n",
       " 427: 'appealing,',\n",
       " 428: 'appealing.',\n",
       " 429: 'appear',\n",
       " 430: 'appearance',\n",
       " 431: 'appearance.',\n",
       " 432: 'appeared',\n",
       " 433: 'appeared!',\n",
       " 434: 'appeared,',\n",
       " 435: 'appeared.',\n",
       " 436: 'appears',\n",
       " 437: 'appetite',\n",
       " 438: 'appetite.',\n",
       " 439: 'appetizing',\n",
       " 440: 'applauded',\n",
       " 441: 'applauded.',\n",
       " 442: 'applause.',\n",
       " 443: 'apple',\n",
       " 444: 'apply',\n",
       " 445: 'appreciate',\n",
       " 446: 'appreciate.',\n",
       " 447: 'appreciated',\n",
       " 448: 'appreciating',\n",
       " 449: 'appreciative',\n",
       " 450: 'approach',\n",
       " 451: 'approached',\n",
       " 452: 'approached,',\n",
       " 453: 'approached.',\n",
       " 454: 'approaches',\n",
       " 455: 'approaching',\n",
       " 456: 'approaching.',\n",
       " 457: 'appropriate',\n",
       " 458: 'appropriately',\n",
       " 459: 'approval',\n",
       " 460: 'approve',\n",
       " 461: 'approved',\n",
       " 462: 'aquarium',\n",
       " 463: 'aquarium.',\n",
       " 464: 'aquariums.',\n",
       " 465: 'ar',\n",
       " 466: 'arcade',\n",
       " 467: 'arch',\n",
       " 468: 'arch.',\n",
       " 469: 'archaeologist',\n",
       " 470: 'arches',\n",
       " 471: 'arching',\n",
       " 472: 'architcture',\n",
       " 473: 'architectur',\n",
       " 474: 'architecture',\n",
       " 475: 'architecture,',\n",
       " 476: 'architecture.',\n",
       " 477: 'archway',\n",
       " 478: 'archway.',\n",
       " 479: 'are',\n",
       " 480: 'are,',\n",
       " 481: 'are.',\n",
       " 482: 'area',\n",
       " 483: 'area!',\n",
       " 484: \"area's\",\n",
       " 485: 'area,',\n",
       " 486: 'area.',\n",
       " 487: 'areas',\n",
       " 488: 'areas.',\n",
       " 489: \"aren't\",\n",
       " 490: 'arena',\n",
       " 491: 'arena.',\n",
       " 492: 'arguments,',\n",
       " 493: 'arm',\n",
       " 494: 'armed',\n",
       " 495: 'arms',\n",
       " 496: 'arms.',\n",
       " 497: 'army',\n",
       " 498: 'aromas.',\n",
       " 499: 'arose',\n",
       " 500: 'aroun',\n",
       " 501: 'around',\n",
       " 502: 'around!',\n",
       " 503: 'around,',\n",
       " 504: 'around.',\n",
       " 505: 'aroung.',\n",
       " 506: 'arr!',\n",
       " 507: 'arrangement',\n",
       " 508: 'array',\n",
       " 509: 'arrested',\n",
       " 510: 'arrival',\n",
       " 511: 'arrival.',\n",
       " 512: 'arrive',\n",
       " 513: 'arrive,',\n",
       " 514: 'arrive.',\n",
       " 515: 'arrived',\n",
       " 516: 'arrived!',\n",
       " 517: 'arrived,',\n",
       " 518: 'arrived.',\n",
       " 519: 'arrives',\n",
       " 520: 'arrives,',\n",
       " 521: 'arriving',\n",
       " 522: 'arriving.',\n",
       " 523: 'art',\n",
       " 524: 'art,',\n",
       " 525: 'art.',\n",
       " 526: 'artful',\n",
       " 527: 'artifact',\n",
       " 528: 'artifacts',\n",
       " 529: 'artisans',\n",
       " 530: 'artist',\n",
       " 531: 'artist.',\n",
       " 532: 'artistic',\n",
       " 533: 'artistic,',\n",
       " 534: 'artistic.',\n",
       " 535: 'artistically',\n",
       " 536: 'artistry',\n",
       " 537: 'artists',\n",
       " 538: 'artists.',\n",
       " 539: 'artsy,',\n",
       " 540: 'artwork',\n",
       " 541: 'artwork,',\n",
       " 542: 'artwork.',\n",
       " 543: 'as',\n",
       " 544: 'as,',\n",
       " 545: 'ashes',\n",
       " 546: 'ashuman',\n",
       " 547: 'asian',\n",
       " 548: 'ask',\n",
       " 549: 'asked',\n",
       " 550: 'asked.',\n",
       " 551: 'asking',\n",
       " 552: 'asks',\n",
       " 553: 'asleep',\n",
       " 554: 'asleep.',\n",
       " 555: 'aspects',\n",
       " 556: 'assemble',\n",
       " 557: 'assembling',\n",
       " 558: 'assembly',\n",
       " 559: 'assembly.',\n",
       " 560: 'asshole',\n",
       " 561: 'assignmen',\n",
       " 562: 'assorted',\n",
       " 563: 'assortment',\n",
       " 564: 'assumed',\n",
       " 565: 'asteroids',\n",
       " 566: 'astonished',\n",
       " 567: 'astonishing.',\n",
       " 568: 'astounding.',\n",
       " 569: 'at',\n",
       " 570: 'at,',\n",
       " 571: 'at.',\n",
       " 572: 'ate',\n",
       " 573: 'ate,',\n",
       " 574: 'ate.',\n",
       " 575: 'athlete',\n",
       " 576: 'athletes',\n",
       " 577: 'atmosphere',\n",
       " 578: 'atmosphere.',\n",
       " 579: 'atop',\n",
       " 580: 'attached',\n",
       " 581: 'attack',\n",
       " 582: 'attempt',\n",
       " 583: 'attend',\n",
       " 584: 'attend.',\n",
       " 585: 'attende',\n",
       " 586: 'attended',\n",
       " 587: 'attended!',\n",
       " 588: 'attended,',\n",
       " 589: 'attended.',\n",
       " 590: 'attendees',\n",
       " 591: 'attendees.',\n",
       " 592: 'attending',\n",
       " 593: 'attention',\n",
       " 594: 'attention.',\n",
       " 595: 'attire',\n",
       " 596: 'attract',\n",
       " 597: 'attraction',\n",
       " 598: 'attraction.',\n",
       " 599: 'attractions',\n",
       " 600: 'attractions!',\n",
       " 601: 'attractions,',\n",
       " 602: 'attractions.',\n",
       " 603: 'attractive',\n",
       " 604: 'atv.',\n",
       " 605: 'auction',\n",
       " 606: 'auction.',\n",
       " 607: 'auctions.',\n",
       " 608: 'audience',\n",
       " 609: 'audience.',\n",
       " 610: 'aunt',\n",
       " 611: 'aunt,',\n",
       " 612: 'aunts',\n",
       " 613: 'aura',\n",
       " 614: 'austerity',\n",
       " 615: 'authentic',\n",
       " 616: 'authentic,',\n",
       " 617: 'auto',\n",
       " 618: 'autographs.',\n",
       " 619: 'autumn',\n",
       " 620: 'available',\n",
       " 621: 'available.',\n",
       " 622: 'avenue',\n",
       " 623: 'avenue,',\n",
       " 624: 'avenue.',\n",
       " 625: 'avoid',\n",
       " 626: 'avoided',\n",
       " 627: 'awa',\n",
       " 628: 'await',\n",
       " 629: 'awaited',\n",
       " 630: 'awaiting',\n",
       " 631: 'awaits',\n",
       " 632: 'awake',\n",
       " 633: 'awakes',\n",
       " 634: 'award',\n",
       " 635: 'award.',\n",
       " 636: 'awarded.',\n",
       " 637: 'awards',\n",
       " 638: 'aware',\n",
       " 639: 'awareness',\n",
       " 640: 'away',\n",
       " 641: 'away!',\n",
       " 642: 'away,',\n",
       " 643: 'away.',\n",
       " 644: 'awe',\n",
       " 645: 'awe,',\n",
       " 646: 'awe.',\n",
       " 647: 'awed',\n",
       " 648: 'awesome',\n",
       " 649: 'awesome!',\n",
       " 650: 'awesome,',\n",
       " 651: 'awesome.',\n",
       " 652: 'awesomely',\n",
       " 653: 'awful',\n",
       " 654: 'awfully',\n",
       " 655: 'awhile',\n",
       " 656: 'awhile.',\n",
       " 657: 'awning',\n",
       " 658: 'awnings.',\n",
       " 659: 'awoke',\n",
       " 660: \"aww''.\",\n",
       " 661: 'aye',\n",
       " 662: 'ayesha',\n",
       " 663: 'ayesha,',\n",
       " 664: \"ayumi's\",\n",
       " 665: 'ayumi.',\n",
       " 666: 'b',\n",
       " 667: 'babe',\n",
       " 668: 'babies',\n",
       " 669: 'baby',\n",
       " 670: \"baby's\",\n",
       " 671: 'baby.',\n",
       " 672: 'babysitter.',\n",
       " 673: 'bac',\n",
       " 674: 'back',\n",
       " 675: 'back,',\n",
       " 676: 'back.',\n",
       " 677: 'backdro',\n",
       " 678: 'backdrop',\n",
       " 679: 'backdrop.',\n",
       " 680: 'background',\n",
       " 681: 'background,',\n",
       " 682: 'background.',\n",
       " 683: 'backgrounds.',\n",
       " 684: 'backing',\n",
       " 685: 'backpack',\n",
       " 686: 'backpack,',\n",
       " 687: 'backs!',\n",
       " 688: 'backwards.',\n",
       " 689: 'backyard',\n",
       " 690: 'backyard.',\n",
       " 691: 'bacon,',\n",
       " 692: 'bad',\n",
       " 693: 'bad!',\n",
       " 694: 'bad,',\n",
       " 695: 'bad.',\n",
       " 696: 'badly',\n",
       " 697: 'badly.',\n",
       " 698: 'bag',\n",
       " 699: 'bag,',\n",
       " 700: 'bagpipers.',\n",
       " 701: 'bagpipes',\n",
       " 702: 'bagpipes!',\n",
       " 703: 'bags',\n",
       " 704: 'bags.',\n",
       " 705: 'baily',\n",
       " 706: 'bake',\n",
       " 707: 'bakery',\n",
       " 708: 'bakery,',\n",
       " 709: 'bakery.',\n",
       " 710: 'baking',\n",
       " 711: 'baking.',\n",
       " 712: 'bal',\n",
       " 713: 'balance',\n",
       " 714: 'balconies.',\n",
       " 715: 'balcony',\n",
       " 716: 'balcony.',\n",
       " 717: 'ball',\n",
       " 718: 'ball.',\n",
       " 719: 'ballgame.',\n",
       " 720: 'balloon',\n",
       " 721: 'balloon.',\n",
       " 722: 'balloons',\n",
       " 723: 'balloons.',\n",
       " 724: 'balls',\n",
       " 725: 'balls.',\n",
       " 726: 'bamboo',\n",
       " 727: 'ban',\n",
       " 728: 'banana',\n",
       " 729: 'band',\n",
       " 730: 'band!',\n",
       " 731: 'band,',\n",
       " 732: 'band.',\n",
       " 733: 'bandmates.',\n",
       " 734: 'bands',\n",
       " 735: 'bands,',\n",
       " 736: 'bands.',\n",
       " 737: 'bang',\n",
       " 738: 'bang!',\n",
       " 739: 'bang.',\n",
       " 740: 'bangs,',\n",
       " 741: 'bank.',\n",
       " 742: 'banned',\n",
       " 743: 'banner',\n",
       " 744: 'banner!',\n",
       " 745: 'banner,',\n",
       " 746: 'banners',\n",
       " 747: 'banners.',\n",
       " 748: 'banquet',\n",
       " 749: 'baptism.',\n",
       " 750: 'bar',\n",
       " 751: 'bar.',\n",
       " 752: 'barb',\n",
       " 753: 'barbecue',\n",
       " 754: 'barbecue!',\n",
       " 755: 'barbecue.',\n",
       " 756: 'barbed',\n",
       " 757: 'barbeque',\n",
       " 758: 'barbeque!',\n",
       " 759: 'barbeque.',\n",
       " 760: 'barbershop',\n",
       " 761: 'barefoot',\n",
       " 762: 'barely',\n",
       " 763: 'barge',\n",
       " 764: 'barges',\n",
       " 765: 'barking',\n",
       " 766: 'barren',\n",
       " 767: 'barren,',\n",
       " 768: 'barricade.',\n",
       " 769: 'bars.',\n",
       " 770: 'base',\n",
       " 771: 'baseball',\n",
       " 772: 'baseball.',\n",
       " 773: 'based',\n",
       " 774: 'baseman',\n",
       " 775: 'bash',\n",
       " 776: 'bash!',\n",
       " 777: 'bashfu',\n",
       " 778: 'basics',\n",
       " 779: 'basket',\n",
       " 780: 'basketball',\n",
       " 781: 'basketball.',\n",
       " 782: 'baskets.',\n",
       " 783: 'bass',\n",
       " 784: 'bass.',\n",
       " 785: 'bat',\n",
       " 786: 'batch',\n",
       " 787: 'bath',\n",
       " 788: 'bath.',\n",
       " 789: 'bathed',\n",
       " 790: 'bathing.',\n",
       " 791: 'bathroom',\n",
       " 792: 'bathrooms.',\n",
       " 793: 'batman',\n",
       " 794: 'batman.',\n",
       " 795: 'batter',\n",
       " 796: 'battery',\n",
       " 797: 'battle',\n",
       " 798: 'battle,',\n",
       " 799: 'battle.',\n",
       " 800: 'battled',\n",
       " 801: 'battleship.',\n",
       " 802: 'bay',\n",
       " 803: 'bay.',\n",
       " 804: 'bazaar.',\n",
       " 805: 'bbq',\n",
       " 806: 'bbq.',\n",
       " 807: 'be',\n",
       " 808: 'be!',\n",
       " 809: 'be.',\n",
       " 810: 'beac',\n",
       " 811: 'beach',\n",
       " 812: 'beach!',\n",
       " 813: 'beach,',\n",
       " 814: 'beach.',\n",
       " 815: 'beaches',\n",
       " 816: 'beaches.',\n",
       " 817: 'beacon.',\n",
       " 818: 'beagle',\n",
       " 819: 'beam',\n",
       " 820: 'beaming',\n",
       " 821: 'bean',\n",
       " 822: 'beans',\n",
       " 823: 'bear',\n",
       " 824: 'bear!',\n",
       " 825: 'bear,',\n",
       " 826: 'bear.',\n",
       " 827: 'bearable.',\n",
       " 828: 'beard',\n",
       " 829: 'bearded',\n",
       " 830: 'bearer',\n",
       " 831: 'bearers',\n",
       " 832: 'bears',\n",
       " 833: 'bears.',\n",
       " 834: 'beast',\n",
       " 835: 'beat',\n",
       " 836: 'beaten',\n",
       " 837: 'beating',\n",
       " 838: 'beautifu',\n",
       " 839: 'beautiful',\n",
       " 840: 'beautiful!',\n",
       " 841: 'beautiful,',\n",
       " 842: 'beautiful.',\n",
       " 843: 'beautifully',\n",
       " 844: 'beautifully.',\n",
       " 845: 'beauty',\n",
       " 846: \"beauty's\",\n",
       " 847: 'beauty.',\n",
       " 848: 'beaver',\n",
       " 849: 'became',\n",
       " 850: 'because',\n",
       " 851: 'become',\n",
       " 852: 'becoming',\n",
       " 853: 'bed',\n",
       " 854: 'bed.',\n",
       " 855: 'bedroom',\n",
       " 856: 'beds',\n",
       " 857: 'beds.',\n",
       " 858: 'beef',\n",
       " 859: 'been',\n",
       " 860: 'been.',\n",
       " 861: 'beer',\n",
       " 862: 'beer!',\n",
       " 863: 'beer,',\n",
       " 864: 'beer.',\n",
       " 865: 'beers',\n",
       " 866: 'beers.',\n",
       " 867: 'bees',\n",
       " 868: 'befor',\n",
       " 869: 'before',\n",
       " 870: 'before!',\n",
       " 871: 'before.',\n",
       " 872: 'beg',\n",
       " 873: 'bega',\n",
       " 874: 'began',\n",
       " 875: 'began,',\n",
       " 876: 'began.',\n",
       " 877: 'begged',\n",
       " 878: 'begging',\n",
       " 879: 'begging,',\n",
       " 880: 'begi',\n",
       " 881: 'begin',\n",
       " 882: 'begin,',\n",
       " 883: 'begin.',\n",
       " 884: 'beginning',\n",
       " 885: 'beginning,',\n",
       " 886: 'beginning.',\n",
       " 887: 'beginnings.',\n",
       " 888: 'begins',\n",
       " 889: 'begins.',\n",
       " 890: 'begun!',\n",
       " 891: 'begun,',\n",
       " 892: 'begun.',\n",
       " 893: 'behaved',\n",
       " 894: 'behaved,',\n",
       " 895: 'behaved.',\n",
       " 896: 'behind',\n",
       " 897: 'behind!',\n",
       " 898: 'behind,',\n",
       " 899: 'behind.',\n",
       " 900: 'behold',\n",
       " 901: 'behold!',\n",
       " 902: 'behold.',\n",
       " 903: 'being',\n",
       " 904: 'being.',\n",
       " 905: 'belief,',\n",
       " 906: 'beliefs.',\n",
       " 907: 'believe',\n",
       " 908: 'believed',\n",
       " 909: 'believes',\n",
       " 910: 'bell',\n",
       " 911: 'bells',\n",
       " 912: 'belong.',\n",
       " 913: 'belonged',\n",
       " 914: 'below',\n",
       " 915: 'below.',\n",
       " 916: 'ben',\n",
       " 917: 'ben.',\n",
       " 918: 'bench',\n",
       " 919: 'bench,',\n",
       " 920: 'bench.',\n",
       " 921: 'benches',\n",
       " 922: 'bend',\n",
       " 923: 'beneath,',\n",
       " 924: 'berries',\n",
       " 925: 'berries.',\n",
       " 926: 'berry',\n",
       " 927: 'berry.',\n",
       " 928: 'beside',\n",
       " 929: 'besides',\n",
       " 930: 'besides,',\n",
       " 931: 'best',\n",
       " 932: 'best!',\n",
       " 933: 'best,',\n",
       " 934: 'best.',\n",
       " 935: 'bet',\n",
       " 936: 'better',\n",
       " 937: 'better.',\n",
       " 938: 'between',\n",
       " 939: 'between.',\n",
       " 940: 'beutifully!',\n",
       " 941: 'beverage',\n",
       " 942: 'beverages',\n",
       " 943: 'beverages.',\n",
       " 944: 'beyond',\n",
       " 945: 'beyond.',\n",
       " 946: 'bib.',\n",
       " 947: 'bible',\n",
       " 948: 'bicycle',\n",
       " 949: 'bicycle.',\n",
       " 950: 'bicycles',\n",
       " 951: 'big',\n",
       " 952: 'big,',\n",
       " 953: 'big.',\n",
       " 954: 'bigger',\n",
       " 955: 'bigger,',\n",
       " 956: 'bigger.',\n",
       " 957: 'biggest',\n",
       " 958: 'biggest.',\n",
       " 959: 'bigsmile',\n",
       " 960: 'bike',\n",
       " 961: 'bike,',\n",
       " 962: 'bike.',\n",
       " 963: 'biked',\n",
       " 964: 'biker',\n",
       " 965: 'bikers.',\n",
       " 966: 'bikes',\n",
       " 967: 'bikes.',\n",
       " 968: 'biking',\n",
       " 969: 'bill',\n",
       " 970: 'billa',\n",
       " 971: 'billboards',\n",
       " 972: 'billy',\n",
       " 973: 'bird',\n",
       " 974: 'bird.',\n",
       " 975: 'birds',\n",
       " 976: 'birds,',\n",
       " 977: 'birds.',\n",
       " 978: 'birth',\n",
       " 979: 'birthday',\n",
       " 980: 'birthday!',\n",
       " 981: \"birthday''\",\n",
       " 982: 'birthday,',\n",
       " 983: 'birthday.',\n",
       " 984: 'birthplace',\n",
       " 985: 'birthplace.',\n",
       " 986: 'biscuits.',\n",
       " 987: 'bit',\n",
       " 988: 'bit,',\n",
       " 989: 'bit.',\n",
       " 990: 'bite',\n",
       " 991: 'bite.',\n",
       " 992: 'black',\n",
       " 993: 'black.',\n",
       " 994: 'blackberries',\n",
       " 995: 'blackness.',\n",
       " 996: 'blanket',\n",
       " 997: 'blanketed',\n",
       " 998: 'blankets',\n",
       " 999: 'blankets!',\n",
       " ...}"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reverse_target_char_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "72"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_token_index['START_']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function to generate sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "    # Generate empty target sequence of length 1.\n",
    "    target_seq = np.zeros((1,1))\n",
    "    #print (\"target_seq: \", target_seq)\n",
    "    # Populate the first character of target sequence with the start character.\n",
    "    target_seq[0, 0] = target_token_index['START_']\n",
    "\n",
    "    # Sampling loop for a batch of sequences\n",
    "    # (to simplify, here we assume a batch of size 1).\n",
    "    stop_condition = False\n",
    "    decoded_sentence = ''\n",
    "    while not stop_condition:\n",
    "        output_tokens, h, c = decoder_model.predict(\n",
    "            [target_seq] + states_value)\n",
    "\n",
    "        # Sample a token\n",
    "        #print (\"output_tokens: \", output_tokens)\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        #print (\"sampled_token_index: \", sampled_token_index)\n",
    "        #print (\"\")\n",
    "        sampled_char = reverse_target_char_index[sampled_token_index]\n",
    "        decoded_sentence += ' ' + sampled_char\n",
    "\n",
    "        # Exit condition: either hit max length\n",
    "        # or find stop character.\n",
    "        if (sampled_char == '_END' or\n",
    "           len(decoded_sentence) > 150): # 52\n",
    "            # print (\"Stop_condition = TRUE\")\n",
    "            stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1,1))\n",
    "        target_seq[0, 0] = sampled_token_index\n",
    "\n",
    "        # Update states\n",
    "        states_value = [h, c]\n",
    "\n",
    "    return decoded_sentence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# load\n",
    "test_input3 = ['an elderly woman hugging a young girl surrounded by other people.',\n",
    "                 'couple taking a picture during a fun night in a bar.',\n",
    "                 'a man is standing next to a sign for the pro football hall of fam.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description_text</th>\n",
       "      <th>story_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>big old tree being photographed on a sunny da</td>\n",
       "      <td>START_ and its magnificent trunk, larger than ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a old curvy tree in the sun light.</td>\n",
       "      <td>START_ and its magnificent trunk, larger than ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a person is taking a picture of a large tree a...</td>\n",
       "      <td>START_ and its magnificent trunk, larger than ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>large tree with many outstretching branches an...</td>\n",
       "      <td>START_ we found this tree when we were walking...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a green sign is describing a historic tree and...</td>\n",
       "      <td>START_ it turns out it is a popular attraction...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a large tree with roots that look like crocodi...</td>\n",
       "      <td>START_ the tree is very unusual, with its root...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>big old tree being photographed on a sunny da</td>\n",
       "      <td>START_ the trunk was really wide, as much as 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>huge brown tree roots rose above the ground.</td>\n",
       "      <td>START_ you can see how big these roots are - p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>a large tree with many branches coming ou</td>\n",
       "      <td>START_ we found this tree when we were walking...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>a plaque describes an historical tree and advi...</td>\n",
       "      <td>START_ it turns out it is a popular attraction...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>a tree has spread its roots high and wide acro...</td>\n",
       "      <td>START_ the tree is very unusual, with its root...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>a old curvy tree in the sun light.</td>\n",
       "      <td>START_ the trunk was really wide, as much as 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>some trees that are surrounded by apartment bu...</td>\n",
       "      <td>START_ you can see how big these roots are - p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>the tree has very long and dated branches.</td>\n",
       "      <td>START_ we found this tree when we were walking...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>a plaque on a stand surround by died leaves of...</td>\n",
       "      <td>START_ it turns out it is a popular attraction...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>a huge tree sits outside with several large ro...</td>\n",
       "      <td>START_ the tree is very unusual, with its root...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>a person is taking a picture of a large tree a...</td>\n",
       "      <td>START_ the trunk was really wide, as much as 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>various parts of the tree are growing out from...</td>\n",
       "      <td>START_ you can see how big these roots are - p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>big old tree being photographed on a sunny da</td>\n",
       "      <td>START_ some more different parts of the tree. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>a old curvy tree in the sun light.</td>\n",
       "      <td>START_ some more different parts of the tree. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>a person is taking a picture of a large tree a...</td>\n",
       "      <td>START_ some more different parts of the tree. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>large tree with many outstretching branches an...</td>\n",
       "      <td>START_ they went to the botanic gardens specif...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>a green sign is describing a historic tree and...</td>\n",
       "      <td>START_ there was an informational sign posted ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>a large tree with roots that look like crocodi...</td>\n",
       "      <td>START_ the roots were huge and spread out over...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>big old tree being photographed on a sunny da</td>\n",
       "      <td>START_ the trunk was incredibly thick and rigi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>huge brown tree roots rose above the ground.</td>\n",
       "      <td>START_ the large roots were almost as thick as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>a large tree with many branches coming ou</td>\n",
       "      <td>START_ they went to the botanic gardens specif...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>a plaque describes an historical tree and advi...</td>\n",
       "      <td>START_ there was an informational sign posted ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>a tree has spread its roots high and wide acro...</td>\n",
       "      <td>START_ the roots were huge and spread out over...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>a old curvy tree in the sun light.</td>\n",
       "      <td>START_ the trunk was incredibly thick and rigi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29970</th>\n",
       "      <td>winnings are displayed as ribbons and a sign o...</td>\n",
       "      <td>START_ the fair was in town. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29971</th>\n",
       "      <td>a little girl jumps excitedly as she pets a bl...</td>\n",
       "      <td>START_ [female] was so excited to see animals....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29972</th>\n",
       "      <td>sheep heavy with wool gather inside a pen at t...</td>\n",
       "      <td>START_ sheep were there, but better yet there ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29973</th>\n",
       "      <td>the llama is waiting for someone to come pet him.</td>\n",
       "      <td>START_ alpacas! the kids loved them. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29974</th>\n",
       "      <td>a sign on a wood fence describes alpacas.</td>\n",
       "      <td>START_ and after they learned all about them. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29975</th>\n",
       "      <td>a place of business displaying their 4 award</td>\n",
       "      <td>START_ the fair was in town. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29976</th>\n",
       "      <td>a small child pets a big animal in a pen.</td>\n",
       "      <td>START_ [female] was so excited to see animals....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29977</th>\n",
       "      <td>a small group sheeps in the ground with leg</td>\n",
       "      <td>START_ sheep were there, but better yet there ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29978</th>\n",
       "      <td>a little brown llama inside of a barn.</td>\n",
       "      <td>START_ alpacas! the kids loved them. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29979</th>\n",
       "      <td>signage on wooden fence presenting information...</td>\n",
       "      <td>START_ and after they learned all about them. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29980</th>\n",
       "      <td>there are ribbons on the horses cage, symboliz...</td>\n",
       "      <td>START_ the fair was in town. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29981</th>\n",
       "      <td>a young child is inside a pen with a black ani...</td>\n",
       "      <td>START_ [female] was so excited to see animals....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29982</th>\n",
       "      <td>sheep standing in a display pen at a county fair.</td>\n",
       "      <td>START_ sheep were there, but better yet there ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29983</th>\n",
       "      <td>a brown llama is enclosed in a pen covered in ...</td>\n",
       "      <td>START_ alpacas! the kids loved them. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29984</th>\n",
       "      <td>a sign that gives information to the people in...</td>\n",
       "      <td>START_ and after they learned all about them. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29985</th>\n",
       "      <td>a trip over the summer vacation, it was great.</td>\n",
       "      <td>START_ we went to organization last summer for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29986</th>\n",
       "      <td>father, son and mother sit on boat as it trave...</td>\n",
       "      <td>START_ we got to ride so many different rides....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29987</th>\n",
       "      <td>the children pondered as they looked on the sc...</td>\n",
       "      <td>START_ the kids were having so much fun. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29988</th>\n",
       "      <td>three boys are dressed up as pirates in the [o...</td>\n",
       "      <td>START_ they loved going around to all the diff...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29989</th>\n",
       "      <td>[organization] [organization] statue with mick...</td>\n",
       "      <td>START_ we had such a good time and can't wait ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29990</th>\n",
       "      <td>tourists are on the courtyard between banner d...</td>\n",
       "      <td>START_ we went to organization last summer for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29991</th>\n",
       "      <td>she was more excited than anyone to see the mo...</td>\n",
       "      <td>START_ we got to ride so many different rides....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29992</th>\n",
       "      <td>three young boys sitting on the stairs in a park.</td>\n",
       "      <td>START_ the kids were having so much fun. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29993</th>\n",
       "      <td>three boys displaying their best pirate faces ...</td>\n",
       "      <td>START_ they loved going around to all the diff...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29994</th>\n",
       "      <td>statue of a man and a mouse in front of a castle.</td>\n",
       "      <td>START_ we had such a good time and can't wait ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>a crowd of people in a village square. three o...</td>\n",
       "      <td>START_ we went to organization last summer for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>people on a safari truck watching as they expl...</td>\n",
       "      <td>START_ we got to ride so many different rides....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>children pose for a photograph on steps in a p...</td>\n",
       "      <td>START_ the kids were having so much fun. _END</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>kids wearing pirate hats are brandishing toy s...</td>\n",
       "      <td>START_ they loved going around to all the diff...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>bronze statue outdoors in foreground, large ca...</td>\n",
       "      <td>START_ we had such a good time and can't wait ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        description_text  \\\n",
       "0          big old tree being photographed on a sunny da   \n",
       "1                     a old curvy tree in the sun light.   \n",
       "2      a person is taking a picture of a large tree a...   \n",
       "3      large tree with many outstretching branches an...   \n",
       "4      a green sign is describing a historic tree and...   \n",
       "5      a large tree with roots that look like crocodi...   \n",
       "6          big old tree being photographed on a sunny da   \n",
       "7           huge brown tree roots rose above the ground.   \n",
       "8              a large tree with many branches coming ou   \n",
       "9      a plaque describes an historical tree and advi...   \n",
       "10     a tree has spread its roots high and wide acro...   \n",
       "11                    a old curvy tree in the sun light.   \n",
       "12     some trees that are surrounded by apartment bu...   \n",
       "13            the tree has very long and dated branches.   \n",
       "14     a plaque on a stand surround by died leaves of...   \n",
       "15     a huge tree sits outside with several large ro...   \n",
       "16     a person is taking a picture of a large tree a...   \n",
       "17     various parts of the tree are growing out from...   \n",
       "18         big old tree being photographed on a sunny da   \n",
       "19                    a old curvy tree in the sun light.   \n",
       "20     a person is taking a picture of a large tree a...   \n",
       "21     large tree with many outstretching branches an...   \n",
       "22     a green sign is describing a historic tree and...   \n",
       "23     a large tree with roots that look like crocodi...   \n",
       "24         big old tree being photographed on a sunny da   \n",
       "25          huge brown tree roots rose above the ground.   \n",
       "26             a large tree with many branches coming ou   \n",
       "27     a plaque describes an historical tree and advi...   \n",
       "28     a tree has spread its roots high and wide acro...   \n",
       "29                    a old curvy tree in the sun light.   \n",
       "...                                                  ...   \n",
       "29970  winnings are displayed as ribbons and a sign o...   \n",
       "29971  a little girl jumps excitedly as she pets a bl...   \n",
       "29972  sheep heavy with wool gather inside a pen at t...   \n",
       "29973  the llama is waiting for someone to come pet him.   \n",
       "29974          a sign on a wood fence describes alpacas.   \n",
       "29975       a place of business displaying their 4 award   \n",
       "29976          a small child pets a big animal in a pen.   \n",
       "29977        a small group sheeps in the ground with leg   \n",
       "29978             a little brown llama inside of a barn.   \n",
       "29979  signage on wooden fence presenting information...   \n",
       "29980  there are ribbons on the horses cage, symboliz...   \n",
       "29981  a young child is inside a pen with a black ani...   \n",
       "29982  sheep standing in a display pen at a county fair.   \n",
       "29983  a brown llama is enclosed in a pen covered in ...   \n",
       "29984  a sign that gives information to the people in...   \n",
       "29985     a trip over the summer vacation, it was great.   \n",
       "29986  father, son and mother sit on boat as it trave...   \n",
       "29987  the children pondered as they looked on the sc...   \n",
       "29988  three boys are dressed up as pirates in the [o...   \n",
       "29989  [organization] [organization] statue with mick...   \n",
       "29990  tourists are on the courtyard between banner d...   \n",
       "29991  she was more excited than anyone to see the mo...   \n",
       "29992  three young boys sitting on the stairs in a park.   \n",
       "29993  three boys displaying their best pirate faces ...   \n",
       "29994  statue of a man and a mouse in front of a castle.   \n",
       "29995  a crowd of people in a village square. three o...   \n",
       "29996  people on a safari truck watching as they expl...   \n",
       "29997  children pose for a photograph on steps in a p...   \n",
       "29998  kids wearing pirate hats are brandishing toy s...   \n",
       "29999  bronze statue outdoors in foreground, large ca...   \n",
       "\n",
       "                                              story_text  \n",
       "0      START_ and its magnificent trunk, larger than ...  \n",
       "1      START_ and its magnificent trunk, larger than ...  \n",
       "2      START_ and its magnificent trunk, larger than ...  \n",
       "3      START_ we found this tree when we were walking...  \n",
       "4      START_ it turns out it is a popular attraction...  \n",
       "5      START_ the tree is very unusual, with its root...  \n",
       "6      START_ the trunk was really wide, as much as 1...  \n",
       "7      START_ you can see how big these roots are - p...  \n",
       "8      START_ we found this tree when we were walking...  \n",
       "9      START_ it turns out it is a popular attraction...  \n",
       "10     START_ the tree is very unusual, with its root...  \n",
       "11     START_ the trunk was really wide, as much as 1...  \n",
       "12     START_ you can see how big these roots are - p...  \n",
       "13     START_ we found this tree when we were walking...  \n",
       "14     START_ it turns out it is a popular attraction...  \n",
       "15     START_ the tree is very unusual, with its root...  \n",
       "16     START_ the trunk was really wide, as much as 1...  \n",
       "17     START_ you can see how big these roots are - p...  \n",
       "18     START_ some more different parts of the tree. ...  \n",
       "19     START_ some more different parts of the tree. ...  \n",
       "20     START_ some more different parts of the tree. ...  \n",
       "21     START_ they went to the botanic gardens specif...  \n",
       "22     START_ there was an informational sign posted ...  \n",
       "23     START_ the roots were huge and spread out over...  \n",
       "24     START_ the trunk was incredibly thick and rigi...  \n",
       "25     START_ the large roots were almost as thick as...  \n",
       "26     START_ they went to the botanic gardens specif...  \n",
       "27     START_ there was an informational sign posted ...  \n",
       "28     START_ the roots were huge and spread out over...  \n",
       "29     START_ the trunk was incredibly thick and rigi...  \n",
       "...                                                  ...  \n",
       "29970                  START_ the fair was in town. _END  \n",
       "29971  START_ [female] was so excited to see animals....  \n",
       "29972  START_ sheep were there, but better yet there ...  \n",
       "29973          START_ alpacas! the kids loved them. _END  \n",
       "29974  START_ and after they learned all about them. ...  \n",
       "29975                  START_ the fair was in town. _END  \n",
       "29976  START_ [female] was so excited to see animals....  \n",
       "29977  START_ sheep were there, but better yet there ...  \n",
       "29978          START_ alpacas! the kids loved them. _END  \n",
       "29979  START_ and after they learned all about them. ...  \n",
       "29980                  START_ the fair was in town. _END  \n",
       "29981  START_ [female] was so excited to see animals....  \n",
       "29982  START_ sheep were there, but better yet there ...  \n",
       "29983          START_ alpacas! the kids loved them. _END  \n",
       "29984  START_ and after they learned all about them. ...  \n",
       "29985  START_ we went to organization last summer for...  \n",
       "29986  START_ we got to ride so many different rides....  \n",
       "29987      START_ the kids were having so much fun. _END  \n",
       "29988  START_ they loved going around to all the diff...  \n",
       "29989  START_ we had such a good time and can't wait ...  \n",
       "29990  START_ we went to organization last summer for...  \n",
       "29991  START_ we got to ride so many different rides....  \n",
       "29992      START_ the kids were having so much fun. _END  \n",
       "29993  START_ they loved going around to all the diff...  \n",
       "29994  START_ we had such a good time and can't wait ...  \n",
       "29995  START_ we went to organization last summer for...  \n",
       "29996  START_ we got to ride so many different rides....  \n",
       "29997      START_ the kids were having so much fun. _END  \n",
       "29998  START_ they loved going around to all the diff...  \n",
       "29999  START_ we had such a good time and can't wait ...  \n",
       "\n",
       "[30000 rows x 2 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000    red and blue dominate the colorful archways in...\n",
       "25001       large colorful building lit up in the evening.\n",
       "25002      a projection of an image made out of binary cod\n",
       "25003    work of modern art being displayed and lit in ...\n",
       "25004    adult man sitting on commode using laptop in w...\n",
       "25005    a view of the building inside of a well-lit sc...\n",
       "25006    a nighttime photograph of people outside of a ...\n",
       "25007    artistic mural projections in green rendering ...\n",
       "25008      a sculpture made of metal on display at a museu\n",
       "25009    man sits on commode in water closet while work...\n",
       "25010    the inside of the church is illuminated with d...\n",
       "25011    a group of spectators is watching a a building...\n",
       "25012    a red wall with a green digital display showin...\n",
       "25013    an object that has been crafted by a creative ...\n",
       "25014    a man sits on a toilet while using a laptop co...\n",
       "25015    red and blue dominate the colorful archways in...\n",
       "25016       large colorful building lit up in the evening.\n",
       "25017      a projection of an image made out of binary cod\n",
       "25018    work of modern art being displayed and lit in ...\n",
       "25019    adult man sitting on commode using laptop in w...\n",
       "25020    a view of the building inside of a well-lit sc...\n",
       "25021    a nighttime photograph of people outside of a ...\n",
       "25022    artistic mural projections in green rendering ...\n",
       "25023      a sculpture made of metal on display at a museu\n",
       "25024    man sits on commode in water closet while work...\n",
       "25025    the inside of the church is illuminated with d...\n",
       "25026    a group of spectators is watching a a building...\n",
       "25027    a red wall with a green digital display showin...\n",
       "25028    an object that has been crafted by a creative ...\n",
       "25029    a man sits on a toilet while using a laptop co...\n",
       "                               ...                        \n",
       "29970    winnings are displayed as ribbons and a sign o...\n",
       "29971    a little girl jumps excitedly as she pets a bl...\n",
       "29972    sheep heavy with wool gather inside a pen at t...\n",
       "29973    the llama is waiting for someone to come pet him.\n",
       "29974            a sign on a wood fence describes alpacas.\n",
       "29975         a place of business displaying their 4 award\n",
       "29976            a small child pets a big animal in a pen.\n",
       "29977          a small group sheeps in the ground with leg\n",
       "29978               a little brown llama inside of a barn.\n",
       "29979    signage on wooden fence presenting information...\n",
       "29980    there are ribbons on the horses cage, symboliz...\n",
       "29981    a young child is inside a pen with a black ani...\n",
       "29982    sheep standing in a display pen at a county fair.\n",
       "29983    a brown llama is enclosed in a pen covered in ...\n",
       "29984    a sign that gives information to the people in...\n",
       "29985       a trip over the summer vacation, it was great.\n",
       "29986    father, son and mother sit on boat as it trave...\n",
       "29987    the children pondered as they looked on the sc...\n",
       "29988    three boys are dressed up as pirates in the [o...\n",
       "29989    [organization] [organization] statue with mick...\n",
       "29990    tourists are on the courtyard between banner d...\n",
       "29991    she was more excited than anyone to see the mo...\n",
       "29992    three young boys sitting on the stairs in a park.\n",
       "29993    three boys displaying their best pirate faces ...\n",
       "29994    statue of a man and a mouse in front of a castle.\n",
       "29995    a crowd of people in a village square. three o...\n",
       "29996    people on a safari truck watching as they expl...\n",
       "29997    children pose for a photograph on steps in a p...\n",
       "29998    kids wearing pirate hats are brandishing toy s...\n",
       "29999    bronze statue outdoors in foreground, large ca...\n",
       "Name: description_text, Length: 5000, dtype: object"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_input5000_0 = clean_df_all.description_text[0:5000]\n",
    "df_input5000_1 = clean_df_all.description_text[5000:10000]\n",
    "df_input5000_2 = clean_df_all.description_text[10000:15000]\n",
    "df_input5000_3 = clean_df_all.description_text[15000:20000]\n",
    "df_input5000_4 = clean_df_all.description_text[20000:25000]\n",
    "df_input5000_5 = clean_df_all.description_text[25000:30000]\n",
    "df_input5000_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        2626983575\n",
       "1        2626983575\n",
       "2        2626983575\n",
       "3        2701863545\n",
       "4        2626977325\n",
       "5        2627795780\n",
       "6        2626983575\n",
       "7        2626982337\n",
       "8        2701863545\n",
       "9        2626977325\n",
       "10       2627795780\n",
       "11       2626983575\n",
       "12       2626982337\n",
       "13       2701863545\n",
       "14       2626977325\n",
       "15       2627795780\n",
       "16       2626983575\n",
       "17       2626982337\n",
       "18       2626983575\n",
       "19       2626983575\n",
       "20       2626983575\n",
       "21       2701863545\n",
       "22       2626977325\n",
       "23       2627795780\n",
       "24       2626983575\n",
       "25       2626982337\n",
       "26       2701863545\n",
       "27       2626977325\n",
       "28       2627795780\n",
       "29       2626983575\n",
       "            ...    \n",
       "29970    4794142562\n",
       "29971    4794143044\n",
       "29972    4793510447\n",
       "29973    4793511023\n",
       "29974    4793511455\n",
       "29975    4794142562\n",
       "29976    4794143044\n",
       "29977    4793510447\n",
       "29978    4793511023\n",
       "29979    4793511455\n",
       "29980    4794142562\n",
       "29981    4794143044\n",
       "29982    4793510447\n",
       "29983    4793511023\n",
       "29984    4793511455\n",
       "29985    4794177464\n",
       "29986    4793545161\n",
       "29987    4794180770\n",
       "29988    4793552387\n",
       "29989    4794196042\n",
       "29990    4794177464\n",
       "29991    4793545161\n",
       "29992    4794180770\n",
       "29993    4793552387\n",
       "29994    4794196042\n",
       "29995    4794177464\n",
       "29996    4793545161\n",
       "29997    4794180770\n",
       "29998    4793552387\n",
       "29999    4794196042\n",
       "Name: Story_Photo_id, Length: 30000, dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_story_and_desc_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[25000,\n",
       " 25001,\n",
       " 25002,\n",
       " 25003,\n",
       " 25004,\n",
       " 25005,\n",
       " 25006,\n",
       " 25007,\n",
       " 25008,\n",
       " 25009,\n",
       " 25010,\n",
       " 25011,\n",
       " 25012,\n",
       " 25013,\n",
       " 25014,\n",
       " 25015,\n",
       " 25016,\n",
       " 25017,\n",
       " 25018,\n",
       " 25019,\n",
       " 25020,\n",
       " 25021,\n",
       " 25022,\n",
       " 25023,\n",
       " 25024,\n",
       " 25025,\n",
       " 25026,\n",
       " 25027,\n",
       " 25028,\n",
       " 25029,\n",
       " 25030,\n",
       " 25031,\n",
       " 25032,\n",
       " 25033,\n",
       " 25034,\n",
       " 25035,\n",
       " 25036,\n",
       " 25037,\n",
       " 25038,\n",
       " 25039,\n",
       " 25040,\n",
       " 25041,\n",
       " 25042,\n",
       " 25043,\n",
       " 25044,\n",
       " 25045,\n",
       " 25046,\n",
       " 25047,\n",
       " 25048,\n",
       " 25049,\n",
       " 25050,\n",
       " 25051,\n",
       " 25052,\n",
       " 25053,\n",
       " 25054,\n",
       " 25055,\n",
       " 25056,\n",
       " 25057,\n",
       " 25058,\n",
       " 25059,\n",
       " 25060,\n",
       " 25061,\n",
       " 25062,\n",
       " 25063,\n",
       " 25064,\n",
       " 25065,\n",
       " 25066,\n",
       " 25067,\n",
       " 25068,\n",
       " 25069,\n",
       " 25070,\n",
       " 25071,\n",
       " 25072,\n",
       " 25073,\n",
       " 25074,\n",
       " 25075,\n",
       " 25076,\n",
       " 25077,\n",
       " 25078,\n",
       " 25079,\n",
       " 25080,\n",
       " 25081,\n",
       " 25082,\n",
       " 25083,\n",
       " 25084,\n",
       " 25085,\n",
       " 25086,\n",
       " 25087,\n",
       " 25088,\n",
       " 25089,\n",
       " 25090,\n",
       " 25091,\n",
       " 25092,\n",
       " 25093,\n",
       " 25094,\n",
       " 25095,\n",
       " 25096,\n",
       " 25097,\n",
       " 25098,\n",
       " 25099,\n",
       " 25100,\n",
       " 25101,\n",
       " 25102,\n",
       " 25103,\n",
       " 25104,\n",
       " 25105,\n",
       " 25106,\n",
       " 25107,\n",
       " 25108,\n",
       " 25109,\n",
       " 25110,\n",
       " 25111,\n",
       " 25112,\n",
       " 25113,\n",
       " 25114,\n",
       " 25115,\n",
       " 25116,\n",
       " 25117,\n",
       " 25118,\n",
       " 25119,\n",
       " 25120,\n",
       " 25121,\n",
       " 25122,\n",
       " 25123,\n",
       " 25124,\n",
       " 25125,\n",
       " 25126,\n",
       " 25127,\n",
       " 25128,\n",
       " 25129,\n",
       " 25130,\n",
       " 25131,\n",
       " 25132,\n",
       " 25133,\n",
       " 25134,\n",
       " 25135,\n",
       " 25136,\n",
       " 25137,\n",
       " 25138,\n",
       " 25139,\n",
       " 25140,\n",
       " 25141,\n",
       " 25142,\n",
       " 25143,\n",
       " 25144,\n",
       " 25145,\n",
       " 25146,\n",
       " 25147,\n",
       " 25148,\n",
       " 25149,\n",
       " 25150,\n",
       " 25151,\n",
       " 25152,\n",
       " 25153,\n",
       " 25154,\n",
       " 25155,\n",
       " 25156,\n",
       " 25157,\n",
       " 25158,\n",
       " 25159,\n",
       " 25160,\n",
       " 25161,\n",
       " 25162,\n",
       " 25163,\n",
       " 25164,\n",
       " 25165,\n",
       " 25166,\n",
       " 25167,\n",
       " 25168,\n",
       " 25169,\n",
       " 25170,\n",
       " 25171,\n",
       " 25172,\n",
       " 25173,\n",
       " 25174,\n",
       " 25175,\n",
       " 25176,\n",
       " 25177,\n",
       " 25178,\n",
       " 25179,\n",
       " 25180,\n",
       " 25181,\n",
       " 25182,\n",
       " 25183,\n",
       " 25184,\n",
       " 25185,\n",
       " 25186,\n",
       " 25187,\n",
       " 25188,\n",
       " 25189,\n",
       " 25190,\n",
       " 25191,\n",
       " 25192,\n",
       " 25193,\n",
       " 25194,\n",
       " 25195,\n",
       " 25196,\n",
       " 25197,\n",
       " 25198,\n",
       " 25199,\n",
       " 25200,\n",
       " 25201,\n",
       " 25202,\n",
       " 25203,\n",
       " 25204,\n",
       " 25205,\n",
       " 25206,\n",
       " 25207,\n",
       " 25208,\n",
       " 25209,\n",
       " 25210,\n",
       " 25211,\n",
       " 25212,\n",
       " 25213,\n",
       " 25214,\n",
       " 25215,\n",
       " 25216,\n",
       " 25217,\n",
       " 25218,\n",
       " 25219,\n",
       " 25220,\n",
       " 25221,\n",
       " 25222,\n",
       " 25223,\n",
       " 25224,\n",
       " 25225,\n",
       " 25226,\n",
       " 25227,\n",
       " 25228,\n",
       " 25229,\n",
       " 25230,\n",
       " 25231,\n",
       " 25232,\n",
       " 25233,\n",
       " 25234,\n",
       " 25235,\n",
       " 25236,\n",
       " 25237,\n",
       " 25238,\n",
       " 25239,\n",
       " 25240,\n",
       " 25241,\n",
       " 25242,\n",
       " 25243,\n",
       " 25244,\n",
       " 25245,\n",
       " 25246,\n",
       " 25247,\n",
       " 25248,\n",
       " 25249,\n",
       " 25250,\n",
       " 25251,\n",
       " 25252,\n",
       " 25253,\n",
       " 25254,\n",
       " 25255,\n",
       " 25256,\n",
       " 25257,\n",
       " 25258,\n",
       " 25259,\n",
       " 25260,\n",
       " 25261,\n",
       " 25262,\n",
       " 25263,\n",
       " 25264,\n",
       " 25265,\n",
       " 25266,\n",
       " 25267,\n",
       " 25268,\n",
       " 25269,\n",
       " 25270,\n",
       " 25271,\n",
       " 25272,\n",
       " 25273,\n",
       " 25274,\n",
       " 25275,\n",
       " 25276,\n",
       " 25277,\n",
       " 25278,\n",
       " 25279,\n",
       " 25280,\n",
       " 25281,\n",
       " 25282,\n",
       " 25283,\n",
       " 25284,\n",
       " 25285,\n",
       " 25286,\n",
       " 25287,\n",
       " 25288,\n",
       " 25289,\n",
       " 25290,\n",
       " 25291,\n",
       " 25292,\n",
       " 25293,\n",
       " 25294,\n",
       " 25295,\n",
       " 25296,\n",
       " 25297,\n",
       " 25298,\n",
       " 25299,\n",
       " 25300,\n",
       " 25301,\n",
       " 25302,\n",
       " 25303,\n",
       " 25304,\n",
       " 25305,\n",
       " 25306,\n",
       " 25307,\n",
       " 25308,\n",
       " 25309,\n",
       " 25310,\n",
       " 25311,\n",
       " 25312,\n",
       " 25313,\n",
       " 25314,\n",
       " 25315,\n",
       " 25316,\n",
       " 25317,\n",
       " 25318,\n",
       " 25319,\n",
       " 25320,\n",
       " 25321,\n",
       " 25322,\n",
       " 25323,\n",
       " 25324,\n",
       " 25325,\n",
       " 25326,\n",
       " 25327,\n",
       " 25328,\n",
       " 25329,\n",
       " 25330,\n",
       " 25331,\n",
       " 25332,\n",
       " 25333,\n",
       " 25334,\n",
       " 25335,\n",
       " 25336,\n",
       " 25337,\n",
       " 25338,\n",
       " 25339,\n",
       " 25340,\n",
       " 25341,\n",
       " 25342,\n",
       " 25343,\n",
       " 25344,\n",
       " 25345,\n",
       " 25346,\n",
       " 25347,\n",
       " 25348,\n",
       " 25349,\n",
       " 25350,\n",
       " 25351,\n",
       " 25352,\n",
       " 25353,\n",
       " 25354,\n",
       " 25355,\n",
       " 25356,\n",
       " 25357,\n",
       " 25358,\n",
       " 25359,\n",
       " 25360,\n",
       " 25361,\n",
       " 25362,\n",
       " 25363,\n",
       " 25364,\n",
       " 25365,\n",
       " 25366,\n",
       " 25367,\n",
       " 25368,\n",
       " 25369,\n",
       " 25370,\n",
       " 25371,\n",
       " 25372,\n",
       " 25373,\n",
       " 25374,\n",
       " 25375,\n",
       " 25376,\n",
       " 25377,\n",
       " 25378,\n",
       " 25379,\n",
       " 25380,\n",
       " 25381,\n",
       " 25382,\n",
       " 25383,\n",
       " 25384,\n",
       " 25385,\n",
       " 25386,\n",
       " 25387,\n",
       " 25388,\n",
       " 25389,\n",
       " 25390,\n",
       " 25391,\n",
       " 25392,\n",
       " 25393,\n",
       " 25394,\n",
       " 25395,\n",
       " 25396,\n",
       " 25397,\n",
       " 25398,\n",
       " 25399,\n",
       " 25400,\n",
       " 25401,\n",
       " 25402,\n",
       " 25403,\n",
       " 25404,\n",
       " 25405,\n",
       " 25406,\n",
       " 25407,\n",
       " 25408,\n",
       " 25409,\n",
       " 25410,\n",
       " 25411,\n",
       " 25412,\n",
       " 25413,\n",
       " 25414,\n",
       " 25415,\n",
       " 25416,\n",
       " 25417,\n",
       " 25418,\n",
       " 25419,\n",
       " 25420,\n",
       " 25421,\n",
       " 25422,\n",
       " 25423,\n",
       " 25424,\n",
       " 25425,\n",
       " 25426,\n",
       " 25427,\n",
       " 25428,\n",
       " 25429,\n",
       " 25430,\n",
       " 25431,\n",
       " 25432,\n",
       " 25433,\n",
       " 25434,\n",
       " 25435,\n",
       " 25436,\n",
       " 25437,\n",
       " 25438,\n",
       " 25439,\n",
       " 25440,\n",
       " 25441,\n",
       " 25442,\n",
       " 25443,\n",
       " 25444,\n",
       " 25445,\n",
       " 25446,\n",
       " 25447,\n",
       " 25448,\n",
       " 25449,\n",
       " 25450,\n",
       " 25451,\n",
       " 25452,\n",
       " 25453,\n",
       " 25454,\n",
       " 25455,\n",
       " 25456,\n",
       " 25457,\n",
       " 25458,\n",
       " 25459,\n",
       " 25460,\n",
       " 25461,\n",
       " 25462,\n",
       " 25463,\n",
       " 25464,\n",
       " 25465,\n",
       " 25466,\n",
       " 25467,\n",
       " 25468,\n",
       " 25469,\n",
       " 25470,\n",
       " 25471,\n",
       " 25472,\n",
       " 25473,\n",
       " 25474,\n",
       " 25475,\n",
       " 25476,\n",
       " 25477,\n",
       " 25478,\n",
       " 25479,\n",
       " 25480,\n",
       " 25481,\n",
       " 25482,\n",
       " 25483,\n",
       " 25484,\n",
       " 25485,\n",
       " 25486,\n",
       " 25487,\n",
       " 25488,\n",
       " 25489,\n",
       " 25490,\n",
       " 25491,\n",
       " 25492,\n",
       " 25493,\n",
       " 25494,\n",
       " 25495,\n",
       " 25496,\n",
       " 25497,\n",
       " 25498,\n",
       " 25499,\n",
       " 25500,\n",
       " 25501,\n",
       " 25502,\n",
       " 25503,\n",
       " 25504,\n",
       " 25505,\n",
       " 25506,\n",
       " 25507,\n",
       " 25508,\n",
       " 25509,\n",
       " 25510,\n",
       " 25511,\n",
       " 25512,\n",
       " 25513,\n",
       " 25514,\n",
       " 25515,\n",
       " 25516,\n",
       " 25517,\n",
       " 25518,\n",
       " 25519,\n",
       " 25520,\n",
       " 25521,\n",
       " 25522,\n",
       " 25523,\n",
       " 25524,\n",
       " 25525,\n",
       " 25526,\n",
       " 25527,\n",
       " 25528,\n",
       " 25529,\n",
       " 25530,\n",
       " 25531,\n",
       " 25532,\n",
       " 25533,\n",
       " 25534,\n",
       " 25535,\n",
       " 25536,\n",
       " 25537,\n",
       " 25538,\n",
       " 25539,\n",
       " 25540,\n",
       " 25541,\n",
       " 25542,\n",
       " 25543,\n",
       " 25544,\n",
       " 25545,\n",
       " 25546,\n",
       " 25547,\n",
       " 25548,\n",
       " 25549,\n",
       " 25550,\n",
       " 25551,\n",
       " 25552,\n",
       " 25553,\n",
       " 25554,\n",
       " 25555,\n",
       " 25556,\n",
       " 25557,\n",
       " 25558,\n",
       " 25559,\n",
       " 25560,\n",
       " 25561,\n",
       " 25562,\n",
       " 25563,\n",
       " 25564,\n",
       " 25565,\n",
       " 25566,\n",
       " 25567,\n",
       " 25568,\n",
       " 25569,\n",
       " 25570,\n",
       " 25571,\n",
       " 25572,\n",
       " 25573,\n",
       " 25574,\n",
       " 25575,\n",
       " 25576,\n",
       " 25577,\n",
       " 25578,\n",
       " 25579,\n",
       " 25580,\n",
       " 25581,\n",
       " 25582,\n",
       " 25583,\n",
       " 25584,\n",
       " 25585,\n",
       " 25586,\n",
       " 25587,\n",
       " 25588,\n",
       " 25589,\n",
       " 25590,\n",
       " 25591,\n",
       " 25592,\n",
       " 25593,\n",
       " 25594,\n",
       " 25595,\n",
       " 25596,\n",
       " 25597,\n",
       " 25598,\n",
       " 25599,\n",
       " 25600,\n",
       " 25601,\n",
       " 25602,\n",
       " 25603,\n",
       " 25604,\n",
       " 25605,\n",
       " 25606,\n",
       " 25607,\n",
       " 25608,\n",
       " 25609,\n",
       " 25610,\n",
       " 25611,\n",
       " 25612,\n",
       " 25613,\n",
       " 25614,\n",
       " 25615,\n",
       " 25616,\n",
       " 25617,\n",
       " 25618,\n",
       " 25619,\n",
       " 25620,\n",
       " 25621,\n",
       " 25622,\n",
       " 25623,\n",
       " 25624,\n",
       " 25625,\n",
       " 25626,\n",
       " 25627,\n",
       " 25628,\n",
       " 25629,\n",
       " 25630,\n",
       " 25631,\n",
       " 25632,\n",
       " 25633,\n",
       " 25634,\n",
       " 25635,\n",
       " 25636,\n",
       " 25637,\n",
       " 25638,\n",
       " 25639,\n",
       " 25640,\n",
       " 25641,\n",
       " 25642,\n",
       " 25643,\n",
       " 25644,\n",
       " 25645,\n",
       " 25646,\n",
       " 25647,\n",
       " 25648,\n",
       " 25649,\n",
       " 25650,\n",
       " 25651,\n",
       " 25652,\n",
       " 25653,\n",
       " 25654,\n",
       " 25655,\n",
       " 25656,\n",
       " 25657,\n",
       " 25658,\n",
       " 25659,\n",
       " 25660,\n",
       " 25661,\n",
       " 25662,\n",
       " 25663,\n",
       " 25664,\n",
       " 25665,\n",
       " 25666,\n",
       " 25667,\n",
       " 25668,\n",
       " 25669,\n",
       " 25670,\n",
       " 25671,\n",
       " 25672,\n",
       " 25673,\n",
       " 25674,\n",
       " 25675,\n",
       " 25676,\n",
       " 25677,\n",
       " 25678,\n",
       " 25679,\n",
       " 25680,\n",
       " 25681,\n",
       " 25682,\n",
       " 25683,\n",
       " 25684,\n",
       " 25685,\n",
       " 25686,\n",
       " 25687,\n",
       " 25688,\n",
       " 25689,\n",
       " 25690,\n",
       " 25691,\n",
       " 25692,\n",
       " 25693,\n",
       " 25694,\n",
       " 25695,\n",
       " 25696,\n",
       " 25697,\n",
       " 25698,\n",
       " 25699,\n",
       " 25700,\n",
       " 25701,\n",
       " 25702,\n",
       " 25703,\n",
       " 25704,\n",
       " 25705,\n",
       " 25706,\n",
       " 25707,\n",
       " 25708,\n",
       " 25709,\n",
       " 25710,\n",
       " 25711,\n",
       " 25712,\n",
       " 25713,\n",
       " 25714,\n",
       " 25715,\n",
       " 25716,\n",
       " 25717,\n",
       " 25718,\n",
       " 25719,\n",
       " 25720,\n",
       " 25721,\n",
       " 25722,\n",
       " 25723,\n",
       " 25724,\n",
       " 25725,\n",
       " 25726,\n",
       " 25727,\n",
       " 25728,\n",
       " 25729,\n",
       " 25730,\n",
       " 25731,\n",
       " 25732,\n",
       " 25733,\n",
       " 25734,\n",
       " 25735,\n",
       " 25736,\n",
       " 25737,\n",
       " 25738,\n",
       " 25739,\n",
       " 25740,\n",
       " 25741,\n",
       " 25742,\n",
       " 25743,\n",
       " 25744,\n",
       " 25745,\n",
       " 25746,\n",
       " 25747,\n",
       " 25748,\n",
       " 25749,\n",
       " 25750,\n",
       " 25751,\n",
       " 25752,\n",
       " 25753,\n",
       " 25754,\n",
       " 25755,\n",
       " 25756,\n",
       " 25757,\n",
       " 25758,\n",
       " 25759,\n",
       " 25760,\n",
       " 25761,\n",
       " 25762,\n",
       " 25763,\n",
       " 25764,\n",
       " 25765,\n",
       " 25766,\n",
       " 25767,\n",
       " 25768,\n",
       " 25769,\n",
       " 25770,\n",
       " 25771,\n",
       " 25772,\n",
       " 25773,\n",
       " 25774,\n",
       " 25775,\n",
       " 25776,\n",
       " 25777,\n",
       " 25778,\n",
       " 25779,\n",
       " 25780,\n",
       " 25781,\n",
       " 25782,\n",
       " 25783,\n",
       " 25784,\n",
       " 25785,\n",
       " 25786,\n",
       " 25787,\n",
       " 25788,\n",
       " 25789,\n",
       " 25790,\n",
       " 25791,\n",
       " 25792,\n",
       " 25793,\n",
       " 25794,\n",
       " 25795,\n",
       " 25796,\n",
       " 25797,\n",
       " 25798,\n",
       " 25799,\n",
       " 25800,\n",
       " 25801,\n",
       " 25802,\n",
       " 25803,\n",
       " 25804,\n",
       " 25805,\n",
       " 25806,\n",
       " 25807,\n",
       " 25808,\n",
       " 25809,\n",
       " 25810,\n",
       " 25811,\n",
       " 25812,\n",
       " 25813,\n",
       " 25814,\n",
       " 25815,\n",
       " 25816,\n",
       " 25817,\n",
       " 25818,\n",
       " 25819,\n",
       " 25820,\n",
       " 25821,\n",
       " 25822,\n",
       " 25823,\n",
       " 25824,\n",
       " 25825,\n",
       " 25826,\n",
       " 25827,\n",
       " 25828,\n",
       " 25829,\n",
       " 25830,\n",
       " 25831,\n",
       " 25832,\n",
       " 25833,\n",
       " 25834,\n",
       " 25835,\n",
       " 25836,\n",
       " 25837,\n",
       " 25838,\n",
       " 25839,\n",
       " 25840,\n",
       " 25841,\n",
       " 25842,\n",
       " 25843,\n",
       " 25844,\n",
       " 25845,\n",
       " 25846,\n",
       " 25847,\n",
       " 25848,\n",
       " 25849,\n",
       " 25850,\n",
       " 25851,\n",
       " 25852,\n",
       " 25853,\n",
       " 25854,\n",
       " 25855,\n",
       " 25856,\n",
       " 25857,\n",
       " 25858,\n",
       " 25859,\n",
       " 25860,\n",
       " 25861,\n",
       " 25862,\n",
       " 25863,\n",
       " 25864,\n",
       " 25865,\n",
       " 25866,\n",
       " 25867,\n",
       " 25868,\n",
       " 25869,\n",
       " 25870,\n",
       " 25871,\n",
       " 25872,\n",
       " 25873,\n",
       " 25874,\n",
       " 25875,\n",
       " 25876,\n",
       " 25877,\n",
       " 25878,\n",
       " 25879,\n",
       " 25880,\n",
       " 25881,\n",
       " 25882,\n",
       " 25883,\n",
       " 25884,\n",
       " 25885,\n",
       " 25886,\n",
       " 25887,\n",
       " 25888,\n",
       " 25889,\n",
       " 25890,\n",
       " 25891,\n",
       " 25892,\n",
       " 25893,\n",
       " 25894,\n",
       " 25895,\n",
       " 25896,\n",
       " 25897,\n",
       " 25898,\n",
       " 25899,\n",
       " 25900,\n",
       " 25901,\n",
       " 25902,\n",
       " 25903,\n",
       " 25904,\n",
       " 25905,\n",
       " 25906,\n",
       " 25907,\n",
       " 25908,\n",
       " 25909,\n",
       " 25910,\n",
       " 25911,\n",
       " 25912,\n",
       " 25913,\n",
       " 25914,\n",
       " 25915,\n",
       " 25916,\n",
       " 25917,\n",
       " 25918,\n",
       " 25919,\n",
       " 25920,\n",
       " 25921,\n",
       " 25922,\n",
       " 25923,\n",
       " 25924,\n",
       " 25925,\n",
       " 25926,\n",
       " 25927,\n",
       " 25928,\n",
       " 25929,\n",
       " 25930,\n",
       " 25931,\n",
       " 25932,\n",
       " 25933,\n",
       " 25934,\n",
       " 25935,\n",
       " 25936,\n",
       " 25937,\n",
       " 25938,\n",
       " 25939,\n",
       " 25940,\n",
       " 25941,\n",
       " 25942,\n",
       " 25943,\n",
       " 25944,\n",
       " 25945,\n",
       " 25946,\n",
       " 25947,\n",
       " 25948,\n",
       " 25949,\n",
       " 25950,\n",
       " 25951,\n",
       " 25952,\n",
       " 25953,\n",
       " 25954,\n",
       " 25955,\n",
       " 25956,\n",
       " 25957,\n",
       " 25958,\n",
       " 25959,\n",
       " 25960,\n",
       " 25961,\n",
       " 25962,\n",
       " 25963,\n",
       " 25964,\n",
       " 25965,\n",
       " 25966,\n",
       " 25967,\n",
       " 25968,\n",
       " 25969,\n",
       " 25970,\n",
       " 25971,\n",
       " 25972,\n",
       " 25973,\n",
       " 25974,\n",
       " 25975,\n",
       " 25976,\n",
       " 25977,\n",
       " 25978,\n",
       " 25979,\n",
       " 25980,\n",
       " 25981,\n",
       " 25982,\n",
       " 25983,\n",
       " 25984,\n",
       " 25985,\n",
       " 25986,\n",
       " 25987,\n",
       " 25988,\n",
       " 25989,\n",
       " 25990,\n",
       " 25991,\n",
       " 25992,\n",
       " 25993,\n",
       " 25994,\n",
       " 25995,\n",
       " 25996,\n",
       " 25997,\n",
       " 25998,\n",
       " 25999,\n",
       " ...]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def return_input_length(start_num, data):\n",
    "    df_input_length = []\n",
    "    for i in range(start_num, start_num + len(data)):\n",
    "        df_input_length.append(i)\n",
    "    return df_input_length\n",
    "\n",
    "df_input5000_0_length = return_input_length(0, df_input5000_0)\n",
    "df_input5000_1_length = return_input_length(5000, df_input5000_1)\n",
    "df_input5000_2_length = return_input_length(10000, df_input5000_2)\n",
    "df_input5000_3_length = return_input_length(15000, df_input5000_3)\n",
    "df_input5000_4_length = return_input_length(20000, df_input5000_4)\n",
    "df_input5000_5_length = return_input_length(25000, df_input5000_5)\n",
    "\n",
    "df_input5000_5_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_input3_length = return_input_length(0, test_input3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>an elderly woman hugging a young girl surround...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>couple taking a picture during a fun night in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a man is standing next to a sign for the pro f...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    description_text\n",
       "0  an elderly woman hugging a young girl surround...\n",
       "1  couple taking a picture during a fun night in ...\n",
       "2  a man is standing next to a sign for the pro f..."
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_input3 = pd.DataFrame(test_input3)\n",
    "df_test_input3.columns = ['description_text']\n",
    "df_test_input3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                            | 0/3 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:00<00:00, 21.64it/s]\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "from tqdm import tqdm\n",
    "# clean_df_all.description_text, clean_df_all.story_text\n",
    "desc_paragraph = []\n",
    "story_paragraph = []\n",
    "#df_input_length_list= [df_input5000_0_length, df_input5000_1_length, df_input5000_2_length, df_input5000_3_length, df_input5000_4_length, df_input5000_5_length]\n",
    "df_input_length_list = [test_input3_length]\n",
    "\n",
    "for item in df_input_length_list: \n",
    "    for seq_index in tqdm(item):\n",
    "        input_seq = encoder_input_data[seq_index: seq_index + 1]\n",
    "        decoded_sentence = decode_sequence(input_seq)\n",
    "        #print('-')\n",
    "        #print('Input sentence:', clean_df_all.description_text[seq_index: seq_index + 1])\n",
    "        #print (type(clean_df_all.description_text[seq_index: seq_index + 1]))\n",
    "        desc_paragraph.append(list(df_test_input3.description_text[seq_index: seq_index + 1]))\n",
    "        #print('decoded sentence: ', decoded_sentence)\n",
    "        re_decoded_sentence = re.sub('_END', '', decoded_sentence).strip()\n",
    "        #print('Re decoded sentence:', re_decoded_sentence)\n",
    "        story_paragraph.append(re_decoded_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "\n",
    "hypothesis = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',\n",
    "             'ensures', 'that', 'the', 'military', 'always',\n",
    "             'obeys', 'the', 'commands', 'of', 'the', 'party']\n",
    "reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',\n",
    "              'ensures', 'that', 'the', 'military', 'will', 'forever',\n",
    "              'heed', 'Party', 'commands']\n",
    "reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which',\n",
    "              'guarantees', 'the', 'military', 'forces', 'always',\n",
    "              'being', 'under', 'the', 'command', 'of', 'the',\n",
    "              'Party']\n",
    "reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',\n",
    "              'army', 'always', 'to', 'heed', 'the', 'directions',\n",
    "              'of', 'the', 'party']\n",
    "\n",
    "sentence_bleu([reference1, reference2, reference3], hypothesis)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 11-02  decoder 1 layer\n",
    "encoder_inputs = Input(shape=(None,))\n",
    "\n",
    "display('encoder_inputs: ', encoder_inputs.shape)\n",
    "\n",
    "en_x=  Embedding(num_encoder_tokens, EMBEDDING_DIM)(encoder_inputs)\n",
    "\n",
    "encoder = Bidirectional(LSTM(50, return_state=True,\n",
    "                             dropout = 0.5, recurrent_dropout = 0.5))\n",
    "\n",
    "encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder(en_x)\n",
    "\n",
    "state_h = Concatenate()([forward_h, backward_h])\n",
    "state_c = Concatenate()([forward_c, backward_c])\n",
    "\n",
    "# We discard `encoder_outputs` and only keep the states.\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "print (\"state_h: \", state_h.shape)\n",
    "print (\"state_c: \", state_c.shape)\n",
    "print (\"encoder_states: \", encoder_states)\n",
    "\n",
    "### decoder\n",
    "\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "\n",
    "dex=  Embedding(num_decoder_tokens, EMBEDDING_DIM)\n",
    "\n",
    "final_dex= dex(decoder_inputs)\n",
    "\n",
    "print (\"final_dex: \", final_dex.shape)\n",
    "\n",
    "decoder_lstm = LSTM(100, return_sequences=True, return_state=True,\n",
    "                    dropout = 0.5, recurrent_dropout = 0.5)\n",
    "\n",
    "print (\"decoder_bilstm type: \", type(decoder_lstm))\n",
    "\n",
    "print (\"decoder_bilstm: \", decoder_lstm)\n",
    "\n",
    "#print (\"encoder_states: \", encoder_states)\n",
    "\n",
    "print (\"#########\")\n",
    "#print ( decoder_bilstm(final_dex, initial_state=encoder_states) )\n",
    "print (\"#########\")\n",
    "\n",
    "decoder_outputs, _, _ = decoder_lstm(final_dex, initial_state = encoder_states)\n",
    "\n",
    "display('decoder_outputs: ', decoder_outputs.shape)\n",
    "\n",
    "decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
    "\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "\n",
    "model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['acc'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}