Table_summarizer_unidirectional_LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Weather data summarizer.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "-JzteLqDVJk4",
        "colab_type": "code",
        "outputId": "945a6774-9424-42a4-a8cd-93e2e3624704",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GHepkYf8VhpE",
        "colab_type": "code",
        "outputId": "d71c2f65-c216-40df-b48b-1f10fac04647",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "!ls '/content/gdrive/My Drive/Resume DL Projects'\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "exp-basic.ipynb\n",
            "facenet_keras.h5\n",
            "Facial_recognition.ipynb\n",
            "How_to_build_own_text_summarizer_using_deep_learning.ipynb\n",
            "neural-machine-translation-with-attention-dates.ipynb\n",
            "summarizing-text-with-amazon-reviews.ipynb\n",
            "text-summarization-with-seq2seq-model.ipynb\n",
            "Text_summarizer.ipynb\n",
            "unidir_model.h5\n",
            "weather_summ.h5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cz-4Jk50x_Gw",
        "colab_type": "code",
        "outputId": "525529f7-2966-480a-876c-21fbf9023cf1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "!wget https://raw.githubusercontent.com/niravnb/Text-Summarizer/master/WeatherGov.zip"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-11 05:06:45--  https://raw.githubusercontent.com/niravnb/Text-Summarizer/master/WeatherGov.zip\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 2779488 (2.7M) [application/zip]\n",
            "Saving to: ‘WeatherGov.zip’\n",
            "\n",
            "\rWeatherGov.zip        0%[                    ]       0  --.-KB/s               \rWeatherGov.zip      100%[===================>]   2.65M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2020-06-11 05:06:46 (19.1 MB/s) - ‘WeatherGov.zip’ saved [2779488/2779488]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8H34ZXLyUbp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip WeatherGov.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_ORTH5hyubf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sentences_to_read = 25000"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3cf4Ly9Xy1m8",
        "colab_type": "code",
        "outputId": "92191238-1be5-4775-f97f-412bd4b5e9a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        }
      },
      "source": [
        "# Loading train.combined as source_sent and summaries.txt as target_sent\n",
        "source_sent = []\n",
        "target_sent = []\n",
        "\n",
        "test_source_sent = []\n",
        "test_target_sent = []\n",
        "\n",
        "\n",
        "with open('WeatherGov/train/train.combined', 'r') as f:\n",
        "    for l_i, line in enumerate(f):\n",
        "        source_sent.append(line)\n",
        "        if len(source_sent)>=sentences_to_read:\n",
        "            break\n",
        "        \n",
        "            \n",
        "with open('WeatherGov/train/summaries.txt', 'r') as f:\n",
        "    for l_i, line in enumerate(f):\n",
        "        target_sent.append(line)\n",
        "        if len(target_sent)>=sentences_to_read:\n",
        "            break\n",
        "        \n",
        "            \n",
        "assert len(source_sent)==len(target_sent),'Source: %d, Target: %d'%(len(source_sent),len(target_sent))\n",
        "\n",
        "print('Sample translations (%d)'%len(source_sent))\n",
        "for i in range(0,sentences_to_read,10000):\n",
        "    print('(',i,') Table: ', source_sent[i])\n",
        "    print('(',i,') Summary: ', target_sent[i])\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sample translations (25000)\n",
            "( 0 ) Table:  temperature time 6-21 min 26 mean 43 max 53 windChill time 6-21 min 0 mean 22 max 46 windSpeed time 6-21 min 6 mean 8 max 10 mode-bucket-0-20-2 0-10 windDir time 6-21 mode SE gust time 6-21 min 0 mean 0 max 0 skyCover time 6-21 mode-bucket-0-100-4 50-75 skyCover time 6-9 mode-bucket-0-100-4 0-25 skyCover time 6-13 mode-bucket-0-100-4 25-50 skyCover time 9-21 mode-bucket-0-100-4 50-75 skyCover time 13-21 mode-bucket-0-100-4 50-75 precipPotential time 6-21 min 0 mean 2 max 12\n",
            "\n",
            "( 0 ) Summary:  Mostly sunny , with a high near 53 . Southeast wind between 7 and 9 mph . \n",
            "\n",
            "( 10000 ) Table:  temperature time 6-21 min 29 mean 31 max 35 windChill time 6-21 min 16 mean 21 max 25 windSpeed time 6-21 min 8 mean 15 max 21 mode-bucket-0-20-2 10-20 windDir time 6-21 mode WNW gust time 6-21 min 0 mean 12 max 29 skyCover time 6-21 mode-bucket-0-100-4 50-75 skyCover time 6-9 mode-bucket-0-100-4 50-75 skyCover time 6-13 mode-bucket-0-100-4 50-75 skyCover time 9-21 mode-bucket-0-100-4 50-75 skyCover time 13-21 mode-bucket-0-100-4 75-100 precipPotential time 6-21 min 15 mean 17 max 18 snowChance time 6-21 mode SChc snowChance time 6-9 mode SChc snowChance time 6-13 mode SChc snowChance time 9-21 mode SChc snowChance time 13-21 mode SChc\n",
            "\n",
            "( 10000 ) Summary:  A 20 percent chance of snow . Mostly cloudy , with a high near 35 . Breezy , with a west northwest wind 8 to 11 mph increasing to between 17 and 20 mph . Winds could gust as high as 28 mph . \n",
            "\n",
            "( 20000 ) Table:  temperature time 17-30 min 44 mean 48 max 57 windChill time 17-30 min 0 mean 29 max 49 windSpeed time 17-30 min 2 mean 2 max 3 mode-bucket-0-20-2 0-10 windDir time 17-30 mode SE gust time 17-30 min 0 mean 0 max 0 skyCover time 17-30 mode-bucket-0-100-4 50-75 skyCover time 17-21 mode-bucket-0-100-4 50-75 skyCover time 17-26 mode-bucket-0-100-4 50-75 skyCover time 21-30 mode-bucket-0-100-4 50-75 skyCover time 26-30 mode-bucket-0-100-4 50-75 precipPotential time 17-30 min 6 mean 11 max 16 rainChance time 26-30 mode SChc\n",
            "\n",
            "( 20000 ) Summary:  A slight chance of showers after 4am . Mostly cloudy , with a low around 43 . Light southeast wind . Chance of precipitation is 20 % . \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MT9PP0y6y1qp",
        "colab_type": "code",
        "outputId": "4cba4ebf-2eeb-4b4c-f3df-e347bc7ca168",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "!pip install nltk"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2z8jykjt0NKP",
        "colab_type": "code",
        "outputId": "da37da80-b3e8-4a64-98cc-c75ad11f6a1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IliVUlcF0O2V",
        "colab_type": "code",
        "outputId": "5abd70bb-b1cd-4264-91f8-98ba7c34c206",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd \n",
        "import re\n",
        "from bs4 import BeautifulSoup\n",
        "from keras.preprocessing.text import Tokenizer \n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from nltk.corpus import stopwords\n",
        "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed, Bidirectional\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "import warnings\n",
        "pd.set_option(\"display.max_colwidth\", 200)\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_OnzIme70l9z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = pd.DataFrame()\n",
        "data['cleaned_text']=source_sent\n",
        "data['cleaned_summary']=target_sent"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j0P_z5N80QIg",
        "colab_type": "code",
        "outputId": "ebc47dff-56b7-4345-f752-ee7eba3ef6cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "data.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cleaned_text</th>\n",
              "      <th>cleaned_summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>temperature time 6-21 min 26 mean 43 max 53 windChill time 6-21 min 0 mean 22 max 46 windSpeed time 6-21 min 6 mean 8 max 10 mode-bucket-0-20-2 0-10 windDir time 6-21 mode SE gust time 6-21 min 0 ...</td>\n",
              "      <td>Mostly sunny , with a high near 53 . Southeast wind between 7 and 9 mph . \\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>temperature time 17-30 min 47 mean 50 max 58 windChill time 17-30 min 0 mean 30 max 49 windSpeed time 17-30 min 3 mean 5 max 9 mode-bucket-0-20-2 0-10 windDir time 17-30 mode E gust time 17-30 min...</td>\n",
              "      <td>A 50 percent chance of showers . Cloudy , with a low around 47 . East wind around 5 mph . \\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>temperature time 17-30 min 51 mean 55 max 62 windChill time 17-30 min 0 mean 0 max 0 windSpeed time 17-30 min 2 mean 3 max 6 mode-bucket-0-20-2 0-10 windDir time 17-30 mode SSE gust time 17-30 min...</td>\n",
              "      <td>Cloudy , with a low around 49 . Calm wind becoming south southeast around 5 mph . \\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>temperature time 17-30 min 26 mean 29 max 38 windChill time 17-30 min 26 mean 28 max 36 windSpeed time 17-30 min 0 mean 0 max 5 mode-bucket-0-20-2 0-10 windDir time 17-30 mode ESE gust time 17-30 ...</td>\n",
              "      <td>Partly cloudy , with a low around 23 . Calm wind . \\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>temperature time 17-30 min 55 mean 60 max 69 windChill time 17-30 min 0 mean 0 max 0 windSpeed time 17-30 min 8 mean 9 max 14 mode-bucket-0-20-2 0-10 windDir time 17-30 mode SSW gust time 17-30 mi...</td>\n",
              "      <td>Partly cloudy , with a low around 53 . South wind around 9 mph . \\n</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                                                                                                                              cleaned_text                                                                               cleaned_summary\n",
              "0  temperature time 6-21 min 26 mean 43 max 53 windChill time 6-21 min 0 mean 22 max 46 windSpeed time 6-21 min 6 mean 8 max 10 mode-bucket-0-20-2 0-10 windDir time 6-21 mode SE gust time 6-21 min 0 ...                  Mostly sunny , with a high near 53 . Southeast wind between 7 and 9 mph . \\n\n",
              "1  temperature time 17-30 min 47 mean 50 max 58 windChill time 17-30 min 0 mean 30 max 49 windSpeed time 17-30 min 3 mean 5 max 9 mode-bucket-0-20-2 0-10 windDir time 17-30 mode E gust time 17-30 min...  A 50 percent chance of showers . Cloudy , with a low around 47 . East wind around 5 mph . \\n\n",
              "2  temperature time 17-30 min 51 mean 55 max 62 windChill time 17-30 min 0 mean 0 max 0 windSpeed time 17-30 min 2 mean 3 max 6 mode-bucket-0-20-2 0-10 windDir time 17-30 mode SSE gust time 17-30 min...          Cloudy , with a low around 49 . Calm wind becoming south southeast around 5 mph . \\n\n",
              "3  temperature time 17-30 min 26 mean 29 max 38 windChill time 17-30 min 26 mean 28 max 36 windSpeed time 17-30 min 0 mean 0 max 5 mode-bucket-0-20-2 0-10 windDir time 17-30 mode ESE gust time 17-30 ...                                         Partly cloudy , with a low around 23 . Calm wind . \\n\n",
              "4  temperature time 17-30 min 55 mean 60 max 69 windChill time 17-30 min 0 mean 0 max 0 windSpeed time 17-30 min 8 mean 9 max 14 mode-bucket-0-20-2 0-10 windDir time 17-30 mode SSW gust time 17-30 mi...                           Partly cloudy , with a low around 53 . South wind around 9 mph . \\n"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2bsQ9iHS0YIQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data.replace('', np.nan, inplace=True)\n",
        "data.dropna(axis=0,inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jNuM7ve_0iLi",
        "colab_type": "code",
        "outputId": "b4218813-d928-4f8c-ca1e-a404a7d189d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "text_word_count = []\n",
        "summary_word_count = []\n",
        "\n",
        "# populate the lists with sentence lengths\n",
        "for i in data['cleaned_text']:\n",
        "      text_word_count.append(len(i.split()))\n",
        "\n",
        "for i in data['cleaned_summary']:\n",
        "      summary_word_count.append(len(i.split()))\n",
        "\n",
        "length_df = pd.DataFrame({'text':text_word_count, 'summary':summary_word_count})\n",
        "\n",
        "length_df.hist(bins = 30)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAcdUlEQVR4nO3df5RU5Z3n8fcn4u8k4q/tGCDTJLLmEJn4o1fJms30SqJosuKcow6uE9Fhlp2zONEJG4Uke0ii7OqZMUbHxDnswIiOERmjkY0myih9MnN2QEWNKMSxoyhwEFR+GDS6afPdP+7TWlRXNdVNdVVXP5/XOXX63uc+997nqb71rVvPfe5zFRGYmVkePtDsApiZWeM46JuZZcRB38wsIw76ZmYZcdA3M8uIg76ZWUYc9M3MMuKgb2bDgqQNkj4/XLYzUjno2x4kjWp2Gcxs6DjoDwFJV0naLOnXkp6TNEXSrZKuKcnTKWlTyfwGSV+T9LSkNyUtktQm6adpO/8o6fCUt11SSLpU0kZJOyT9maR/l9bfKenmkm1/QtIjkl6X9JqkOySNLtv3VZKeBt5M5fhRWZ1uknTjkL5xli1JtwMfA/6PpN2SrpQ0WdL/TcfzLyR1prz/Ph3H49L8p9Nn4JOVttO0Sg1XEeFXHV/AccBG4KNpvh34BHArcE1Jvk5gU8n8BmAV0AaMAbYBTwAnAgcBjwDzS7YZwN+kZWcAbwM/Bv5Nyfp/kPIfC3wBOBA4Gvg58L2yfT8FjAMOBo4B3gRGp+Wj0vZObvb769fIfaXj8PNpegzwOnA2xcnpF9L80Wn5gvSZOBhYC1xWaTt+9X35TL/+3qUIrhMl7R8RGyLiVzWu+9cRsTUiNgP/BKyOiCcj4m3gXoovgFJXR8TbEfEQRZC+MyK2lax/IkBEdEfEioh4JyJeBb4L/EHZtm6KiI0R8ZuI2ELxxXB+WjYVeC0i1gzonTAbvD8GHoiIByLidxGxAnic4ksA4FvAYcCjwGbg+00pZQty0K+ziOgGrqA4KLdJWirpozWuvrVk+jcV5j84mPypmWhpanJ6A/h74KiybW0sm19C8cEj/b29xjqY1cPvAeenpp2dknYCn6X4FUpE/Jbi1/PxwPWRTvFt7xz0h0BE/DAiPktx4AZwHcWZ+CEl2T7SwCL9z1SOSRHxYYogrrI85R+aHwO/L+l44EvAHUNeSstd6TG4Ebg9IkaXvA6NiGsBJI0B5gN/B1wv6cAq27EyDvp1Juk4Saeng/BtijPu31G0mZ8t6QhJH6H4NdAoHwJ2A7vSh+Vre1shNSndDfwQeDQiXh7aIpqxFfh4mv574D9JOlPSfpIOSp0fxkoSxVn+ImAmsAW4usp2rIyDfv0dCFwLvAa8QnFhdR5F88gvKC4yPQTc1cAyfRs4CdgF3A/cU+N6S4BJuGnHGuN/Ad9MTTl/BEwDvg68SnHm/zWKmPUVis/V/0jNOpcCl0r6D+XbkfTfG1yHYU9uCrNqJH0M+CXwkYh4o9nlMbN95zN9q0jSB4CvAksd8M1GDt99aX1IOpSiXfQliu6aZjZCuHnHzCwjbt6xbElaLGmbpGdK0v5S0i/TcBb3lg1XMU9Sdxpa48yS9KkprVvS3JL08ZJWp/S7JB3QuNqZVTasz/SPOuqoaG9vb8q+33zzTQ499NCm7HugXNb+rVmz5rWIOLo8XdLnKLqy3hYRx6e0M4BHIqJH0nUAEXGVpInAncApwEeBfwT+bdrUv1IME7AJeAy4MCLWSVoG3BMRSyX9DfCLiLhlb+Wt13HfSsfFYOVQRxh4Pasd88DwHnvn5JNPjmZZuXJl0/Y9UC5r/4DHo/p4L+3AM1WW/SFwR5qeB8wrWfYg8Jn0erAkfV56iaLb7qiUvke+/l71Ou5b6bgYrBzqGDHwevZ3zPtCrll1f8L791OMoRgQr9emlAZ7DmGxCTgVOBLYGRE9FfL3IWkWMAugra2Nrq6ufS07u3fvrst2hrMc6gj1raeDvlkFkr4B9NCg4SciYiGwEKCjoyM6Ozv3eZtdXV3UYzvDWQ51hPrW00HfrIykSyjGG5qSfipDMZLjuJJsY1MaVdJfB0ZLGpXO9kvzmzWNe++YlZA0FbgSOCci3ipZtByYLulASeOBCRTD+j4GTEg9dQ4ApgPL05fFSuC8tP4M4L5G1cOsGgd9y5akO4F/AY6TtEnSTOBmigHqVkh6KvW6ISKeBZYB64CfAbMj4t10Fn8ZxYXd9cCylBfgKuCrkrop2vgXNbB6ZhW5eceyFREXVkiuGpgjYgHFE5vK0x8AHqiQ/gJFF0+zYcNn+mZmGXHQNzPLiIO+mVlGRnSbfvvc+/eY33DtF5tUErN9V348g49pGzif6ZuZZcRB38wsIw76ZmYZcdA3M8uIg76ZWUYc9M3MMuKgb2aWEQd9M7OMOOibmWXEQd/MLCMO+mZmGXHQNzPLiIO+mVlGHPTNzDLioG9mlhEHfTOzjDjom5llxEHfzCwjDvpmZhmpOehL2k/Sk5J+kubHS1otqVvSXZIOSOkHpvnutLy9ZBvzUvpzks6sd2XMzKx/AznTvxxYXzJ/HXBDRBwL7ABmpvSZwI6UfkPKh6SJwHTgU8BU4AeS9tu34puZ2UDUFPQljQW+CPxtmhdwOnB3yrIEODdNT0vzpOVTUv5pwNKIeCciXgS6gVPqUQkzM6vNqBrzfQ+4EvhQmj8S2BkRPWl+EzAmTY8BNgJERI+kXSn/GGBVyTZL13mPpFnALIC2tja6urpqrUsfcyb17DE/kG3t3r17n/bdSC6rmdVqr0Ff0peAbRGxRlLnUBcoIhYCCwE6Ojqis3Pwu7xk7v17zG+4qPZtdXV1sS/7biSXdXAkLQZ6j+/jU9oRwF1AO7ABuCAidqRfqzcCZwNvAZdExBNpnRnAN9Nmr4mIJSn9ZOBW4GDgAeDyiIiGVM6silqad04DzpG0AVhK0axzIzBaUu+Xxlhgc5reDIwDSMsPA14vTa+wjlkz3EpxfanUXODhiJgAPJzmAc4CJqTXLOAWeO9LYj5wKkVz5XxJh6d1bgH+S8l65fsya7i9Bv2ImBcRYyOineJC7CMRcRGwEjgvZZsB3Jeml6d50vJH0tnNcmB66t0znuJD8GjdamI2QBHxc2B7WXLpNanya1W3RWEVxUnPMcCZwIqI2B4RO4AVwNS07MMRsSod/7eVbMusaWpt06/kKmCppGuAJ4FFKX0RcLukbooP1HSAiHhW0jJgHdADzI6Id/dh/2ZDoS0itqTpV4C2NP3etaqk95pUf+mbKqRXVMu1rPJrVND/daocrp/kUEeobz0HFPQjogvoStMvUKH3TUS8DZxfZf0FwIKBFtKsGSIiJDWkDb6Wa1nl16ig/+tUw+n6yVDJoY5Q33r6jlyzPW1NTTOkv9tSerVrUv2lj62QbtZUDvpmeyq9JlV+repiFSYDu1Iz0IPAGZIOTxdwzwAeTMvekDQ59fy5uGRbZk2zL236Zi1N0p1AJ3CUpE0UvXCuBZZJmgm8BFyQsj9A0V2zm6LL5qUAEbFd0tXAYynfdyKi9+Lwf+P9Lps/TS+zpnLQt2xFxIVVFk2pkDeA2VW2sxhYXCH9ceD4fSmjWb25ecfMLCMO+mZmGXHQNzPLiIO+mVlGHPTNzDLioG9mlhEHfTOzjDjom5llxEHfzCwjDvpmZhlx0Dczy4iDvplZRhz0zcwy4qBvZpYRB30zs4w46JuZZcRB38wsIw76ZmYZcdA3M8uIg76ZWUYc9M3MMuKgb2aWEQd9M7OMOOibmWXEQd/MLCMO+mZmGXHQNzPLiIO+WQWS/kLSs5KekXSnpIMkjZe0WlK3pLskHZDyHpjmu9Py9pLtzEvpz0k6s1n1MevloG9WRtIY4CtAR0QcD+wHTAeuA26IiGOBHcDMtMpMYEdKvyHlQ9LEtN6ngKnADyTt18i6mJVz0DerbBRwsKRRwCHAFuB04O60fAlwbpqeluZJy6dIUkpfGhHvRMSLQDdwSoPKb1bRqGYXwGy4iYjNkv4KeBn4DfAQsAbYGRE9KdsmYEyaHgNsTOv2SNoFHJnSV5VsunSdPUiaBcwCaGtro6urq0+eOZN6+qRVytdr9+7d/S4fCXKoI9S3ng76ZmUkHU5xlj4e2An8A0XzzJCJiIXAQoCOjo7o7Ozsk+eSuff3SdtwUd98vbq6uqi0nZEkhzpCfevp5h2zvj4PvBgRr0bEb4F7gNOA0am5B2AssDlNbwbGAaTlhwGvl6ZXWMesKfYa9FOvhUcl/SL1Zvh2SndPBhupXgYmSzoktc1PAdYBK4HzUp4ZwH1penmaJy1/JCIipU9Pn4nxwATg0QbVwayiWs703wFOj4hPAycAUyVNxj0ZbISKiNUUF2SfANZSfE4WAlcBX5XUTdFmvyitsgg4MqV/FZibtvMssIziC+NnwOyIeLeBVTHrY69t+umMZXea3T+9gqInw39O6UuAbwG3ULSFfiul3w3cXN6TAXgxfUBOAf6lHhUxq6eImA/ML0t+gQq9byLibeD8KttZACyoewHNBqmmC7npjHwNcCzwfeBXDFFPhlp6MdSqvLfDQLbVSr0CXFYzq1VNQT/9JD1B0mjgXuCTQ1WgWnox1Kq8t0N/PR3KtVKvAJfVzGo1oN47EbGT4mLWZ3BPBjOzllNL752j0xk+kg4GvgCsxz0ZzMxaTi3NO8cAS1K7/geAZRHxE0nrgKWSrgGeZM+eDLenC7XbKXrsEBHPSurtydCDezKYmTVcLb13ngZOrJDungxmZi3Gd+SamWXEQd/MLCMO+mZmGXHQNzPLiIO+mVlGHPTNzDLioG9mlhEH/TLtc++nfe79rN28i/YKTyoyM2tlDvpmZhlx0Dczy4iDvplZRhz0zcwy4qBvZpYRB30zs4w46JuZZcRB38wsIw76ZmYZcdA3M8uIg76ZWUYc9M3MMuKgb2aWEQd9swokjZZ0t6RfSlov6TOSjpC0QtLz6e/hKa8k3SSpW9LTkk4q2c6MlP95STOaVyOzgoO+WWU3Aj+LiE8CnwbWA3OBhyNiAvBwmgc4C5iQXrOAWwAkHQHMB04FTgHm935RmDWLg75ZGUmHAZ8DFgFExP+LiJ3ANGBJyrYEODdNTwNui8IqYLSkY4AzgRURsT0idgArgKkNrIpZH6OaXQCzYWg88Crwd5I+DawBLgfaImJLyvMK0JamxwAbS9bflNKqpfchaRbFrwTa2tro6urqk2fOpJ4+aZXy9dq9e3e/y0eCHOoI9a2ng75ZX6OAk4A/j4jVkm7k/aYcACIiJEW9dhgRC4GFAB0dHdHZ2dknzyUVnuS24aK++Xp1dXVRaTsjSQ51hPrW0807Zn1tAjZFxOo0fzfFl8DW1GxD+rstLd8MjCtZf2xKq5Zu1jQO+mZlIuIVYKOk41LSFGAdsBzo7YEzA7gvTS8HLk69eCYDu1Iz0IPAGZIOTxdwz0hpZk3j5h2zyv4cuEPSAcALwKUUJ0nLJM0EXgIuSHkfAM4GuoG3Ul4iYrukq4HHUr7vRMT2xlXBrC8HfbMKIuIpoKPCoikV8gYwu8p2FgOL61s6s8Fz846ZWUYc9M3MMuKgb2aWEQd9M7OMOOibmWXEQd/MLCMO+mZmGdlrP31J44DbKAaXCmBhRNyYho29C2gHNgAXRMQOSaIYlvZsihtVLomIJ9K2ZgDfTJu+JiKWMAK0VxoT5dovNqEkZmb9q+VMvweYExETgcnAbEkT8djiZmYtZ69BPyK29J6pR8SvKR4mMQaPLW5m1nIGNAyDpHbgRGA1QzS2eC3jiteqfPzxWrbVu07bwcX0QNYZ6L7qpZXGFG+lspqNRDUHfUkfBH4EXBERbxRN94V6ji1ey7jitSoff7y/scfL15kzqYfr144a0DoD3Ve9tNKY4q1UVrORqKagL2l/ioB/R0Tck5K3SjomIrYMYGzxzrL0rsEXfU+VLqaamdme9tqmn3rjLALWR8R3SxZ5bHEzsxZTy5n+acCXgbWSnkppXweuxWOLm9kwUP5L312mq9tr0I+IfwZUZbHHFjczayG+I9fMLCMO+mZmGXHQNzPLiIO+mVlGHPTNzDLioG9mlpEBjb1jBd/9a2atymf6ZmYZcdA3M8uIg76ZWUYc9M3MMuKgb1aFpP0kPSnpJ2l+vKTVkrol3SXpgJR+YJrvTsvbS7YxL6U/J+nM5tTE7H0O+mbVXU7xeNBe1wE3RMSxwA5gZkqfCexI6TekfKRnSU8HPkXxaNAfSNqvQWU3q8hB36wCSWOBLwJ/m+YFnA7cnbKUPxe693nRdwNTUv5pwNKIeCciXqQYbvyUxtTArLKs+ulX6l/vcbetiu8BVwIfSvNHAjsjoveByKXPeH7v+c8R0SNpV8o/BlhVss2Kz4WG2p4NPdBnMefwPOLeOg7meditpJ7/y6yCvlktJH0J2BYRayR1NmKftTwbeqDPYs7hecS9dRzM87BbST3/lw76Zn2dBpwj6WzgIODDwI3AaEmj0tl+77Of4f3nQm+SNAo4DHid6s+LNmsat+mblYmIeRExNiLaKS7EPhIRFwErgfNStvLnQvc+L/q8lD9S+vTUu2c8MAF4tEHVMKvIZ/pmtbsKWCrpGuBJYFFKXwTcLqkb2E7xRUFEPCtpGbAO6AFmR8S7jS+22fsc9M36ERFdQFeafoEKvW8i4m3g/CrrLwAWDF0JzQbGzTtmZhlx0Dczy4iDvplZRhz0zcwy4qBvZpYRB30zs4w46JuZZcRB38wsIw76ZmYZ8R25w0j50M8e9tnM6s1n+mZmGXHQNzPLiIO+mVlGHPTNzDLiC7l7Uem5umZmrcpn+mZmGXHQNzPLyF6DvqTFkrZJeqYk7QhJKyQ9n/4entIl6SZJ3ZKelnRSyTozUv7nJc2otC8zMxtatbTp3wrcDNxWkjYXeDgirpU0N81fBZxF8fDnCcCpwC3AqZKOAOYDHUAAayQtj4gd9arIcOcbr8xsONjrmX5E/JziYc+lpgFL0vQS4NyS9NuisAoYLekY4ExgRURsT4F+BTC1HhUwM7PaDbb3TltEbEnTrwBtaXoMsLEk36aUVi29D0mzgFkAbW1tdHV11VSgOZN6aiz6nsq337udtoMHv02Av77jvrLt9r/f0n33l6eS3bt315y32VqprGYj0T532YyIkBT1KEza3kJgIUBHR0d0dnbWtN4lg+xaueGiPbffu505k3q4fu3Q9Wgt32/pvvvLU0lXVxe1vk/N1kplNRuJBtt7Z2tqtiH93ZbSNwPjSvKNTWnV0s3MrIEGG/SXA709cGYA95WkX5x68UwGdqVmoAeBMyQdnnr6nJHSzMysgfbafiHpTqATOErSJopeONcCyyTNBF4CLkjZHwDOBrqBt4BLASJiu6SrgcdSvu9ERPnFYTMzG2J7DfoRcWGVRVMq5A1gdpXtLAYWD6h0mas0BIS7eprZvvAduWZmGXHQNysjaZyklZLWSXpW0uUp3XeiW8tz0DfrqweYExETgcnAbEkTef9O9AnAw2ke9rwTfRbFneiU3Il+KnAKML/3i8KsWRz0zcpExJaIeCJN/xpYT3Ezoe9Et5bn8fTN+iGpHTgRWE2T70SvdId4f3c31/Pu57Wbd+0xP2nMYXXZ7r7qreNg72ZvFfX8Xzrom1Uh6YPAj4ArIuINSe8ta8ad6JXuOu/vru163v082LvFh1pvHYdr+eqlnv9LN++YVSBpf4qAf0dE3JOSfSe6tTwHfbMyKk7pFwHrI+K7JYt8J7q1PDfvmPV1GvBlYK2kp1La1/Gd6A3hZ08MLQf9JhnsA9f9gRh6EfHPgKos9p3o1tLcvGNmlpHsz/QHe8ZtZtaKfKZvZpYRB30zs4xk37xjZtYK6jXUus/0zcwy4jP9Ftc+937mTOrZ4zZ0d+M0s2oc9M1GMD99zcq5ecfMLCMte6bv/vVmZgPXskHfauehG8ysl4P+CORfQWZWjdv0zcwy4jN9A9wEZJYLB30zGxLuLjo8uXnHzCwjDvpmZhlx0Dczy4iDvplZRhz0zcwy4t47GfLNW2b102rdnR30raJavhiG+8FtZn25ecfMLCM+0zczI5+byRz0zcwarJnXARz0bUiVH9y3Tj20SSUxM3DQt33gXkBmrafhF3IlTZX0nKRuSXMbvX+zRvMxb8NJQ8/0Je0HfB/4ArAJeEzS8ohY18hyWPOs3byLS1qsX/O+8DFvw02jm3dOAboj4gUASUuBaYA/ABnb20WtFu9V4WPehhVFRON2Jp0HTI2IP03zXwZOjYjLSvLMAmal2eOA5xpWwD0dBbzWpH0PlMvav9+LiKMbvE+gtmM+pQ/Fcd9Kx8Vg5VBHGHg9qx7zw+5CbkQsBBY2uxySHo+IjmaXoxYua+sbiuM+h/c6hzpCfevZ6Au5m4FxJfNjU5rZSOVj3oaVRgf9x4AJksZLOgCYDixvcBnMGsnHvA0rDW3eiYgeSZcBDwL7AYsj4tlGlmEAmt7ENAAu6zDV5GM+h/c6hzpCHevZ0Au5ZmbWXB5l08wsIw76ZmYZyT7oSxonaaWkdZKelXR5Sv+WpM2Snkqvs5tdVgBJGyStTWV6PKUdIWmFpOfT38OHQTmPK3nvnpL0hqQrhuv72uok/UU6fp+RdKekg9LF49Vp+Ie70oXkliJpsaRtkp4pSat4vKtwU6rv05JOal7Ja1eljn8p6ZepHvdKGl2ybF6q43OSzhzo/rIP+kAPMCciJgKTgdmSJqZlN0TECen1QPOK2Md/TGXq7bc7F3g4IiYAD6f5poqI53rfO+Bk4C3g3rR4uL6vLUnSGOArQEdEHE9xwXg6cB3Fe30ssAOY2bxSDtqtwNSytGrH+1nAhPSaBdzSoDLuq1vpW8cVwPER8fvAvwLzAFJsmg58Kq3zgzTUR82yD/oRsSUinkjTvwbWA2OaW6oBmwYsSdNLgHObWJZKpgC/ioiXml2QEWwUcLCkUcAhwBbgdODutHw4Hhd7FRE/B7aXJVc73qcBt0VhFTBa0jGNKengVapjRDwUET1pdhXF/R1Q1HFpRLwTES8C3RRDfdQs+6BfSlI7cCKwOiVdln5eLR4OTSZJAA9JWpNu3Qdoi4gtafoVoK05RatqOnBnyfxwfF9bVkRsBv4KeJki2O8C1gA7SwLHJlrvZKaaasf7GGBjSb6RUuc/AX6apve5jg76iaQPAj8CroiINyh+Gn4COIHig3R9E4tX6rMRcRLFT9nZkj5XujCKPrjDph9uakc+B/iHlDRc39eWlb44pwHjgY8Ch9K3uWBEGm7He71J+gZFE/Qd9dqmgz4gaX+KgH9HRNwDEBFbI+LdiPgd8L8Z4E+ooZLO6oiIbRRt5KcAW3t/xqa/25pXwj7OAp6IiK0wfN/XFvd54MWIeDUifgvcA5xG0bzRewPmSBr+odrxPqKGvJB0CfAl4KJ4/4aqfa5j9kFfkoBFwPqI+G5Jemlb4B8Cz5Sv22iSDpX0od5p4AyKci0HZqRsM4D7mlPCii6kpGlnOL6vI8DLwGRJh6TjeQrF0M0rgfNSnuF2XOyLasf7cuDi1ItnMrCrpBmopUiaClwJnBMRb5UsWg5Ml3SgpPEUF60fHdC2c78jV9JngX8C1gK/S8lfpwhWJ1D8dNwA/NdmH0CSPs77PWBGAT+MiAWSjgSWAR8DXgIuiIjyi18Nl76YXgY+HhG7UtrtDLP3dSSQ9G3gjyiaAp4E/pSirXcpcERK++OIeKdphRwESXcCnRRDC28F5gM/psLxnr7wbqZo2noLuDQiHm9GuQeiSh3nAQcCr6dsqyLiz1L+b1C08/dQNEf/tHyb/e4v96BvZpaT7Jt3zMxy4qBvZpYRB30zs4w46JuZZcRB38wsIw76ZmYZcdA3M8vI/wc2dO0SBmnCDAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hh2VGlAi0qlv",
        "colab_type": "code",
        "outputId": "68a4f2e5-c43d-49bd-c9c5-6209c005bb91",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "cnt=0\n",
        "for i in data['cleaned_summary']:\n",
        "    if(len(i.split())<=60):\n",
        "        cnt=cnt+1\n",
        "print(cnt/len(data['cleaned_summary']))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.96256\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4i4apTs_2Ywu",
        "colab_type": "code",
        "outputId": "0ea0013d-4266-4522-d869-51d257ed821d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "cnt=0\n",
        "for i in data['cleaned_text']:\n",
        "    if(len(i.split())<=120):\n",
        "        cnt=cnt+1\n",
        "print(cnt/len(data['cleaned_text']))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.94176\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3UHiN1No1Nq_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_text_len=120\n",
        "max_summary_len=60"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ppJe-qQ1PxG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cleaned_text =np.array(data['cleaned_text'])\n",
        "cleaned_summary=np.array(data['cleaned_summary'])\n",
        "\n",
        "short_text=[]\n",
        "short_summary=[]\n",
        "\n",
        "for i in range(len(cleaned_text)):\n",
        "    if(len(cleaned_summary[i].split())<=max_summary_len and len(cleaned_text[i].split())<=max_text_len):\n",
        "        short_text.append(cleaned_text[i])\n",
        "        short_summary.append(cleaned_summary[i])\n",
        "        \n",
        "df=pd.DataFrame({'text':short_text,'summary':short_summary})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CMgKY08_2jd7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['summary'] = df['summary'].apply(lambda x : 'sostok '+ x + ' eostok')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wi2TLL0Z2oBq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.to_csv('cleaned_weather_data.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_rw9-f6gTOu4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_SEQUENCE_LENGTH = max_text_len\n",
        "MAX_NUM_WORDS = 140\n",
        "EMBEDDING_DIM = 100\n",
        "VALIDATION_SPLIT = 0.1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zgap6TER2tag",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "x_tr,x_val,y_tr,y_val=train_test_split(np.array(df['text']),np.array(df['summary']),test_size=0.1,random_state=0,shuffle=True) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_BozpjTo228U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer \n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "#prepare a tokenizer for reviews on training data\n",
        "x_tokenizer = Tokenizer() \n",
        "x_tokenizer.fit_on_texts(list(x_tr))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IK48_FPD4lo0",
        "colab_type": "code",
        "outputId": "c587dcc8-fe5f-42d9-83a5-faa95d9e247f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "thresh=4\n",
        "\n",
        "cnt=0\n",
        "tot_cnt=0\n",
        "freq=0\n",
        "tot_freq=0\n",
        "\n",
        "for key,value in x_tokenizer.word_counts.items():\n",
        "    tot_cnt=tot_cnt+1\n",
        "    tot_freq=tot_freq+value\n",
        "    if(value<thresh):\n",
        "        cnt=cnt+1\n",
        "        freq=freq+value\n",
        "    \n",
        "print(\"% of rare words in vocabulary:\",(cnt/tot_cnt)*100)\n",
        "print(\"Total Coverage of rare words:\",(freq/tot_freq)*100)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "% of rare words in vocabulary: 0.0\n",
            "Total Coverage of rare words: 0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZTfZJTO26on",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#prepare a tokenizer for reviews on training data\n",
        "x_tokenizer = Tokenizer(num_words=tot_cnt-cnt) \n",
        "x_tokenizer.fit_on_texts(list(x_tr))\n",
        "\n",
        "#convert text sequences into integer sequences\n",
        "x_tr_seq    =   x_tokenizer.texts_to_sequences(x_tr) \n",
        "x_val_seq   =   x_tokenizer.texts_to_sequences(x_val)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6x3oNAgJ2_pZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#padding zero upto maximum length\n",
        "x_tr    =   pad_sequences(x_tr_seq,  maxlen=max_text_len, padding='post')\n",
        "x_val   =   pad_sequences(x_val_seq, maxlen=max_text_len, padding='post')\n",
        "\n",
        "#size of vocabulary ( +1 for padding token)\n",
        "x_voc   =  x_tokenizer.num_words + 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kjSQRw9Z3PBX",
        "colab_type": "code",
        "outputId": "74bccdbd-7f10-4005-fd38-344f3c7dff05",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x_voc"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "140"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VOLV4HYn3frR",
        "colab_type": "code",
        "outputId": "5523accf-271d-45a9-bdab-3421afffe96c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "x_val[10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 25,   1,   9,   8,  11,  72,  12,  87,  13, 103,  26,   1,   9,\n",
              "         8,  11,   2,  12,  23,  13,  16,  27,   1,   9,   8,  11,  40,\n",
              "        12,  33,  13,  23,   3,   6,   2,  19,  24,   2,  23,  28,   1,\n",
              "         9,   8,   3, 100,  29,   1,   9,   8,  11,   2,  12,   2,  13,\n",
              "         2,  14,   1,   9,   8,   3,   6,   2,   4,  10,   2,  17,  14,\n",
              "         1,   9,   5,   3,   6,   2,   4,  10,   2,  17,  14,   1,   9,\n",
              "        22,   3,   6,   2,   4,  10,   2,  17,  14,   1,   5,   8,   3,\n",
              "         6,   2,   4,  10,   2,  17,  14,   1,  22,   8,   3,   6,   2,\n",
              "         4,  10,   2,  17,  30,   1,   9,   8,  11,   2,  12,   2,  13,\n",
              "         2,   0,   0], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9iZzN_RY3hOW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#prepare a tokenizer for reviews on training data\n",
        "y_tokenizer = Tokenizer()   \n",
        "y_tokenizer.fit_on_texts(list(y_tr))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-zeUnYhs45sI",
        "colab_type": "code",
        "outputId": "357920e8-aabd-41d5-dc27-0c26aa2039ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "thresh=3\n",
        "\n",
        "cnt=0\n",
        "tot_cnt=0\n",
        "freq=0\n",
        "tot_freq=0\n",
        "\n",
        "for key,value in y_tokenizer.word_counts.items():\n",
        "    tot_cnt=tot_cnt+1\n",
        "    tot_freq=tot_freq+value\n",
        "    if(value<thresh):\n",
        "        cnt=cnt+1\n",
        "        freq=freq+value\n",
        "    \n",
        "print(\"% of rare words in vocabulary:\",(cnt/tot_cnt)*100)\n",
        "print(\"Total Coverage of rare words:\",(freq/tot_freq)*100)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "% of rare words in vocabulary: 9.090909090909092\n",
            "Total Coverage of rare words: 0.0070722033736321495\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FItKe8tS3lhU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#prepare a tokenizer for reviews on training data\n",
        "y_tokenizer = Tokenizer(num_words=tot_cnt-cnt) \n",
        "y_tokenizer.fit_on_texts(list(y_tr))\n",
        "\n",
        "#convert text sequences into integer sequences\n",
        "y_tr_seq    =   y_tokenizer.texts_to_sequences(y_tr) \n",
        "y_val_seq   =   y_tokenizer.texts_to_sequences(y_val) \n",
        "\n",
        "#padding zero upto maximum length\n",
        "y_tr    =   pad_sequences(y_tr_seq, maxlen=max_summary_len, padding='post')\n",
        "y_val   =   pad_sequences(y_val_seq, maxlen=max_summary_len, padding='post')\n",
        "\n",
        "#size of vocabulary\n",
        "y_voc  =   y_tokenizer.num_words +1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jH1tsX3F3ogc",
        "colab_type": "code",
        "outputId": "3aa14e92-469b-43e3-c4ee-15909f21ae60",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y_tokenizer.word_counts['sostok'],len(y_tr)   "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(20584, 20584)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LUly4xcs3_qz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !wget http://nlp.stanford.edu/data/glove.6B.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "biUF8sRdlDoY",
        "colab_type": "code",
        "outputId": "802cec0a-4df3-4a93-805a-7491773d38c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# !wget https://media.githubusercontent.com/media/allenai/spv2/master/model/glove.6B.100d.txt.gz\n",
        "!wget https://www.floydhub.com/api/v1/resources/Av2ThePYtAHXMAuSXEBV8X/glove.6B.100d.txt?content=true&rename=glove6b100dtxt"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-11 05:07:06--  https://www.floydhub.com/api/v1/resources/Av2ThePYtAHXMAuSXEBV8X/glove.6B.100d.txt?content=true\n",
            "Resolving www.floydhub.com (www.floydhub.com)... 104.26.1.30, 172.67.72.144, 104.26.0.30, ...\n",
            "Connecting to www.floydhub.com (www.floydhub.com)|104.26.1.30|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [application/octet-stream]\n",
            "Saving to: ‘glove.6B.100d.txt?content=true’\n",
            "\n",
            "glove.6B.100d.txt?c     [            <=>     ] 331.04M  35.3MB/s    in 11s     \n",
            "\n",
            "2020-06-11 05:07:17 (30.9 MB/s) - ‘glove.6B.100d.txt?content=true’ saved [347116733]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MrPUnBJiRzAw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !unzip glove.6B.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "okBG5clTTBlj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "EMBEDDING_DIM = 100"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HoYz7YbuSQz-",
        "colab_type": "code",
        "outputId": "d89728c3-faf2-4dd3-853f-c7e894f2baa6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "embeddings_index = {}\n",
        "f = open( 'glove.6B.100d.txt?content=true')\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "f.close()\n",
        "\n",
        "print('Found %s word vectors.' % len(embeddings_index))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 400000 word vectors.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l0GFEmmSUZso",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reverse_target_word_index=y_tokenizer.index_word\n",
        "reverse_source_word_index=x_tokenizer.index_word\n",
        "target_word_index=y_tokenizer.word_index\n",
        "word_index=x_tokenizer.word_index"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3GcDgXU2ShON",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
        "for word, i in word_index.items():\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        # words not found in embedding index will be all-zeros.\n",
        "        embedding_matrix[i] = embedding_vector"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L7xhJN9OYrlB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_matrix_1 = np.zeros((len(target_word_index) + 1, EMBEDDING_DIM))\n",
        "for word, i in target_word_index.items():\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        # words not found in embedding index will be all-zeros.\n",
        "        embedding_matrix_1[i] = embedding_vector"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FAb5miza5HWI",
        "colab_type": "code",
        "outputId": "2071fad8-41b1-4511-d2b3-9f54eeee37de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "!wget https://raw.githubusercontent.com/thushv89/attention_keras/master/layers/attention.py\n",
        "from attention import AttentionLayer"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-06-11 05:07:30--  https://raw.githubusercontent.com/thushv89/attention_keras/master/layers/attention.py\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 5230 (5.1K) [text/plain]\n",
            "Saving to: ‘attention.py’\n",
            "\n",
            "\rattention.py          0%[                    ]       0  --.-KB/s               \rattention.py        100%[===================>]   5.11K  --.-KB/s    in 0s      \n",
            "\n",
            "2020-06-11 05:07:30 (67.4 MB/s) - ‘attention.py’ saved [5230/5230]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "imulutl-XDQo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from math import log\n",
        "# from numpy import array\n",
        "# from numpy import argmax\n",
        " \n",
        "# # beam search\n",
        "# def beam_search_decoder(data, k):\n",
        "# \tsequences = [[list(), 0.0]]\n",
        "# \t# walk over each step in sequence\n",
        "# \tfor row in data:\n",
        "# \t\tall_candidates = list()\n",
        "# \t\t# expand each current candidate\n",
        "# \t\tfor i in range(len(sequences)):\n",
        "# \t\t\tseq, score = sequences[i]\n",
        "# \t\t\tfor j in range(len(row)):\n",
        "# \t\t\t\tcandidate = [seq + [j], score - log(row[j])]\n",
        "# \t\t\t\tall_candidates.append(candidate)\n",
        "# \t\t# order all candidates by score\n",
        "# \t\tordered = sorted(all_candidates, key=lambda tup:tup[1])\n",
        "# \t\t# select k best\n",
        "# \t\tsequences = ordered[:k]\n",
        "# \treturn sequences\n",
        " \n",
        "# # define a sequence of 10 words over a vocab of 5 words\n",
        "# data = [[0.1, 0.2, 0.3, 0.4, 0.5],\n",
        "# \t\t[0.5, 0.4, 0.3, 0.2, 0.1],\n",
        "# \t\t[0.1, 0.2, 0.3, 0.4, 0.5],\n",
        "# \t\t[0.5, 0.4, 0.3, 0.2, 0.1],\n",
        "# \t\t[0.1, 0.2, 0.3, 0.4, 0.5],\n",
        "# \t\t[0.5, 0.4, 0.3, 0.2, 0.1],\n",
        "# \t\t[0.1, 0.2, 0.3, 0.4, 0.5],\n",
        "# \t\t[0.5, 0.4, 0.3, 0.2, 0.1],\n",
        "# \t\t[0.1, 0.2, 0.3, 0.4, 0.5],\n",
        "# \t\t[0.5, 0.4, 0.3, 0.2, 0.1]]\n",
        "# data = array(data)\n",
        "# # decode sequence\n",
        "# result = beam_search_decoder(data, 3)\n",
        "# # print result\n",
        "# for seq in result:\n",
        "# \tprint(seq)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tkb0C6U85Hdl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Bi-directional model\n",
        "# from keras import backend as K \n",
        "# K.clear_session()\n",
        "\n",
        "# latent_dim = 512\n",
        "# embedding_dim=EMBEDDING_DIM\n",
        "\n",
        "# # Encoder\n",
        "# encoder_inputs = Input(shape=(max_text_len,))\n",
        "\n",
        "# #embedding layer\n",
        "# # enc_emb =  Embedding(x_voc, embedding_dim,trainable=True)(encoder_inputs)\n",
        "# enc_emb = Embedding(len(word_index) + 1,\n",
        "#                             EMBEDDING_DIM,\n",
        "#                             weights=[embedding_matrix],\n",
        "#                             input_length=MAX_SEQUENCE_LENGTH,\n",
        "#                             trainable=False)(encoder_inputs)\n",
        "\n",
        "# # embedding_layer = Embedding(num_words,\n",
        "# #                             EMBEDDING_DIM,\n",
        "# #                             embeddings_initializer=Constant(embedding_matrix),\n",
        "# #                             input_length=MAX_SEQUENCE_LENGTH,\n",
        "# #                             trainable=False)\n",
        "\n",
        "# # # Define Keras embedding layer with the correct output/input sizes, make it trainable. Use Embedding(...). Make sure to set trainable=False. \n",
        "# #     embedding_layer = Embedding(vocab_len, emb_dim, trainable = False)\n",
        "# #     ### END CODE HERE ###\n",
        "\n",
        "# #     # Build the embedding layer, it is required before setting the weights of the embedding layer. Do not modify the \"None\".\n",
        "# #     embedding_layer.build((None,))\n",
        "    \n",
        "# #     # Set the weights of the embedding layer to the embedding matrix. Your layer is now pretrained.\n",
        "# #     embedding_layer.set_weights([emb_matrix])\n",
        "\n",
        "# #encoder lstm 1\n",
        "\n",
        "# encoder_lstm = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.3,recurrent_dropout=0.2)\n",
        "\n",
        "# encoder_lstm1 = Bidirectional(encoder_lstm)\n",
        "\n",
        "# # encoder_output1, state_h1, state_c1 = encoder_lstm1(enc_emb)\n",
        "\n",
        "# # #encoder lstm 2\n",
        "# # encoder_lstm2 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.3,recurrent_dropout=0.2)\n",
        "# # encoder_output2, state_h2, state_c2 = encoder_lstm2(encoder_output1)\n",
        "\n",
        "# # # #encoder lstm 3\n",
        "# # encoder_lstm3=LSTM(latent_dim, return_state=True, return_sequences=True,dropout=0.3,recurrent_dropout=0.2)\n",
        "# # encoder_outputs, state_h, state_c = encoder_lstm1(enc_emb)\n",
        "# encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder_lstm1(enc_emb)\n",
        "\n",
        "# # lstm, forward_h, forward_c, backward_h, backward_c = Bidirectional(LSTM(128, return_sequences=True, return_state=True))(input)\n",
        "# state_h = Concatenate()([forward_h, backward_h])\n",
        "# state_c = Concatenate()([forward_c, backward_c])\n",
        "# # encoder_states = [state_h, state_c]\n",
        "\n",
        "# # forward_layer = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "# # backward_layer = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4, go_backwards=True)\n",
        "# # encoder_lstm = Bidirectional(forward_layer, backward_layer=backward_layer)\n",
        "# # encoder_outputs, state_h, state_c= encoder_lstm(enc_emb)\n",
        "\n",
        "\n",
        "# # Set up the decoder, using `encoder_states` as initial state.\n",
        "# decoder_inputs = Input(shape=(None,))\n",
        "\n",
        "# #embedding layer\n",
        "# # dec_emb_layer = Embedding(y_voc, embedding_dim,trainable=True)\n",
        "\n",
        "# dec_emb_layer = Embedding(len(target_word_index) + 1,\n",
        "#                             EMBEDDING_DIM,\n",
        "#                             weights=[embedding_matrix_1],\n",
        "#                             input_length=max_summary_len,\n",
        "#                             trainable=False)\n",
        "\n",
        "# dec_emb = dec_emb_layer(decoder_inputs)\n",
        "\n",
        "\n",
        "# decoder_lstm = LSTM(latent_dim* 2, return_sequences=True, return_state=True,dropout=0.3,recurrent_dropout=0.2)\n",
        "# decoder_outputs,decoder_fwd_state, decoder_back_state = decoder_lstm(dec_emb,initial_state=[state_h, state_c])\n",
        "\n",
        "# # Attention layer\n",
        "# attn_layer = AttentionLayer(name='attention_layer')\n",
        "# attn_out, attn_states = attn_layer([encoder_outputs, decoder_outputs])\n",
        "\n",
        "# # Concat attention input and decoder LSTM output\n",
        "# decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attn_out])\n",
        "\n",
        "# #dense layer\n",
        "# decoder_dense =  TimeDistributed(Dense(y_voc, activation='softmax'))\n",
        "# decoder_outputs = decoder_dense(decoder_concat_input)\n",
        "\n",
        "# # Define the model \n",
        "# model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "\n",
        "# model.summary() "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfizOahJc7qR",
        "colab_type": "code",
        "outputId": "9571bb71-9989-44c3-83a5-4125d4d64318",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 581
        }
      },
      "source": [
        "from keras import backend as K \n",
        "K.clear_session()\n",
        "\n",
        "latent_dim = 128\n",
        "embedding_dim=EMBEDDING_DIM\n",
        "\n",
        "# Encoder\n",
        "encoder_inputs = Input(shape=(max_text_len,))\n",
        "\n",
        "#embedding layer\n",
        "enc_emb = Embedding(len(word_index) + 1,\n",
        "                            EMBEDDING_DIM,\n",
        "                            weights=[embedding_matrix],\n",
        "                            input_length=MAX_SEQUENCE_LENGTH,\n",
        "                            trainable=False)(encoder_inputs)\n",
        "\n",
        "#encoder lstm 1\n",
        "\n",
        "encoder_lstm1 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.3,recurrent_dropout=0.2)\n",
        "# encoder_output1, state_h1, state_c1 = encoder_lstm1(enc_emb)\n",
        "\n",
        "# #encoder lstm 2\n",
        "# encoder_lstm2 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.3,recurrent_dropout=0.2)\n",
        "# encoder_output2, state_h2, state_c2 = encoder_lstm2(encoder_output1)\n",
        "\n",
        "# # #encoder lstm 3\n",
        "# encoder_lstm3=LSTM(latent_dim, return_state=True, return_sequences=True,dropout=0.3,recurrent_dropout=0.2)\n",
        "encoder_outputs, state_h, state_c = encoder_lstm1(enc_emb)\n",
        "\n",
        "# forward_layer = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "# backward_layer = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4, go_backwards=True)\n",
        "# encoder_lstm = Bidirectional(forward_layer, backward_layer=backward_layer)\n",
        "# encoder_outputs, state_h, state_c= encoder_lstm(enc_emb)\n",
        "\n",
        "\n",
        "# Set up the decoder, using `encoder_states` as initial state.\n",
        "decoder_inputs = Input(shape=(None,))\n",
        "\n",
        "#embedding layer\n",
        "# dec_emb_layer = Embedding(y_voc, embedding_dim,trainable=True)\n",
        "\n",
        "dec_emb_layer = Embedding(len(target_word_index) + 1,\n",
        "                            EMBEDDING_DIM,\n",
        "                            weights=[embedding_matrix_1],\n",
        "                            input_length=max_summary_len,\n",
        "                            trainable=False)\n",
        "\n",
        "dec_emb = dec_emb_layer(decoder_inputs)\n",
        "\n",
        "\n",
        "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True,dropout=0.3,recurrent_dropout=0.2)\n",
        "decoder_outputs,decoder_fwd_state, decoder_back_state = decoder_lstm(dec_emb,initial_state=[state_h, state_c])\n",
        "\n",
        "# Attention layer\n",
        "attn_layer = AttentionLayer(name='attention_layer')\n",
        "attn_out, attn_states = attn_layer([encoder_outputs, decoder_outputs])\n",
        "\n",
        "# Concat attention input and decoder LSTM output\n",
        "decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attn_out])\n",
        "\n",
        "#dense layer\n",
        "decoder_dense =  TimeDistributed(Dense(y_voc, activation='softmax'))\n",
        "decoder_outputs = decoder_dense(decoder_concat_input)\n",
        "\n",
        "# Define the model \n",
        "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "\n",
        "model.summary() "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
            "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 120)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding (Embedding)           (None, 120, 100)     14000       input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, None, 100)    30900       input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm (LSTM)                     [(None, 120, 128), ( 117248      embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   [(None, None, 128),  117248      embedding_1[0][0]                \n",
            "                                                                 lstm[0][1]                       \n",
            "                                                                 lstm[0][2]                       \n",
            "__________________________________________________________________________________________________\n",
            "attention_layer (AttentionLayer ((None, None, 128),  32896       lstm[0][0]                       \n",
            "                                                                 lstm_1[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "concat_layer (Concatenate)      (None, None, 256)    0           lstm_1[0][0]                     \n",
            "                                                                 attention_layer[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "time_distributed (TimeDistribut (None, None, 281)    72217       concat_layer[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 384,509\n",
            "Trainable params: 339,609\n",
            "Non-trainable params: 44,900\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QiCiMRV2vEuH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from tensorflow import keras\n",
        "# opt = keras.optimizers.Adam(learning_rate=0.01, beta_1=0.9, beta_2=0.999, decay=0.01)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ky67YJi55HaB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "biq7Lbhk5led",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=2)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YorSwH0E5mqH",
        "colab_type": "code",
        "outputId": "61985f72-4b49-4078-a472-a930179a6c72",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "history=model.fit([x_tr,y_tr[:,:-1]], y_tr.reshape(y_tr.shape[0],y_tr.shape[1], 1)[:,1:] ,epochs=3,callbacks=[es],batch_size=64, validation_data=([x_val,y_val[:,:-1]], y_val.reshape(y_val.shape[0],y_val.shape[1], 1)[:,1:]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "322/322 [==============================] - 243s 755ms/step - loss: 0.1000 - accuracy: 0.9636 - val_loss: 0.1138 - val_accuracy: 0.9606\n",
            "Epoch 2/3\n",
            "322/322 [==============================] - 242s 752ms/step - loss: 0.0967 - accuracy: 0.9645 - val_loss: 0.1127 - val_accuracy: 0.9614\n",
            "Epoch 3/3\n",
            "322/322 [==============================] - 241s 750ms/step - loss: 0.0938 - accuracy: 0.9657 - val_loss: 0.1131 - val_accuracy: 0.9619\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OLtxX4uhavmi",
        "colab_type": "code",
        "outputId": "fc733370-2d71-4612-ec02-dca071e689f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# model_save_name = 'weather_summ_unidir.h5'\n",
        "# path = F\"/content/gdrive/My Drive/Resume DL Projects/{model_save_name}\" \n",
        "# # torch.save(model.state_dict(), path)\n",
        "# model.save(path)\n",
        "# print(\"Saved model to disk\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saved model to disk\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vc4YJuXZbwio",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # save model and architecture to single file\n",
        "# model.save(\"model.h5\")\n",
        "# print(\"Saved model to disk\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "28awpsmOb0kL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.models import load_model\n",
        " \n",
        "# # load model\n",
        "# model = load_model('model.h5')\n",
        "# # summarize model.\n",
        "# model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FzTfjASjWzHW",
        "colab_type": "code",
        "outputId": "a001a9c4-a4f5-4518-b481-f1b053d047b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model_save_name = 'unidir_model.h5'\n",
        "path = F\"/content/gdrive/My Drive/Resume DL Projects/{model_save_name}\" \n",
        "model.save(path)\n",
        "print(\"Saved model to disk\")\n",
        "\n",
        "# model_save_name = 'weather_summ_unidir.h5'\n",
        "# path = F\"/content/gdrive/My Drive/Resume DL Projects/{model_save_name}\" \n",
        "# model = load_model(path, custom_objects={'AttentionLayer': AttentionLayer})\n",
        "# # # # summarize model.\n",
        "# model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saved model to disk\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-40wYLAH5vRZ",
        "colab_type": "code",
        "outputId": "085a1e1b-b280-4bcb-d0da-acdf4716502c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 268
        }
      },
      "source": [
        "from matplotlib import pyplot\n",
        "pyplot.plot(history.history['loss'], label='train')\n",
        "pyplot.plot(history.history['val_loss'], label='test')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD7CAYAAABkO19ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deVjWVf7/8eebXRZFARdAFHdwV8TdLLVwSTMrtWwx06ayqWmZsWmaZpqpb79qGitbNNN2rcxKS80lTS0VEXdcwA3BBdxFkPX8/vhgg+aCsnzu++b9uC6uvD8L95v7yhfHc87nHDHGoJRSynW52V2AUkqpiqVBr5RSLk6DXimlXJwGvVJKuTgNeqWUcnEa9Eop5eJKFfQiEiciO0QkRUQmXOT8EyKSJCKbRGSJiDQoce4VEdkqIttE5E0RkfL8AZRSSl3eFYNeRNyBt4H+QDQwUkSiL7hsPRBjjGkDzAJeKb63G9AdaAO0AjoB15Vb9Uoppa7IoxTXxAIpxpjdACIyExgCJJ27wBiztMT1q4FR504BPoAXIIAncPhybxYcHGwaNmxYyvKVUkoBrFu37ogxJuRi50oT9GHA/hKv04DOl7l+DDAfwBizSkSWAgexgn6SMWbb5d6sYcOGJCQklKIspZRS54jIvkudK9fBWBEZBcQArxa/bgJEAeFYvzBuEJGeF7lvnIgkiEhCZmZmeZaklFJVXmmCPh2oX+J1ePGx84hIX+BZYLAxJrf48FBgtTEmyxiThdXS73rhvcaYKcaYGGNMTEjIRf/loZRS6hqVJujXAk1FJFJEvIARwJySF4hIe2AyVshnlDiVClwnIh4i4ok1EHvZrhullFLl64p99MaYAhEZD/wIuAPTjDFbReQFIMEYMwerq8Yf+Kp49mSqMWYw1gycG4DNWAOzC4wxcyvmR1FKVWX5+fmkpaVx9uxZu0upUD4+PoSHh+Pp6Vnqe8TRlimOiYkxOhirlLpae/bsISAggKCgIFz1cR1jDEePHuX06dNERkaed05E1hljYi52nz4Zq5RyCWfPnnXpkAcQEYKCgq76Xy0a9Eopl+HKIX/OtfyMLhP0hUWGl+ZtI+14tt2lKKWUQ3GZoE89ls2M+FSGT17NvqNn7C5HKVXFnDhxgnfeeeeq7xswYAAnTpyogIr+x2WCPjLYjxlju3Amr4Dhk1ezKzPL7pKUUlXIpYK+oKDgsvfNmzePwMDAiioLcKGgB2gVVoOZ47qQX1jE8Mmr2Xn4tN0lKaWqiAkTJrBr1y7atWtHp06d6NmzJ4MHDyY62loD8pZbbqFjx460bNmSKVOm/HZfw4YNOXLkCHv37iUqKoqxY8fSsmVLbrzxRnJycsqlNpecXpmScZo7319DQZHhkzGxtAytUU7VKaUc1bZt24iKigLgn3O3knTgVLl+/+jQ6jx/c8tLnt+7dy+DBg1iy5YtLFu2jIEDB7Jly5bfpkEeO3aMWrVqkZOTQ6dOnfj5558JCgr6bX2vrKwsmjRpQkJCAu3ateOOO+5g8ODBjBo16nfvVfJnPafKTa9sUjuALx/sio+HG3e+v4aN+yu2/0sppS4UGxt73lz3N998k7Zt29KlSxf2799PcnLy7+6JjIykXbt2AHTs2JG9e/eWSy2lWb3SKTUM9uOLB7ty59TVjJq6hg/v70THBrXsLkspVQku1/KuLH5+fr/9edmyZSxevJhVq1bh6+tL7969LzoX3tvb+7c/u7u7l1vXjUu26M+pX8uXLx/sSnCAN3d/EM/q3UftLkkp5aICAgI4ffri44InT56kZs2a+Pr6sn37dlavXl2ptbl00APUq1GNL8Z1ISywGvdNj2dFsi6DrJQqf0FBQXTv3p1WrVrx9NNPn3cuLi6OgoICoqKimDBhAl26dKnU2lxyMPZijmblMuqDeHZlZvHeqA7c0KJOub+HUso+FxugdFU6GHsJQf7ezBjbmeZ1Anjwk3Us2HLI7pKUUqpSVJmgBwj09eKzsZ1pHVaDRz5PZO7GA3aXpJRSFa5KBT1AdR9PPh7TmY4NavLYzPV8vS7N7pKUUqpCVbmgB/D39uCj0bF0axzMU7M2MiM+1e6SlFKqwlTJoAeo5uXO1Htj6N0shGdmb+ajX/faXZJSSlWIKhv0AD6e7rx3d0dujK7D83O28v7y3XaXpJRS5a5KBz2At4c7b9/VgYFt6vHivG1M+un3jyUrpdSVXOsyxQATJ04kO7vi9tKo8kEP4OnuxhvD23Fr+zBeW7iT/yzcgaM9X6CUcmyOHPQuu9bN1fJwd+PV29vi5eHGWz+lkFdQxIT+LarE1mRKqbIruUxxv379qF27Nl9++SW5ubkMHTqUf/7zn5w5c4Y77riDtLQ0CgsLee655zh8+DAHDhzg+uuvJzg4mKVLl5Z7bRr0Jbi7CS8NbY2XhxuTl+8mt6CI52+O1rBXytnMnwCHNpfv96zbGvq/fMnTL7/8Mlu2bGHDhg0sXLiQWbNmER8fjzGGwYMHs3z5cjIzMwkNDeWHH34ArDVwatSoweuvv87SpUsJDg4u35qLadBfwM1N+Ofglni5uzF15R5yC4p48ZZWuLlp2CulSmfhwoUsXLiQ9u3bA5CVlUVycjI9e/bkySef5C9/+QuDBg2iZ8+elVKPBv1FiAjPDozC29ONt5fuIq+giFdua4O7hr1SzuEyLe/KYIzhmWee4cEHH/zducTERObNm8ff/vY3+vTpw9///vcKr0cHYy9BRHj6phY80a8ZXyem8fgXG8gvLLK7LKWUgyq5TPFNN93EtGnTyMqy9q5OT08nIyODAwcO4Ovry6hRo3j66adJTEz83b0VQVv0V/DHPk3x8nDj5fnbySso5K2RHfDy0N+PSqnzlVymuH///tx555107doVAH9/fz799FNSUlJ4+umncXNzw9PTk3fffReAcePGERcXR2hoaIUMxlaZZYrLavove/jn3CRuaFGbd+7qgI+nu90lKaVK0GWKdZniMhvdPZIXh7bip+0ZjP04gZy8QrtLUkqpUtGgvwp3dW7Aq7e1YWXKEe6bHs+Z3AK7S1JKqSsqVdCLSJyI7BCRFBGZcJHzT4hIkohsEpElItKgxLkIEVkoItuKr2lYfuVXvttj6jNxeDsS9h3nnmnxnDqbb3dJSqlijtYVXRGu5We8YtCLiDvwNtAfiAZGikj0BZetB2KMMW2AWcArJc59DLxqjIkCYoGMq67SwQxpF8akke3ZuP8Eo6au4UR2nt0lKVXl+fj4cPToUZcOe2MMR48excfH56ruK82sm1ggxRizG0BEZgJDgKQSb15ymHg1MKr42mjAwxizqPi6rKuqzoH1b12P99zdePizRO58fw2fjIklyN/b7rKUqrLCw8NJS0sjMzPT7lIqlI+PD+Hh4Vd1T2mCPgzYX+J1GtD5MtePAeYX/7kZcEJEZgORwGJggjHmvJFMERkHjAOIiIgoXeUOoG90Hd6/N4ZxHycw8v3VfPpAZ2oHXN1vWqVU+fD09CQyMtLuMhxSuQ7GisgoIAZ4tfiQB9ATeAroBDQC7rvwPmPMFGNMjDEmJiQkpDxLqnDXNQth+uhO7D+Ww4jJqzl08qzdJSml1HlKE/TpQP0Sr8OLj51HRPoCzwKDjTG5xYfTgA3GmN3GmALgW6BD2Up2PN0aB/PxmFgyTudyx+RVpB2vuOVGlVLqapUm6NcCTUUkUkS8gBHAnJIXiEh7YDJWyGdccG+giJxrpt9Aib59V9KpYS0+faAzJ7LzGD55NTsPV9zjzEopdTWuGPTFLfHxwI/ANuBLY8xWEXlBRAYXX/Yq4A98JSIbRGRO8b2FWN02S0RkMyDA+xXwcziEdvUD+XxsF3ILihj69i/8uPWQ3SUppZQugVARDp7M4Q+frGNj2kn+2Kcpj/dpqsscK6UqlC6BUMnq1ajGFw92ZViHcN5cksy4TxI4rQ9WKaVsokFfQXw83Xnt9jb84+Zolu7I5Ja3f2FXpss8RqCUciIa9BVIRLiveySfjunM8ex8bpn0Cz9tP2x3WUqpKkaDvhJ0bRzEnPHdiQjyZcxHCUz6KdmlH9NWSjkWDfpKEl7Tl1l/6MbgtqG8tnAnD3+WqKtfKqUqhesEfUEefPMHOLzV7kouqZqXOxOHt+PZAVH8uPUQQ9/5hX1Hz9hdllLKxblO0J9Kh93LYFoc7P7Z7mouSUQY26sRH91vPUl781sr+Xmnay/CpJSyl+sEfa1IeGAxVA+DT4fBxpl2V3RZPZuGMOeRHoQGVmP09Hgm/7xL++2VUhXCdYIeoEY43L8AIrrANw/C8tfAgcMzIsiX2Q93o3+revzf/O38ceYG3aJQKVXuXCvoAaoFwqjZ0PoO+Olf8P3jUOi4g56+Xh5MurM9f45rzvebDjDs3V/Zf0wXRVNKlR/XC3oADy+4dQr0eALWfQgzR0Ku4z6sJCI83LsJ0+7rxP7j2QyetJJfU47YXZZSykW4ZtADiEDf52HQfyFlMXw4EE479sNK1zevzZzxPQjy9+buafF8sHKP9tsrpcrMdYP+nJj7YcQMOLITPugLmTvtruiyIoP9+PaR7vRpUZt/fZ/Ek19t5Gy+9tsrpa6d6wc9QPM4uO8HyM+BD/rBvl/truiy/L09eG9UR/7UtxmzE9O5Y/IqDpzIsbsspZSTqhpBDxDWAcYsAr9g+PgW2PqN3RVdlpub8Fjfprx/Twy7M88weNJK1uw+andZSiknVHWCHqy59mMWQWh7+Oo++PUth55+CdAvug7fPtKNAB9P7pq6hk9W7dV+e6XUValaQQ/gWwvu+Q6ih8DCv8H8v0CRY/eBN6kdwLePdKdn02Ce+24rE77eTG6BY9eslHIcVS/oATx94LYPocsjED8ZvrzH6r93YDWqeTL13k6Mv74JXyTsZ8SU1Rw+ddbuspRSTqBqBj2AmxvEvQRxL8P2H+Cjm+GMY89dd3cTnrqpOe/c1YEdh04z6K2VrNt33O6ylFIOruoG/TldHoI7PoZDm60ZOUd32V3RFQ1oXY/ZD3ejmqc7I6asYmZ8qt0lKaUcmAY9QPRguGcO5Jywwj7N8Tcnb1G3OnPGd6dLoyAmzN7M377dTF5Bkd1lKaUckAb9ORGdrRk53gHw4SCrO8fBBfp6Mf2+TjzYqxGfrk7lrqmryTyda3dZSikHo0FfUnATGLMY6kTDzLsg/n27K7oiD3c3nhkQxRsj2rE5/SQ3v7WSjftP2F2WUsqBaNBfyD8E7p0LzeJg3lOw8DkocvwukSHtwvj6oW64uwm3T17F1+vS7C5JKeUgNOgvxssPRnwGMWPg1zfh6zGQ7/hTGVuG1mDuoz3oGFGTJ7/ayD/mbNX59kopDfpLcnOHgf+Bvv+ErbPh01sh+5jdVV1RLT8vPhkTy+juDfnw170MfftXUjJO212WUspGGvSXIwI9HodhH0DaWms/2hOOP5XRw92N529uyZS7O3Lo1FkGvrmSj1fp0glKVVUa9KXR+ja4+xvIOgRT+8KBDXZXVCo3tqzLgsd70qVREH//biv3f7hWZ+UoVQWVKuhFJE5EdohIiohMuMj5J0QkSUQ2icgSEWlwwfnqIpImIpPKq/BK17AH3P8juHvB9AGQvMjuikqldoAPH47uxD9ujuaXXUeJm7icxUmOvQGLUqp8XTHoRcQdeBvoD0QDI0Uk+oLL1gMxxpg2wCzglQvO/wtYXvZybVY7ypprH9QIPh8O6z6yu6JSERHu6x7J94/2oHZ1Hx74OIG/frOZ7DzH3UtXKVV+StOijwVSjDG7jTF5wExgSMkLjDFLjTHndrReDYSfOyciHYE6wMLyKdlm1evB6PnQqDfM/SP89KLDL3V8TrM6AXz7SDfG9WrE52tSGfTWSjannbS7LKVUBStN0IcB+0u8Tis+diljgPkAIuIG/Ad46loLdEjeAXDnF9B+FCx/Bb59CAry7K6qVLw93PnrgCg+f6Az2bmFDH3nF95ZlkJhkXP8slJKXb1yHYwVkVFADPBq8aGHgXnGmMs+vSMi40QkQUQSMjMzy7OkiuPuCYMnQe+/wsYZ8PntcNZ5WsfdmgSz4PGe3NSyLq8s2MHI91eTrtsVKuWSShP06UD9Eq/Di4+dR0T6As8Cg40x56Z2dAXGi8he4DXgHhF5+cJ7jTFTjDExxpiYkJCQq/wRbCQCvf8CQ96BvSutQdqTv/toHFagrxeT7mzPa7e3ZWv6SeImLue7Dc5Tv1KqdORKc6tFxAPYCfTBCvi1wJ3GmK0lrmmPNQgbZ4xJvsT3uQ9rwHb85d4vJibGJCQ4/uqRv5OyBL681+rWGTUL6rS0u6Krsu/oGf70xQYSU09wS7tQXrilFdV9PO0uSylVSiKyzhgTc7FzV2zRG2MKgPHAj8A24EtjzFYReUFEBhdf9irgD3wlIhtEZE451e48mvSB0fMAYz1YlbzY7oquSoMgP758sCuP923K3E0H6T9xBfF7HP9JYKXUlV2xRV/ZnLZFf87JNGvqZUaStXxCt0etLh4nkph6nMdnbiDteDYP9W7MY32a4eWhz9Yp5cjK1KJXV6lGOIxZCC0GwaLn4JsHnWJBtJI6RNRk3mM9ua1jOG8v3cWwd39lV2aW3WUppa6RBn1F8PKD2z+yZuRs+gI+HACnDtpd1VXx9/bgldva8t6oDuw/ns2gN1fy2Zp9ul6OUk5Ig76iuLlZM3KGfwoZ22FKb6fYovBCca3qseCxXnRsUJNnv9nC2I/XcTRL18tRyplo0Fe0qJvhgUXg4W1Nv9www+6KrlrdGj58fH8sfxsYxfKdmdw0cQVLd2TYXZZSqpQ06CtDnZYwdinUj4Vv/wA/PgtFzrUhiJub8EDPRnw3vjtBfl6Mnr6W57/bwtl85/o5lKqKNOgri1+QtdRxp7GwahJ8fgfkON/erlH1qvPd+O6M7t6Qj1bt4+a3VrL1gPM8EaxUVaRBX5ncPWHgazBoIuxeBlP7wJGLPl/m0Hw83Xn+5pZ8dH8sJ3LyGfr2r0xZvosiXS9HKYekQW+HmNHWBuQ5J+D9G2Cncy7seV2zEH58vBe9m4fw0rztjPpgDQdP6no5SjkaDXq7NOgG45ZCzQZWN87KiU6z3HFJtfy8mHx3R16+tTXrU08QN3EF8zY711RSpVydBr2dAiOsXauih8Di52H2OMh3vhaxiDAiNoJ5j/WkYZAvD3+WyFNfbeT02Xy7S1NKoUFvPy8/uP1DuP5vsPlLmN4fTh2wu6prEhnsx6yHuvHoDU2YnZjGgDdXsG6frpejlN006B2BCFz3NIz43BqcndIb9q+1u6pr4unuxpM3NueLB7tiDNz+3ipeX7ST/MIiu0tTqsrSoHckLQZae9J6VrOWTdjwud0VXbNODWsx77Ge3NIujDeXJHPrO7+SfPi03WUpVSVp0DuaOtHWw1URXawtCn98FgqdcxPv6j6evD68He/e1YG049kMfGsl7y/frdsWKlXJNOgdkW8tGDUbYh8sfrjqdsg5bndV16x/63os/NN19GoawovztjHy/dXsP5Z95RuVUuVCg95RuXvCgFfg5jdhzwp4vw9k7rC7qmsWEuDN+/d05NXb2pB04BRxE5czMz5VV8NUqhJo0Du6jvdaD1flnrLCfuePdld0zUSE22Pqs+DxnrQJD2TC7M2M+SiBjFPOtV6/Us5Gg94ZNOhq9dvXirR2r1r5X6d8uOqc8Jq+fPZAZ56/OZpfUo5w48TlfL/JOaeUKuUMNOidRWB96+GqlkNh8T9g9linfLjqHDc3YXT3SH74Y08a1PJl/OfreXTGek5k59ldmlIuR4PemXj5wm3T4IbnYPMsaxPyk+l2V1UmTWr78/VD3XiyXzPmbz7Ijf9drmvdK1XONOidjQj0egpGzoCjKcUPV8XbXVWZeLi78Wifpnz7SHcCfT0ZPX0tz8zezJlc55xWqpSj0aB3Vs37wwOLrSUUPhwI6z+1u6IyaxVWgznje/Bgr0bMXJtK3BvLid+jSygoVVYa9M6sdhSM/QkiusJ3j8CCZ5z24apzfDzdeWZAFF+M6wrA8CmreGneNt3JSqky0KB3ducerur8B1j9Dnx2G2Q7fys4NrIW8x/rxYhOEUxZvpvBk1ayJV13slLqWmjQuwJ3D+j//2DwJNi70trMJGO73VWVmb+3B/93a2umj+7Eiex8bnn7F95ckkyBLpCm1FXRoHclHe6G+36AvDMwtS/smG93ReXi+ua1WfinXgxoXY/XF+1k2Lu/kpKRZXdZSjkNDXpXE9HZ2rkqqDHMGAHf/AGyMu2uqswCfb14c2R7Jt3Znn3Hshn45gqm/7JH96lVqhQ06F1RjXC4fwH0fNKabz8pBhKmQZHzd3kMahPKwsd70a1xEP+cm8RdU9eQdlwXSFPqcjToXZVnNejzd3joF6jbGr7/E3zQDw5utLuyMqtd3Ydp93Xi5VtbsynN2qf2y4T9ukCaUpdQqqAXkTgR2SEiKSIy4SLnnxCRJBHZJCJLRKRB8fF2IrJKRLYWnxte3j+AuoKQ5taiaEOnwIl91gNW8yfA2VN2V1Ym5/apXfB4L6JDq/PnWZsY+/E6Mk/n2l2aUg5HrtQKEhF3YCfQD0gD1gIjjTFJJa65HlhjjMkWkYeA3saY4SLSDDDGmGQRCQXWAVHGmBOXer+YmBiTkJBQ5h9MXUTOcVjyL6sbx78OxP2ftXaOiN2VlUlRkWHaL3t45ccd+Ht78OItrejfup7dZSlVqURknTEm5mLnStOijwVSjDG7jTF5wExgSMkLjDFLjTHnOkpXA+HFx3caY5KL/3wAyABCru3HUGVWrSYMeh0eWAIBdWDWaPj0Vji6y+7KysTNTXigZyN+eLQHoYE+PPRZIn/6YgMnc/LtLk0ph1CaoA8D9pd4nVZ87FLGAL+b1ycisYAX4Nyp4grCO1rLHvd/xdqE/J2usOxlyHfudeGb1gngm4e781ifpszZeICb/ruc5Tudf8aRUmVVroOxIjIKiAFeveB4PeATYLQx5ndTP0RknIgkiEhCZqb+xawUbu7Q+UF4NAGiBsGy/4N3u8Kun+yurEw83d34U79mfPNwN/x9PLhnWjzPfbuF7DznXhpCqbIoTdCnA/VLvA4vPnYeEekLPAsMNsbkljheHfgBeNYYs/pib2CMmWKMiTHGxISEaM9OpQqoay19fPc3gMAnQ+Gr0XDqoN2VlUmb8EC+f7QHY3pE8umafQx4Y4UukKaqrNIE/VqgqYhEiogXMAKYU/ICEWkPTMYK+YwSx72Ab4CPjTGzyq9sVe4a3wAP/Qq9/wrbf4BJnWD1e069SJqPpzvPDYrm8we6UFBkuGPyKsZ/nkj6CefdsEWpa3HFWTcAIjIAmAi4A9OMMS+KyAtAgjFmjogsBloD55qBqcaYwcVdOdOBrSW+3X3GmA2Xei+ddeMAju6CeU/DriVQtw0Mmmj16zux7LwC3vt5N5N/toaIHuzViD/0boyvl4fNlSlVPi4366ZUQV+ZNOgdhDGQ9K219PHpQxAz2noAq1pNuysrk/QTObw8fztzNx6gbnUfJvRvwZB2oYiTTzFVSoNeXbuzp6yB2jXvgW8Q3PhvaDPc6efer917jBfmJrE5/STtIwJ5/uaWtKsfaHdZSl0zDXpVdgc3wQ9PQNpaaNgTBv7HeurWiRUVGWYlpvHqjzvIPJ3Lre3D+HNcC+rW8LG7NKWumga9Kh9FRZD4ESz+h7UUcrdHodfT1qblTiwrt4C3l6bwwYo9uLsJD/duzNhejfDxdLe7NKVKTYNela+sTFj0d9j4OdSIgAGvQvM4u6sqs9Sj2bw0bxsLth4iLLAafx0QxYDWdbX/XjmFsi6BoNT5/ENg6Ltw3zyrNT9jOMy8C07sv/K9DiwiyJf37u7I52M7E+DjwSOfJzJ88mrdwlA5PW3Rq7IpyIPVb8PPr1ivr/sLdH0E3D3trauMCosMX6zdz2sLd3A8O487OtbnqZuaExLgbXdpSl2Udt2oinciFeb/BXbMg5Aoa/G0Bt3srqrMTubk89aSZD78dS8+nu6Mv6EJo7s3xNtD+++VY9GgV5Vn+zyY/2c4uR/a3QX9XgC/YLurKrPdmVm8+MM2lmzPoEGQL38dEMWN0XW0/145DA16VbnyzlhdOasmgXcA9P0ntL8b3Jx/SGj5zkz+9X0SyRlZdG8SxHODomlRt7rdZSmlQa9skrENfngS9v0CDbpbSymENLO7qjIrKCziszWpvL5oJ6fP5nNn5wie6NecWn5edpemqjANemUfY2D9p7Dwb5CfDT2fgh6Pg4fzD2oeP5PHxMU7+XRNKn5e7jzWtxn3dG2Ap7vz/8tFOR8NemW/rAxr3ZwtsyC4Gdz8hksM1gLsPHyaf32fxIrkIzQK8eO5gdFc36K23WWpKkbn0Sv7+deG2z6Au2ZZO1lN7w9zH4OcS24f7DSa1Qng4/tj+eDeGIyB0R+u5d5p8aRknLa7NKUAbdErO+SdgaUvwep3wC8E+v8/iL7F6RdKA8grKOLjVXt5Y0kyOXmF3N21AY/3aUYNX+d+rkA5Pu26UY7pwAaY+0c4uBGaxcGA1yCw/pXvcwJHs3L5z6KdzIxPpUY1T57o14yRsRF4aP+9qiAa9MpxFRZYSyAvfREQ6PMcxI6z9rR1AUkHTvHC91tZvfsYzesE8NygaHo0df7nCpTj0aBXju/4PmsqZsoiCO1gDdbWa2N3VeXCGMOPWw/x4rxt7D+WQ0yDmoyMjWBgm3q6QqYqNxr0yjkYA1u+hgUTIPsYdBsP101w+mWQzzmbX8inq/fx2ZpU9hw5Q3UfD4a2D2Nk5wh96EqVmQa9ci7Zx6xlkNd/AoENYNB/oUkfu6sqN8YYVu8+xoz4VBZsOUReYRHtIwIZ2SmCQW3r6T626ppo0CvntHelNQXzaIq1feFNL7nEujklHTuTx+zENGbEp7Ir8wwB3h4MaR/KiE4RtAqrYXd5yolo0CvnlX8WVvwHVv4XvP2tsG870iWmYpZkjCFh33FmrEnlh80HyS0ook14DUZ0imBwu1D8vbWVry5Pg145v4ztVut+/2qI7GWtmxPU2O6qKsTJ7Hy+WZ/GjPj97Dh8Gl8vd4a0s1r5baPFBSQAABMcSURBVMJr6IqZ6qI06JVrKCqCxA9h0fNQmAfX/Rm6/dHpNzm5FGMM6/efYMaaVOZuOsDZ/CKi61VnZOcIhrQLpbqPa/7c6tpo0CvXcuogLPgLJH0HtaPh5jehfie7q6pQp87m892GA8xYk0rSwVNU83RnUJt6jIiNoENEoLbylQa9clHb58G8p+DUAej0APT5O/i49jRFYwyb008yIz6V7zYcIDuvkOZ1AhgZW5+h7cN1qYUqTINeua7c0/DTv2HNZAioBwNehahBdldVKbJyC5i78QAz4lPZlHYSbw83Bra2WvmdGtbUVn4Vo0GvXF/aOmvdnMNboMUgK/Crh9pdVaXZUqKVn5VbQOMQP0bGRjCsQzg1dUOUKkGDXlUNhfnW9oXLXgY3T+j7PMSMcYktDEsrO6+A7zceZMbaVNannsDL3Y24VnUZGRtBl0a1tJXvwjToVdVybDd8/yfYvQzCY611c+pE211Vpdt28BQz41OZvT6d02cLiAz2Y0Sn+gzrGE6wv/Pv8KXOV+agF5E44A3AHZhqjHn5gvNPAA8ABUAmcL8xZl/xuXuBvxVf+m9jzEeXey8NelUujIFNX1i7WuWegu6PQ6+nwLOa3ZVVupy8QuZtPsjMtams3XscT3fhxui63BYTTs8mwbp0sosoU9CLiDuwE+gHpAFrgZHGmKQS11wPrDHGZIvIQ0BvY8xwEakFJAAxgAHWAR2NMccv9X4a9KpcnTkKC5+FjTPArzZ0vBc6joYaYXZXZovkw6eZEb+f2evTOJGdT0iAN0Pbh3FrhzBdWM3JlTXouwL/MMbcVPz6GQBjzP9d4vr2wCRjTHcRGYkV+g8Wn5sMLDPGzLjU+2nQqwqxdyX8+hbs/BHEDVoMhNix0LCnyy2nUBq5BYUs3Z7BrHXpLNuRQUGRoWVodYZ1CGdIu1CCtGvH6Vwu6EuzgEYYsL/E6zSg82WuHwPMv8y9v2tKicg4YBxAREREKUpS6io17GF9Hd8Laz+wVsbcNgdCWlhz8NuOAO8Au6usNN4e7sS1qkdcq3oczcplzsYDfJ2YxgvfJ/HSvG30bh7CsA7h3BBVG28PXTPf2ZWmRX8bEGeMeaD49d1AZ2PM+ItcOwoYD1xnjMkVkacAH2PMv4vPPwfkGGNeu9T7aYteVYr8HNgyG+KnwMEN4BVghX3sWAhpbnd1ttlx6DSzE9P4Zn06GadzqVHNk5vb1mNYh3Da1dcncB1ZpXTdiEhf4C2skM8oPqZdN8qxGQPp6yD+fdg621pDJ7IXdBoLzQeAe9VcNbKgsIiVKUeYnZjOj1sPkVtQRKMQP4Z1CGdo+zBCA6veoLajK2vQe2ANxvYB0rEGY+80xmwtcU17YBZWyz+5xPFaWAOwHYoPJWINxh671Ptp0CvbnDkCiR9BwnQ4uR+qh1kDtx3vBf/adldnm1Nn85m36SCzE9OJ33sMEejWOIhhHcKJa1VXN0pxEOUxvXIAMBFreuU0Y8yLIvICkGCMmSMii4HWwMHiW1KNMYOL770f+Gvx8ReNMdMv914a9Mp2RYWwc4HVyt+91Hr4quUtViu/fmyVHLw9Z9/RM8xOTGf2+jT2H8vB18ud/q3qMaxjGF0ig3Bzq7qfjd30gSmlrtWRZFg7FTZ8bs3Hr9saYsdBq9tcZi/ba1FUZG2U8vW6NH7YfJCs3ALCAqtxa4cwbu0QTmSwn90lVjka9EqVVW4WbP7SauVnJIFPILQfBTH3u+wGKKWVk1fIwqRDfJ2YzsrkTIoMdIgIZFjHcAa1DtUVNSuJBr1S5cUY2PerNVtn+/dQVABN+lmzdZr0BbeqPRXx8KmzfLM+na/XpZGckYWXhxv9ouowrGMYvZqG6FO4FUiDXqmKcOogrPsQ1k2HrMMQ2AA6jYH2d4NvLburs5Uxhi3pp/g6MY3vNqRzPDufYH9vbmkXyq0dwokO1adwy5sGvVIVqSAPts+F+KmQ+it4+Fh9+LEPQGh7u6uzXV5BEct2ZPB1Yho/bc8gv9AQVa86wzqEMaRdGCEB+hRuedCgV6qyHNpiDd5u+gLysyG8kzVbp+Ut4KGBduxMHnM3HmB2Yhob007iJhDToBZ9o2vTN6oOjUL87S7RaWnQK1XZck5YC6mtnQpHU8A3GDrcYw3eBta3uzqHkHz4NHM3HmDxtgySDp4CoFGIH/2i6tA3ug4dImrirtM1S02DXim7FBXBnmXWbJ2dC6xjzfpb3TqRvavUpiiXk3Y8m5+2Z7Ao6TCrdx8lv9BQy8+L65vXpl90bXo2DcHPWx/MuhwNeqUcwYlUSJgGiR9D9lEIalK8oNpIqBZod3UO4/TZfJbvPMLibYf5aXsGJ3Py8XJ3o1uTIPpG1aFvVB3q1vCxu0yHo0GvlCPJPwtJ31rdOmlrwdMXWt9uTdGs29ru6hxKQWERa/ceZ/G2wyzedph9R7MBaB1Wwwr96NpE16uui62hQa+U4zqwwQr8zbOgIAfqd7YGb6MH6+DtBYwxpGRksWjbYRYnHWb9/hMYA6E1fOgbbbX0uzQKwsujanaHadAr5ehyjsP6zyDhA2vPW78Qa/C242gdvL2EzNO5LN2ewaJth1mRnMnZ/CL8vT24rlkIfaNrc33z2gT6etldZqXRoFfKWRQVwe6frDn5OxdYC6jp4O0Vnc0v5JeUI8VdPBlkns7F3U2IaVCTfsWt/YYuvv6OBr1Szuj4Puup25KDtzFjoN2dOnh7GUVFhk3pJ1mcZPXrbz90GoAmtf3pG1WHftF1aFc/0OWmbmrQK+XMCnJh67ew9v3zB287PQD12thdncPbfyz7t8HcNbuPUVBkCPb34oYW1kNaPZoGu8Sa+hr0SrmKiw7ePgDRQ3TwthRO5uTz885MFicdZumODE6fLcDbw41ujYPo2TSEXs2CaRzi75SzeDTolXI1OcetNfLXTtXB22uUX1jE2j3HWLTtMEu3Z7C3eOpm3eo+9GwaTM9mIXRvHESQv3P8AtWgV8pVnRu8XfuBPnlbRvuPZbMy5QgrkjP5JeUoJ3PyAWgVVp0eTULo1TSYjg1r4u3hmEtRa9ArVRWcSLX2u038GLKP6OBtGRQWGTann2RlcibLk4+QuO84BUUGH083OkcGWS3+piE0q+M43Twa9EpVJb8N3k6FtHjwqAZtbrcexNLB22uSlVvAmt1HWZFstfh3ZZ4BoHaANz2aBtOzaTDdmwRTO8C+pRk06JWqqg5ssB7C2vSVNXgbHmsttaCDt2Vy4EQOK5OPsDw5k19SjnA82+rmaVE3gF7NQujZNJhODWvh41l53Twa9EpVdRcO3voGQ7M4aNIHGvWu8jtilUVRkWHrgVOsSMlkxc4jJOw7Rn6hwcvDjc6RtejRxOrmaVE3ALcKnLuvQa+UshQVwe6lsP4T2PUTnD0J4gZhHaFxHyv4wzpW+b1vyyI7r4A1e46xYucRVqZksvNwFgDB/l70aBJMj6ZWi79O9fLt5tGgV0r9XmEBpK+DXUsgZYn1Zwz4BFqt/CZ9reCvHmpzoc7t0MmzJWbzHOFIVh4Azer407NpCD2aBtM5slaZH9rSoFdKXVn2Mau1n/ITpCyGrEPW8ZAoK/Cb9IGIbuCpa8Ffq6Iiw7ZDp1iZfIQVyUeI33uMvIIivNzd6NigJn2j6zCmR+Q1fW8NeqXU1TEGMpKswE9ZAqmroDDPmsHTsEdx8Pe1pnA6yPRCZ3Q2v5D4PcdYkZzJiuQj1K7uw8f3x17T99KgV0qVTd4Z2LvSCv2UxXBsl3W8RgQ0ucEK/che4FPD3jqdXF5B0TWvp3+5oHf+lXyUUhXPyw+a3WR9ARzfWxz6S6x1d9Z9COJurb3T5AZrYLdeO30y9ypV1KYp2qJXSpVNQZ71YFbKEmtg9+BG67hvMDS+3mrtN74B/GvbW6eLK3PXjYjEAW8A7sBUY8zLF5zvBUwE2gAjjDGzSpx7BRgIuAGLgMfMZd5Ug14pJ5eVAbuWWl08u36ylmMAaz/cJn2t1n79zuBRdXZ/qgxl6roREXfgbaAfkAasFZE5xpikEpelAvcBT11wbzegO9YvAICVwHXAsqv7EZRSTsO/NrQdbn0VFcGhTf8L/V/fgpX/BS9/q0+/SV+IvgX8guyu2qWVpo8+FkgxxuwGEJGZwBDgt6A3xuwtPld0wb0G8AG8AAE8gcNlrlop5Rzc3CC0nfXV6yk4ewr2LC+eu78YdsyD+X+GpjdZvxiaxenSDBWgNEEfBuwv8ToN6Fyab26MWSUiS4GDWEE/yRiz7aqrVEq5Bp/qEDXI+jIGDm+FTTNh05ew4wfrYa1Wt0LbkRDeSadulpMKnXUjIk2AKCC8+NAiEelpjFlxwXXjgHEAERERFVmSUspRiEDdVlD339DnH7BnGWz8AjbMgIRpUKsRtBlhtfRrNrS5WOdWmrk86UDJLWvCi4+VxlBgtTEmyxiTBcwHul54kTFmijEmxhgTExISUspvrZRyGe4eVn/9sPfh6WQY8g5UD4NlL8EbbWFaf1j3kbU2j7pqpQn6tUBTEYkUES9gBDCnlN8/FbhORDxExBNrIFa7bpRSl+YdAO3vgvu+h8c3ww3PwZlMmPtHeK0ZfDUadv4Ihfl2V+o0Sju9cgDW9El3YJox5kUReQFIMMbMEZFOwDdATeAscMgY07J4xs47QC+sgdkFxpgnLvdeOr1SKfU7xsCBRNg403pAK+eYtU9u69uh7Qio26bK9+frEghKKddRkAcpi6zQ37nAWoOndrQV+K1vr7KrbWrQK6VcU/Yx2PqNFfpp8YBYSyy3HWnN7PHys7nAyqNBr5RyfUd3waYvYOMMa6N0Tz9ry8S2w6FhT5ffTEWDXilVdRQVwf7VVuBv/RZyT1kzeNrcYbX0Q5rbXWGF0KBXSlVN+TmwY77VtZOyGEwhhLa3Ar/VMPALtrvCcqNBr5RSWRnWjJ2NM6z1d9w8oEk/axC3WZzT75ylQa+UUiUdTvrf0gunD1o7Z4W2szZGP/cVGOFUUzY16JVS6mKKCmHPz7BzobU5+sGNUJhrnfMLOT/4wzpAtZr21nsZusOUUkpdjJu7tSlK4xus1wV5kLHVCv20ddZ/d/6I9bwnUKsxhMcUB3+MtVaPE6y2qS16pZS6nLMn4cB6K/TTEyEtAbIOWefcPK0NVX4L/47WLwMbtlDUFr1SSl0rnxrWQ1iNeluvjYFTB4qDP8EK/w2fQ/yU/10f2uH88Ld5G0UNeqWUuhoiUCPM+ooebB0rKoTMHSXCfx2seN2azglQI8Lq4w/raP0CqNe2Up/a1aBXSqmycnOHOtHWV4e7rWN52dbgbvq6//0CSPrWOifu1vo8JcM/pEWFPb2rQa+UUhXByxcadLW+zsnKtFbhTCtu9Sd9B4kfWec8/aDZTXD79HIvRYNeKaUqi3+IFebNbrJeGwPHdv+v1V9B3Tka9EopZRcRCGpsfbW5o8LepvLnACmllKpUGvRKKeXiNOiVUsrFadArpZSL06BXSikXp0GvlFIuToNeKaVcnAa9Ukq5OIdbplhEMoF9ZfgWwcCRcirH2elncT79PM6nn8f/uMJn0cAYE3KxEw4X9GUlIgmXWpO5qtHP4nz6eZxPP4//cfXPQrtulFLKxWnQK6WUi3PFoJ9idwEORD+L8+nncT79PP7HpT8Ll+ujV0opdT5XbNErpZQqwWWCXkTiRGSHiKSIyAS767GTiNQXkaUikiQiW0XkMbtrspuIuIvIehH53u5a7CYigSIyS0S2i8g2Eel65btcl4j8qfjvyRYRmSEiPnbXVN5cIuhFxB14G+gPRAMjRSTa3qpsVQA8aYyJBroAj1TxzwPgMWCb3UU4iDeABcaYFkBbqvDnIiJhwB+BGGNMK8AdGGFvVeXPJYIeiAVSjDG7jTF5wExgiM012cYYc9AYk1j859NYf5HD7K3KPiISDgwEptpdi91EpAbQC/gAwBiTZ4w5YW9VtvMAqomIB+ALHLC5nnLnKkEfBuwv8TqNKhxsJYlIQ6A9sMbeSmw1EfgzUGR3IQ4gEsgEphd3ZU0VkYrZqNQJGGPSgdeAVOAgcNIYs9DeqsqfqwS9uggR8Qe+Bh43xpyyux47iMggIMMYs87uWhyEB9ABeNcY0x44A1TZMS0RqYn1r/9IIBTwE5FR9lZV/lwl6NOB+iVehxcfq7JExBMr5D8zxsy2ux4bdQcGi8herC69G0TkU3tLslUakGaMOfcvvFlYwV9V9QX2GGMyjTH5wGygm801lTtXCfq1QFMRiRQRL6zBlDk212QbERGsPthtxpjX7a7HTsaYZ4wx4caYhlj/X/xkjHG5FltpGWMOAftFpHnxoT5Ako0l2S0V6CIivsV/b/rggoPTHnYXUB6MMQUiMh74EWvUfJoxZqvNZdmpO3A3sFlENhQf+6sxZp6NNSnH8SjwWXGjaDcw2uZ6bGOMWSMis4BErNlq63HBp2T1yVillHJxrtJ1o5RS6hI06JVSysVp0CullIvToFdKKRenQa+UUi5Og14ppVycBr1SSrk4DXqllHJx/x/eOjxokMsI5gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NHuno8p75vKE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Encode the input sequence to get the feature vector\n",
        "encoder_model = Model(inputs=encoder_inputs,outputs=[encoder_outputs, state_h, state_c])\n",
        "\n",
        "# Decoder setup\n",
        "# Below tensors will hold the states of the previous time step\n",
        "decoder_state_input_h = Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = Input(shape=(latent_dim,))\n",
        "decoder_hidden_state_input = Input(shape=(max_text_len,latent_dim))\n",
        "\n",
        "# Get the embeddings of the decoder sequence\n",
        "dec_emb2= dec_emb_layer(decoder_inputs) \n",
        "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
        "decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=[decoder_state_input_h, decoder_state_input_c])\n",
        "\n",
        "#attention inference\n",
        "attn_out_inf, attn_states_inf = attn_layer([decoder_hidden_state_input, decoder_outputs2])\n",
        "decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_outputs2, attn_out_inf])\n",
        "\n",
        "# A dense softmax layer to generate prob dist. over the target vocabulary\n",
        "decoder_outputs2 = decoder_dense(decoder_inf_concat) \n",
        "\n",
        "# Final decoder model\n",
        "decoder_model = Model(\n",
        "    [decoder_inputs] + [decoder_hidden_state_input,decoder_state_input_h, decoder_state_input_c],\n",
        "    [decoder_outputs2] + [state_h2, state_c2])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F3M-PqdJ5vHC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def decode_sequence(input_seq):\n",
        "    # Encode the input as state vectors.\n",
        "    e_out, e_h, e_c = encoder_model.predict(input_seq)\n",
        "    \n",
        "    # Generate empty target sequence of length 1.\n",
        "    target_seq = np.zeros((1,1))\n",
        "    # e_h = e_h*0\n",
        "    # e_c = e_c*0\n",
        "\n",
        "    # Populate the first word of target sequence with the start word.\n",
        "    target_seq[0, 0] = target_word_index['sostok']\n",
        "\n",
        "    stop_condition = False\n",
        "    decoded_sentence = ''\n",
        "    while not stop_condition:\n",
        "      \n",
        "        output_tokens, h, c = decoder_model.predict([target_seq] + [e_out, e_h, e_c])\n",
        "\n",
        "        # Sample a token\n",
        "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
        "        sampled_token = reverse_target_word_index[sampled_token_index]\n",
        "        # print(sampled_token)\n",
        "        if(sampled_token!='eostok'):\n",
        "            decoded_sentence += ' '+sampled_token\n",
        "\n",
        "        # Exit condition: either hit max length or find stop word.\n",
        "        if (sampled_token == 'eostok'  or len(decoded_sentence.split()) >= (max_summary_len-1)):\n",
        "            stop_condition = True\n",
        "\n",
        "        # Update the target sequence (of length 1).\n",
        "        target_seq = np.zeros((1,1))\n",
        "        target_seq[0, 0] = sampled_token_index\n",
        "\n",
        "        # Update internal states\n",
        "        e_h, e_c = h, c\n",
        "\n",
        "    return decoded_sentence"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XnLCbbxE5vDp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def seq2summary(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "        if((i!=0 and i!=target_word_index['sostok']) and i!=target_word_index['eostok']):\n",
        "            newString=newString+reverse_target_word_index[i]+' '\n",
        "    return newString\n",
        "\n",
        "def seq2text(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "        if(i!=0):\n",
        "            newString=newString+reverse_source_word_index[i]+' '\n",
        "    return newString"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "neS0-iA_56NQ",
        "colab_type": "code",
        "outputId": "a288ca73-8393-4b77-bd83-5a80dde3f4eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 462
        }
      },
      "source": [
        "for i in range(0,5):\n",
        "    print(\"Tabular data:\",seq2text(x_tr[i]))\n",
        "    print(\"Original summary:\",seq2summary(y_tr[i]))\n",
        "    print(\"Predicted summary:\",decode_sequence(x_tr[i].reshape(1,max_text_len)))\n",
        "    print(\"\\n\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tabular data: temperature time 17 30 min 67 mean 70 max 77 windchill time 17 30 min 0 mean 0 max 0 windspeed time 17 30 min 11 mean 12 max 16 mode bucket 0 20 2 10 20 winddir time 17 30 mode ese gust time 17 30 min 0 mean 3 max 21 skycover time 17 30 mode bucket 0 100 4 25 50 skycover time 17 21 mode bucket 0 100 4 25 50 skycover time 17 26 mode bucket 0 100 4 25 50 skycover time 21 30 mode bucket 0 100 4 25 50 skycover time 26 30 mode bucket 0 100 4 25 50 precippotential time 17 30 min 2 mean 4 max 5 \n",
            "Original summary: partly cloudy with a low around 67 east wind between 11 and 13 mph \n",
            "Predicted summary:  partly cloudy with a low around 67 east wind between 11 and 14 mph\n",
            "\n",
            "\n",
            "Tabular data: max 9 mode bucket 0 20 2 0 10 winddir time 6 21 mode nw gust time 6 21 min 0 mean 0 max 0 skycover time 6 21 mode bucket 0 100 4 75 100 skycover time 6 9 mode bucket 0 100 4 75 100 skycover time 6 13 mode bucket 0 100 4 75 100 skycover time 9 21 mode bucket 0 100 4 75 100 skycover time 13 21 mode bucket 0 100 4 50 75 precippotential time 6 21 min 17 mean 36 max 60 snowchance time 6 21 mode chc snowchance time 6 9 mode lkly snowchance time 6 13 mode lkly snowchance time 9 21 mode chc snowchance time 13 21 mode schc \n",
            "Original summary: snow likely mainly before 11am mostly cloudy with a high near 37 east wind 6 to 9 mph becoming northwest chance of precipitation is 60 new snow accumulation of less than one inch possible \n",
            "Predicted summary:  a chance of rain or drizzle mainly before noon patchy fog before noon otherwise mostly cloudy with a high near 55 calm wind becoming southwest between 4 and 7 mph chance of precipitation is 40\n",
            "\n",
            "\n",
            "Tabular data: windchill time 17 30 min 35 mean 36 max 39 windspeed time 17 30 min 10 mean 14 max 16 mode bucket 0 20 2 10 20 winddir time 17 30 mode sse gust time 17 30 min 0 mean 0 max 0 skycover time 17 30 mode bucket 0 100 4 50 75 skycover time 17 21 mode bucket 0 100 4 50 75 skycover time 17 26 mode bucket 0 100 4 50 75 skycover time 21 30 mode bucket 0 100 4 50 75 skycover time 26 30 mode bucket 0 100 4 75 100 precippotential time 17 30 min 5 mean 27 max 77 rainchance time 21 30 mode schc rainchance time 26 30 mode def \n",
            "Original summary: occasional rain mainly after 3am low around 42 south wind between 11 and 16 mph chance of precipitation is 80 new rainfall amounts between a quarter and half of an inch possible \n",
            "Predicted summary:  occasional rain mainly after 3am low around 41 south wind between 11 and 14 mph chance of precipitation is 80 new rainfall amounts between a quarter and half of an inch possible\n",
            "\n",
            "\n",
            "Tabular data: min 43 mean 44 max 46 windchill time 17 30 min 36 mean 38 max 41 windspeed time 17 30 min 10 mean 10 max 13 mode bucket 0 20 2 10 20 winddir time 17 30 mode se gust time 17 30 min 0 mean 0 max 0 skycover time 17 30 mode bucket 0 100 4 50 75 skycover time 17 21 mode bucket 0 100 4 50 75 skycover time 17 26 mode bucket 0 100 4 50 75 skycover time 21 30 mode bucket 0 100 4 50 75 skycover time 26 30 mode bucket 0 100 4 75 100 precippotential time 17 30 min 5 mean 26 max 75 rainchance time 26 30 mode def \n",
            "Original summary: occasional rain after 3am low around 43 south wind between 10 and 13 mph chance of precipitation is 80 new rainfall amounts between a quarter and half of an inch possible \n",
            "Predicted summary:  occasional rain after 3am low around 41 south wind between 11 and 14 mph chance of precipitation is 80 new rainfall amounts between a quarter and half of an inch possible\n",
            "\n",
            "\n",
            "Tabular data: windchill time 6 21 min 0 mean 38 max 48 windspeed time 6 21 min 3 mean 5 max 7 mode bucket 0 20 2 0 10 winddir time 6 21 mode n gust time 6 21 min 0 mean 0 max 0 skycover time 6 21 mode bucket 0 100 4 25 50 skycover time 6 9 mode bucket 0 100 4 75 100 skycover time 6 13 mode bucket 0 100 4 75 100 skycover time 9 21 mode bucket 0 100 4 25 50 skycover time 13 21 mode bucket 0 100 4 25 50 precippotential time 6 21 min 2 mean 17 max 53 rainchance time 6 9 mode chc rainchance time 6 13 mode chc \n",
            "Original summary: scattered showers mainly before 10am cloudy then gradually becoming mostly sunny with a high near 52 north wind around 7 mph chance of precipitation is 50 new rainfall amounts of less than a tenth of an inch possible \n",
            "Predicted summary:  a chance of snow showers before noon mostly cloudy with a high near 52 calm wind becoming south between 4 and 7 mph chance of precipitation is 40 little or no snow accumulation expected\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UPVUyyRICJV8",
        "colab_type": "code",
        "outputId": "6e675d41-566c-44de-e8a1-c1bdc37c8e09",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 462
        }
      },
      "source": [
        "for i in range(0,5):\n",
        "    print(\"Tabular data:\",seq2text(x_val[i]))\n",
        "    print(\"Original summary:\",seq2summary(y_val[i]))\n",
        "    print(\"Predicted summary:\",decode_sequence(x_val[i].reshape(1,max_text_len)))\n",
        "    print(\"\\n\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tabular data: max 14 mode bucket 0 20 2 10 20 winddir time 17 30 mode s gust time 17 30 min 0 mean 1 max 21 skycover time 17 30 mode bucket 0 100 4 75 100 skycover time 17 21 mode bucket 0 100 4 50 75 skycover time 17 26 mode bucket 0 100 4 50 75 skycover time 21 30 mode bucket 0 100 4 75 100 skycover time 26 30 mode bucket 0 100 4 75 100 precippotential time 17 30 min 0 mean 37 max 50 rainchance time 17 30 mode chc rainchance time 17 21 mode chc rainchance time 17 26 mode chc rainchance time 21 30 mode chc rainchance time 26 30 mode chc \n",
            "Original summary: a 40 percent chance of showers mostly cloudy with a low around 50 south wind between 10 and 15 mph with gusts as high as 20 mph \n",
            "Predicted summary:  a chance of showers patchy fog otherwise cloudy with a low around 52 south wind between 11 and 14 mph chance of precipitation is 50 new rainfall amounts between a tenth and quarter of an inch possible\n",
            "\n",
            "\n",
            "Tabular data: max 13 mode bucket 0 20 2 0 10 winddir time 17 30 mode s gust time 17 30 min 0 mean 1 max 20 skycover time 17 30 mode bucket 0 100 4 75 100 skycover time 17 21 mode bucket 0 100 4 75 100 skycover time 17 26 mode bucket 0 100 4 75 100 skycover time 21 30 mode bucket 0 100 4 75 100 skycover time 26 30 mode bucket 0 100 4 75 100 precippotential time 17 30 min 31 mean 47 max 58 rainchance time 17 30 mode chc rainchance time 17 21 mode chc rainchance time 17 26 mode chc rainchance time 21 30 mode chc rainchance time 26 30 mode chc \n",
            "Original summary: showers likely mainly between 10pm and midnight cloudy with a low around 52 south southwest wind between 5 and 10 mph chance of precipitation is 60 \n",
            "Predicted summary:  a chance of showers patchy fog otherwise cloudy with a low around 52 south wind between 6 and 11 mph chance of precipitation is 50 new rainfall amounts between a tenth and quarter of an inch possible\n",
            "\n",
            "\n",
            "Tabular data: max 17 mode bucket 0 20 2 10 20 winddir time 17 30 mode sw gust time 17 30 min 0 mean 16 max 25 skycover time 17 30 mode bucket 0 100 4 50 75 skycover time 17 21 mode bucket 0 100 4 50 75 skycover time 17 26 mode bucket 0 100 4 50 75 skycover time 21 30 mode bucket 0 100 4 50 75 skycover time 26 30 mode bucket 0 100 4 50 75 precippotential time 17 30 min 8 mean 19 max 21 rainchance time 17 30 mode schc rainchance time 17 21 mode schc rainchance time 17 26 mode schc rainchance time 21 30 mode schc rainchance time 26 30 mode schc \n",
            "Original summary: mostly cloudy with a low around 36 southwest wind 10 to 16 mph becoming north northwest winds could gust as high as 21 mph \n",
            "Predicted summary:  a slight chance of showers mostly cloudy with a low around 36 west wind between 11 and 16 mph with gusts as high as 25 mph chance of precipitation is 20\n",
            "\n",
            "\n",
            "Tabular data: temperature time 17 30 min 42 mean 47 max 56 windchill time 17 30 min 0 mean 34 max 50 windspeed time 17 30 min 3 mean 5 max 9 mode bucket 0 20 2 0 10 winddir time 17 30 mode w gust time 17 30 min 0 mean 0 max 0 skycover time 17 30 mode bucket 0 100 4 0 25 skycover time 17 21 mode bucket 0 100 4 0 25 skycover time 17 26 mode bucket 0 100 4 0 25 skycover time 21 30 mode bucket 0 100 4 0 25 skycover time 26 30 mode bucket 0 100 4 25 50 precippotential time 17 30 min 9 mean 9 max 9 \n",
            "Original summary: mostly clear with a low around 42 west wind around 5 mph becoming east \n",
            "Predicted summary:  partly cloudy with a low around 41 west wind around 5 mph\n",
            "\n",
            "\n",
            "Tabular data: temperature time 17 30 min 44 mean 52 max 66 windchill time 17 30 min 0 mean 23 max 49 windspeed time 17 30 min 2 mean 3 max 8 mode bucket 0 20 2 0 10 winddir time 17 30 mode ene gust time 17 30 min 0 mean 0 max 0 skycover time 17 30 mode bucket 0 100 4 0 25 skycover time 17 21 mode bucket 0 100 4 0 25 skycover time 17 26 mode bucket 0 100 4 0 25 skycover time 21 30 mode bucket 0 100 4 0 25 skycover time 26 30 mode bucket 0 100 4 0 25 precippotential time 17 30 min 0 mean 0 max 0 \n",
            "Original summary: mostly clear with a low around 43 east northeast wind around 5 mph becoming calm \n",
            "Predicted summary:  mostly clear with a low around 46 east wind around 5 mph becoming calm\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1vjugfXRCP76",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_num = 200\n",
        "reference = []\n",
        "for i in range(test_num):\n",
        "  reference.append(seq2summary(y_val[i]))\n",
        "\n",
        "prediction = []\n",
        "for i in range(test_num):\n",
        "  prediction.append(decode_sequence(x_val[i].reshape(1,max_text_len)))\n",
        "\n",
        "\n",
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "\n",
        "score = 0.\n",
        "\n",
        "for i in range(len(reference)):\n",
        "    score += sentence_bleu([reference[i].strip().split()], prediction[i].strip().split())\n",
        "\n",
        "score /= len(reference)\n",
        "print(\"The bleu score is: \"+str(score))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-FYmmhThYIc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"The bleu score is: \"+str(score))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BBHf2X35h_90",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}