Present-5pmJuly2_Latest_WithScope.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing word counts and creating vocab based on word count threshold 2\n",
      "filtered words from 18418 to 9900\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding=utf-8\n",
    "\n",
    "__author__ = \"Xinpeng.Chen\"\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "\n",
    "import h5py\n",
    "import ipdb\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------------------------------------------\n",
    "# Initialization class\n",
    "#  1. Pooling the visual features into a single dense feature\n",
    "#  2. Then, build sentence LSTM, word LSTM\n",
    "# ------------------------------------------------------------------------------------------------------\n",
    "class RegionPooling_HierarchicalRNN():\n",
    "    def __init__(self, n_words,\n",
    "                       batch_size,\n",
    "                       num_boxes,\n",
    "                       feats_dim,\n",
    "                       project_dim,\n",
    "                       sentRNN_lstm_dim,\n",
    "                       sentRNN_FC_dim,\n",
    "                       wordRNN_lstm_dim,\n",
    "                       S_max,\n",
    "                       N_max,\n",
    "                       word_embed_dim,\n",
    "                       bias_init_vector=None):\n",
    "\n",
    "        self.n_words = n_words\n",
    "        self.batch_size = batch_size\n",
    "        self.num_boxes = num_boxes # 50\n",
    "        self.feats_dim = feats_dim # 4096\n",
    "        self.project_dim = project_dim # 1024\n",
    "        self.S_max = S_max # 6\n",
    "        self.N_max = N_max # 50\n",
    "        self.word_embed_dim = word_embed_dim # 1024\n",
    "\n",
    "        self.sentRNN_lstm_dim = sentRNN_lstm_dim # 512 hidden size\n",
    "        self.sentRNN_FC_dim = sentRNN_FC_dim # 1024 in fully connected layer\n",
    "        self.wordRNN_lstm_dim = wordRNN_lstm_dim # 512 hidden size\n",
    "\t\n",
    "\n",
    "\n",
    "        # word embedding, parameters of embedding\n",
    "        # embedding shape: n_words x wordRNN_lstm_dim\n",
    "        with tf.device('/cpu:0'):\n",
    "            self.Wemb = tf.Variable(tf.random_uniform([n_words, word_embed_dim], -0.1, 0.1), name='Wemb')\n",
    "        print(\"Wemb output>>>>>>>>>>>>\",self.Wemb,tf.shape(self.Wemb))\n",
    "        #self.bemb = tf.Variable(tf.zeros([word_embed_dim]), name='bemb')\n",
    "\n",
    "        # regionPooling_W shape: 4096 x 1024\n",
    "        # regionPooling_b shape: 1024\n",
    "        self.regionPooling_W = tf.Variable(tf.random_uniform([feats_dim, project_dim], -0.1, 0.1), name='regionPooling_W')\n",
    "        self.regionPooling_b = tf.Variable(tf.zeros([project_dim]), name='regionPooling_b')\n",
    "\n",
    "        # sentence LSTM\n",
    "        self.sent_LSTM = tf.nn.rnn_cell.BasicLSTMCell(sentRNN_lstm_dim, state_is_tuple=True)\n",
    "\n",
    "        # logistic classifier\n",
    "        self.logistic_Theta_W = tf.Variable(tf.random_uniform([sentRNN_lstm_dim, 2], -0.1, 0.1), name='logistic_Theta_W')\n",
    "        self.logistic_Theta_b = tf.Variable(tf.zeros(2), name='logistic_Theta_b')\n",
    "\n",
    "        # fc1_W: 512 x 1024, fc1_b: 1024\n",
    "        # fc2_W: 1024 x 1024, fc2_b: 1024\n",
    "        self.fc1_W = tf.Variable(tf.random_uniform([sentRNN_lstm_dim, sentRNN_FC_dim], -0.1, 0.1), name='fc1_W')\n",
    "        self.fc1_b = tf.Variable(tf.zeros(sentRNN_FC_dim), name='fc1_b')\n",
    "        self.fc2_W = tf.Variable(tf.random_uniform([sentRNN_FC_dim, 1024], -0.1, 0.1), name='fc2_W')\n",
    "        self.fc2_b = tf.Variable(tf.zeros(1024), name='fc2_b')\n",
    "        def get_a_cell(lstm_size):\n",
    "          lstm = tf.nn.rnn_cell.BasicLSTMCell(lstm_size)\n",
    "          return lstm\n",
    "        # word LSTM\n",
    "        self.word_LSTM = tf.nn.rnn_cell.BasicLSTMCell(wordRNN_lstm_dim, state_is_tuple=True)\n",
    "        #self.word_LSTM = tf.nn.rnn_cell.MultiRNNCell([self.word_LSTM] * 2, state_is_tuple=True)\n",
    "        self.word_LSTM = tf.nn.rnn_cell.MultiRNNCell([get_a_cell(wordRNN_lstm_dim) for i in range(2)], state_is_tuple=True)\n",
    "        #self.word_LSTM2 = tf.nn.rnn_cell.BasicLSTMCell(wordRNN_lstm_dim, state_is_tuple=True)\n",
    "\n",
    "\n",
    "        self.embed_word_W = tf.Variable(tf.random_uniform([wordRNN_lstm_dim, n_words], -0.1,0.1), name='embed_word_W')\n",
    "        if bias_init_vector is not None:\n",
    "            self.embed_word_b = tf.Variable(bias_init_vector.astype(np.float32), name='embed_word_b')\n",
    "        else:\n",
    "            self.embed_word_b = tf.Variable(tf.zeros([n_words]), name='embed_word_b')\n",
    "\n",
    "    def build_model(self):\n",
    "        # receive the feats in the current image\n",
    "        # it's shape is 10 x 50 x 4096\n",
    "        # tmp_feats: 500 x 4096\n",
    "        feats = tf.placeholder(tf.float32, [self.batch_size, self.num_boxes, self.feats_dim])\n",
    "        tmp_feats = tf.reshape(feats, [-1, self.feats_dim])\n",
    "\n",
    "        # project_vec_all: 500 x 4096 * 4096 x 1024 --> 500 x 1024\n",
    "        # project_vec: 10 x 1024\n",
    "        project_vec_all = tf.matmul(tmp_feats, self.regionPooling_W) + self.regionPooling_b\n",
    "        project_vec_all = tf.reshape(project_vec_all, [self.batch_size, 50, self.project_dim])\n",
    "        project_vec = tf.reduce_max(project_vec_all, reduction_indices=1)\n",
    "\n",
    "        # receive the [continue:0, stop:1] lists\n",
    "        # example: [0, 0, 0, 0, 1, 1], it means this paragraph has five sentences\n",
    "        num_distribution = tf.placeholder(tf.int32, [self.batch_size, self.S_max])\n",
    "\n",
    "        # receive the ground truth words, which has been changed to idx use word2idx function\n",
    "        captions = tf.placeholder(tf.int32, [self.batch_size, self.S_max, self.N_max+1])\n",
    "        #print(\"Captions:>>>>>>>>>>>>>>>>>>>>>>>\",captions)\n",
    "        captions_masks = tf.placeholder(tf.float32, [self.batch_size, self.S_max, self.N_max+1])\n",
    "\n",
    "        # ---------------------------------------------------------------------------------------------------------------------\n",
    "        # The method which initialize the state, is refered from below sites:\n",
    "        # 1. http://stackoverflow.com/questions/38241410/tensorflow-remember-lstm-state-for-next-batch-stateful-lstm/38417699\n",
    "        # 2. https://www.tensorflow.org/api_docs/python/rnn_cell/classes_storing_split_rnncell_state#LSTMStateTuple\n",
    "        # 3. https://medium.com/@erikhallstrm/using-the-tensorflow-lstm-api-3-7-5f2b97ca6b73#.u4w9z6h0h\n",
    "        # ---------------------------------------------------------------------------------------------------------------------\n",
    "        sent_state = self.sent_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state = self.word_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state1 = self.word_LSTM1.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state2 = self.word_LSTM2.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #sent_state = tf.zeros([self.batch_size, self.sent_LSTM1.state_size])\n",
    "        #word_state1 = tf.zeros([self.batch_size, self.word_LSTM1.state_size])\n",
    "        #word_state2 = tf.zeros([self.batch_size, self.word_LSTM2.state_size])\n",
    "\n",
    "        probs = []\n",
    "        loss = 0.0\n",
    "        loss_sent = 0.0\n",
    "        loss_word = 0.0\n",
    "        lambda_sent = 5.0\n",
    "        lambda_word = 1.0\n",
    "\n",
    "        print('Start build model:')\n",
    "        #----------------------------------------------------------------------------------------------\n",
    "        # Hierarchical RNN: sentence RNN and words RNN\n",
    "        # The word RNN has the max number, N_max = 50, the number in the papar is 50\n",
    "        #----------------------------------------------------------------------------------------------\n",
    "        for i in range(0, self.S_max):\n",
    "            if i > 0:\n",
    "                tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.variable_scope('sent_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                sent_output, sent_state = self.sent_LSTM(project_vec, sent_state)\n",
    "\n",
    "            with tf.name_scope('fc1'):\n",
    "                hidden1 = tf.nn.relu( tf.matmul(sent_output, self.fc1_W) + self.fc1_b )\n",
    "            with tf.name_scope('fc2'):\n",
    "                sent_topic_vec = tf.nn.relu( tf.matmul(hidden1, self.fc2_W) + self.fc2_b )\n",
    "\n",
    "            # sent_state is a tuple, sent_state = (c, h)\n",
    "            # 'c': shape=(1, 512) dtype=float32, 'h': shape=(1, 512) dtype=float32\n",
    "            # The loss here, I refer from the web which is very helpful for me:\n",
    "            # 1. http://stackoverflow.com/questions/34240703/difference-between-tensorflow-tf-nn-softmax-and-tf-nn-softmax-cross-entropy-with\n",
    "            # 2. http://stackoverflow.com/questions/35277898/tensorflow-for-binary-classification\n",
    "            # 3. http://stackoverflow.com/questions/35226198/is-this-one-hot-encoding-in-tensorflow-fast-or-flawed-for-any-reason\n",
    "            # 4. http://stackoverflow.com/questions/35198528/reshape-y-train-for-binary-text-classification-in-tensorflow\n",
    "            sentRNN_logistic_mu = tf.nn.xw_plus_b( sent_output, self.logistic_Theta_W, self.logistic_Theta_b )\n",
    "            sentRNN_label = tf.stack([ 1 - num_distribution[:, i], num_distribution[:, i] ])\n",
    "            sentRNN_label = tf.transpose(sentRNN_label)\n",
    "            sentRNN_loss = tf.nn.softmax_cross_entropy_with_logits(logits=sentRNN_logistic_mu, labels=sentRNN_label)\n",
    "            sentRNN_loss = tf.reduce_sum(sentRNN_loss)/self.batch_size\n",
    "            loss += sentRNN_loss * lambda_sent\n",
    "            loss_sent += sentRNN_loss\n",
    "\n",
    "            # the begining input of word_LSTM is topic vector, and DON'T compute the loss\n",
    "            # This is follow the paper: Show and Tell\n",
    "            #word_state = self.word_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "            #with tf.variable_scope('word_LSTM'):\n",
    "            #    word_output, word_state = self.word_LSTM(sent_topic_vec)\n",
    "            topic = tf.nn.rnn_cell.LSTMStateTuple(sent_topic_vec[:, 0:512], sent_topic_vec[:, 512:])\n",
    "            word_state = (topic, topic)\n",
    "            for j in range(0, self.N_max):\n",
    "                if j > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                with tf.device('/cpu:0'):\n",
    "                    current_embed = tf.nn.embedding_lookup(self.Wemb, captions[:, i, j])\n",
    "\n",
    "                with tf.variable_scope('word_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                    word_output, word_state = self.word_LSTM(current_embed, word_state)\n",
    "\n",
    "                # How to make one-hot encoder, I refer from this excellent web:\n",
    "                # http://stackoverflow.com/questions/33681517/tensorflow-one-hot-encoder\n",
    "                labels = tf.reshape(captions[:, i, j+1], [-1, 1])\n",
    "                #print(\"Labels and its shape +++++++++++++++\",labels,tf.shape(labels))\n",
    "                indices = tf.reshape(tf.range(0, self.batch_size, 1), [-1, 1])\n",
    "                #print(\"Indices and its shape +++++++++++++++\",indices,tf.shape(indices))\n",
    "                concated = tf.concat([indices, labels],1)\n",
    "                #print(\"Concated+++++++++++++++++++\",concated)\n",
    "                print(\"Success\")\n",
    "                onehot_labels = tf.sparse_to_dense(concated, tf.stack([self.batch_size, self.n_words]), 1.0, 0.0)\n",
    "\n",
    "                # At each timestep the hidden state of the last LSTM layer is used to predict a distribution\n",
    "                # over the words in the vocbulary\n",
    "                logit_words = tf.nn.xw_plus_b(word_output[:], self.embed_word_W, self.embed_word_b)\n",
    "                cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logit_words, labels=onehot_labels)\n",
    "                cross_entropy = cross_entropy * captions_masks[:, i, j]\n",
    "                loss_wordRNN = tf.reduce_sum(cross_entropy) / self.batch_size\n",
    "                loss += loss_wordRNN * lambda_word\n",
    "                loss_word += loss_wordRNN\n",
    "\n",
    "        return feats, num_distribution, captions, captions_masks, loss, loss_sent, loss_word\n",
    "\n",
    "    def generate_model(self):\n",
    "        # feats: 1 x 50 x 4096\n",
    "        feats = tf.placeholder(tf.float32, [1, self.num_boxes, self.feats_dim])\n",
    "        # tmp_feats: 50 x 4096\n",
    "        tmp_feats = tf.reshape(feats, [-1, self.feats_dim])\n",
    "\n",
    "        # project_vec_all: 50 x 4096 * 4096 x 1024 + 1024 --> 50 x 1024\n",
    "        project_vec_all = tf.matmul(tmp_feats, self.regionPooling_W) + self.regionPooling_b\n",
    "        project_vec_all = tf.reshape(project_vec_all, [1, 50, self.project_dim])\n",
    "        project_vec = tf.reduce_max(project_vec_all, reduction_indices=1)\n",
    "\n",
    "        # initialize the sent_LSTM state\n",
    "        sent_state = self.sent_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "\n",
    "        # save the generated paragraph to list, here I named generated_sents\n",
    "        generated_paragraph = []\n",
    "\n",
    "        # pred\n",
    "        pred_re = []\n",
    "\n",
    "        # T_stop: run the sentence RNN forward until the stopping probability p_i (STOP) exceeds a threshold T_stop\n",
    "        T_stop = tf.constant(0.5)\n",
    "\n",
    "        # Start build the generation model\n",
    "        print('Start build the generation model: ')\n",
    "\n",
    "        # sentence RNN\n",
    "        #word_state = self.word_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "        #with tf.variable_scope('word_LSTM'):\n",
    "        #    word_output, word_state = self.word_LSTM(sent_topic_vec, word_state)\n",
    "        for i in range(0, self.S_max):\n",
    "            if i > 0:\n",
    "                tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            # sent_state:\n",
    "            # LSTMStateTuple(c=<tf.Tensor 'sent_LSTM/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "            #                h=<tf.Tensor 'sent_LSTM/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>)\n",
    "            with tf.variable_scope('sent_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                sent_output, sent_state = self.sent_LSTM(project_vec, sent_state)\n",
    "\n",
    "            # self.fc1_W: 512 x 1024, self.fc1_b: 1024\n",
    "            # hidden1: 1 x 1024\n",
    "            # sent_topic_vec: 1 x 1024\n",
    "            with tf.name_scope('fc1'):\n",
    "                hidden1 = tf.nn.relu( tf.matmul(sent_output, self.fc1_W) + self.fc1_b )\n",
    "            with tf.name_scope('fc2'):\n",
    "                sent_topic_vec = tf.nn.relu( tf.matmul(hidden1, self.fc2_W) + self.fc2_b )\n",
    "\n",
    "            sentRNN_logistic_mu = tf.nn.xw_plus_b(sent_output, self.logistic_Theta_W, self.logistic_Theta_b)\n",
    "            pred = tf.nn.softmax(sentRNN_logistic_mu)\n",
    "            pred_re.append(pred)\n",
    "\n",
    "            # save the generated sentence to list, named generated_sent\n",
    "            generated_sent = []\n",
    "\n",
    "            # initialize the word LSTM state\n",
    "            #word_state = self.word_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "            #with tf.variable_scope('word_LSTM'):\n",
    "            #    word_output, word_state = self.word_LSTM(sent_topic_vec, word_state)\n",
    "            topic = tf.nn.rnn_cell.LSTMStateTuple(sent_topic_vec[:, 0:512], sent_topic_vec[:, 512:])\n",
    "            word_state = (topic, topic)\n",
    "            # word RNN, unrolled to N_max time steps\n",
    "            for j in range(0, self.N_max):\n",
    "                if j > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                if j == 0:\n",
    "                    with tf.device('/cpu:0'):\n",
    "                        # get word embedding of BOS (index = 0)\n",
    "                        current_embed = tf.nn.embedding_lookup(self.Wemb, tf.zeros([1], dtype=tf.int64))\n",
    "\n",
    "                with tf.variable_scope('word_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                    word_output, word_state = self.word_LSTM(current_embed, word_state)\n",
    "\n",
    "                # word_state:\n",
    "                # (\n",
    "                #     LSTMStateTuple(c=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell0/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "                #                    h=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell0/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>),\n",
    "                #     LSTMStateTuple(c=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell1/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "                #                    h=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell1/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>)\n",
    "                # )\n",
    "                logit_words = tf.nn.xw_plus_b(word_output, self.embed_word_W, self.embed_word_b)\n",
    "                max_prob_index = tf.argmax(logit_words, 1)[0]\n",
    "                generated_sent.append(max_prob_index)\n",
    "\n",
    "                with tf.device('/cpu:0'):\n",
    "                    current_embed = tf.nn.embedding_lookup(self.Wemb, max_prob_index)\n",
    "                    current_embed = tf.expand_dims(current_embed, 0)\n",
    "\n",
    "            generated_paragraph.append(generated_sent)\n",
    "\n",
    "        return feats, generated_paragraph, pred_re, sent_topic_vec\n",
    "\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "# Preparing Functions\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "def preProBuildWordVocab(sentence_iterator, word_count_threshold=5):\n",
    "    # borrowed this function from NeuralTalk\n",
    "    print('preprocessing word counts and creating vocab based on word count threshold %d' % (word_count_threshold, ))\n",
    "\n",
    "    word_counts = {}\n",
    "    nsents = 0\n",
    "\n",
    "    for sent in sentence_iterator:\n",
    "        nsents += 1\n",
    "        tmp_sent = sent.lower().split(' ')\n",
    "        if '' in tmp_sent:\n",
    "            tmp_sent.remove('')\n",
    "\n",
    "        for w in tmp_sent:\n",
    "           word_counts[w] = word_counts.get(w, 0) + 1\n",
    "\n",
    "    vocab = [w for w in word_counts if word_counts[w] >= word_count_threshold]\n",
    "    print('filtered words from %d to %d' % (len(word_counts), len(vocab)))\n",
    "\n",
    "    ixtoword = {}\n",
    "    ixtoword[0] = '<bos>'\n",
    "    ixtoword[1] = '<eos>'\n",
    "    ixtoword[2] = '<pad>'\n",
    "    ixtoword[3] = '<unk>'\n",
    "\n",
    "    wordtoix = {}\n",
    "    wordtoix['<bos>'] = 0\n",
    "    wordtoix['<eos>'] = 1\n",
    "    wordtoix['<pad>'] = 2\n",
    "    wordtoix['<unk>'] = 3\n",
    "\n",
    "    for idx, w in enumerate(vocab):\n",
    "        wordtoix[w] = idx + 4\n",
    "        ixtoword[idx+4] = w\n",
    "\n",
    "    word_counts['<eos>'] = nsents\n",
    "    word_counts['<bos>'] = nsents\n",
    "    word_counts['<pad>'] = nsents\n",
    "    word_counts['<unk>'] = nsents\n",
    "\n",
    "    bias_init_vector = np.array([1.0 * word_counts[ ixtoword[i] ] for i in ixtoword])\n",
    "    bias_init_vector /= np.sum(bias_init_vector) # normalize to frequencies\n",
    "    bias_init_vector = np.log(bias_init_vector)\n",
    "    bias_init_vector -= np.max(bias_init_vector) # shift to nice numeric range\n",
    "\n",
    "    return wordtoix, ixtoword, bias_init_vector\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Parameters Setting\n",
    "#######################################################################################################\n",
    "batch_size = 5 # Being support batch_size\n",
    "num_boxes = 50 # number of Detected regions in each image\n",
    "feats_dim = 4096 # feature dimensions of each regions\n",
    "project_dim = 1024 # project the features to one vector, which is 1024 dimensions\n",
    "\n",
    "sentRNN_lstm_dim = 512 # the sentence LSTM hidden units\n",
    "sentRNN_FC_dim = 1024 # the fully connected units\n",
    "wordRNN_lstm_dim = 512 # the word LSTM hidden units\n",
    "word_embed_dim = 1024 # the learned embedding vectors for the words\n",
    "\n",
    "S_max = 6\n",
    "N_max = 50\n",
    "T_stop = 0.5\n",
    "\n",
    "n_epochs = 10\n",
    "learning_rate = 0.0001\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Word vocubulary and captions preprocessing stage\n",
    "#######################################################################################################\n",
    "img2paragraph = pickle.load(open('./img2paragraph', 'rb'))\n",
    "all_sentences = []\n",
    "for key, paragraph in img2paragraph.items():\n",
    "    for each_sent in paragraph[1]:\n",
    "        each_sent.replace(',', ' ,')\n",
    "        all_sentences.append(each_sent)\n",
    "word2idx, idx2word, bias_init_vector = preProBuildWordVocab(all_sentences, word_count_threshold=2)\n",
    "np.save('./idx2word_batch', idx2word)\n",
    "\n",
    "img2paragraph_modify = {}\n",
    "for img_name, img_paragraph in img2paragraph.items():\n",
    "    img_paragraph_1 = img_paragraph[1]\n",
    "\n",
    "    # img_paragraph_1 is a list\n",
    "    # it may contain the element: '' or ' ', like this:\n",
    "    # [[\"a man is walking\"], [\"the dog is running\"], [\"\"], [\" \"]]\n",
    "    # so, we should remove them ' ' and '' element\n",
    "    if '' in img_paragraph_1:\n",
    "        img_paragraph_1.remove('')\n",
    "    if ' ' in paragraph[1]:\n",
    "        img_paragraph_1.remove(' ')\n",
    "\n",
    "    # the number sents in each paragraph\n",
    "    # if the sents is bigger than S_max,\n",
    "    # we force the number of sents to be S_max\n",
    "    img_num_sents = len(img_paragraph_1)\n",
    "    if img_num_sents > S_max:\n",
    "        img_num_sents = S_max\n",
    "\n",
    "    # if a paragraph has 4 sentences\n",
    "    # then the img_num_distribution will be like this:\n",
    "    # [0, 0, 0, 1, 1, 1]\n",
    "    img_num_distribution = np.zeros([S_max], dtype=np.int32)\n",
    "    img_num_distribution[img_num_sents-1:] = 1\n",
    "\n",
    "    # we multiply the number 2, because the <pad> is encoded into 2\n",
    "    img_captions_matrix = np.ones([S_max, N_max+1], dtype=np.int32) * 2 # zeros([6, 50])\n",
    "    for idx, img_sent in enumerate(img_paragraph_1):\n",
    "        # the number of sentences is img_num_sents\n",
    "        if idx == img_num_sents:\n",
    "            break\n",
    "\n",
    "        # because we treat the ',' as a word\n",
    "        img_sent = img_sent.replace(',', ' ,')\n",
    "\n",
    "        # Because I have preprocess the paragraph_v1.json file in VScode before,\n",
    "        # and I delete all the 2, 3, 4...bankspaces\n",
    "        # so, actually, the 'elif' code will never run\n",
    "        if img_sent[0] == ' ' and img_sent[1] != ' ':\n",
    "            img_sent = img_sent[1:]\n",
    "        elif img_sent[0] == ' ' and img_sent[1] == ' ' and img_sent[2] != ' ':\n",
    "            img_sent = img_sent[2:]\n",
    "\n",
    "        # Be careful the last part in a sentence, like this:\n",
    "        # '...world.'\n",
    "        # '...world. '\n",
    "        if img_sent[-1] == '.':\n",
    "            img_sent = img_sent[0:-1]\n",
    "        elif img_sent[-1] == ' ' and img_sent[-2] == '.':\n",
    "            img_sent = img_sent[0:-2]\n",
    "\n",
    "        # Last, we add the <bos> and the <eos> in each sentences\n",
    "        img_sent = '<bos> ' + img_sent + ' <eos>'\n",
    "\n",
    "        # translate each word in a sentence into the unique number in word2idx dict\n",
    "        # when we meet the word which is not in the word2idx dict, we use the mark: <unk>\n",
    "        for idy, word in enumerate(img_sent.lower().split(' ')):\n",
    "            # because the biggest number of words in a sentence is N_max, here is 50\n",
    "            if idy == N_max:\n",
    "                break\n",
    "\n",
    "            if word in word2idx:\n",
    "                img_captions_matrix[idx, idy] = word2idx[word]\n",
    "            else:\n",
    "                img_captions_matrix[idx, idy] = word2idx['<unk>']\n",
    "\n",
    "    # Pay attention, the value type 'img_name' here is NUMBER, I change it to STRING type\n",
    "    img2paragraph_modify[str(img_name)] = [img_num_distribution, img_captions_matrix]\n",
    "\n",
    "with open('./img2paragraph_modify_batch', 'wb') as f:\n",
    "    pickle.dump(img2paragraph_modify, f)\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Train, validation and testing stage\n",
    "#######################################################################################################\n",
    "def train():\n",
    "    ##############################################################################\n",
    "    # some preparing work\n",
    "    ##############################################################################\n",
    "    model_path = './models_batch/'\n",
    "    train_feats_path = './im2p_train_output.h5'\n",
    "    train_output_file = h5py.File(train_feats_path, 'r')\n",
    "    train_feats = train_output_file.get('feats')\n",
    "    train_imgs_full_path_lists = open('./imgs_train_path.txt').read().splitlines()\n",
    "    train_imgs_names = map(lambda x: os.path.basename(x).split('.')[0], train_imgs_full_path_lists)\n",
    "\n",
    "\n",
    "    # Model Initialization:\n",
    "    # n_words, batch_size, num_boxes, feats_dim, project_dim, sentRNN_lstm_dim, sentRNN_FC_dim, wordRNN_lstm_dim, S_max, N_max\n",
    "    model = RegionPooling_HierarchicalRNN(n_words = len(word2idx),\n",
    "                                          batch_size = batch_size,\n",
    "                                          num_boxes = num_boxes,\n",
    "                                          feats_dim = feats_dim,\n",
    "                                          project_dim = project_dim,\n",
    "                                          sentRNN_lstm_dim = sentRNN_lstm_dim,\n",
    "                                          sentRNN_FC_dim = sentRNN_FC_dim,\n",
    "                                          wordRNN_lstm_dim = wordRNN_lstm_dim,\n",
    "                                          S_max = S_max,\n",
    "                                          N_max = N_max,\n",
    "                                          word_embed_dim = word_embed_dim,\n",
    "                                          bias_init_vector = bias_init_vector)\n",
    "\n",
    "    tf_feats, tf_num_distribution, tf_captions_matrix, tf_captions_masks, tf_loss, tf_loss_sent, tf_loss_word = model.build_model()\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    saver = tf.train.Saver(max_to_keep=500, write_version=1)\n",
    "    train_op = tf.train.AdamOptimizer(learning_rate).minimize(tf_loss)\n",
    "    tf.global_variables_initializer().run()\n",
    "\n",
    "    # when you want to train the model from the front model\n",
    "    #new_saver = tf.train.Saver(max_to_keep=500)\n",
    "    #new_saver = tf.train.import_meta_graph('./models_batch/model-92.meta')\n",
    "    #new_saver.restore(sess, tf.train.latest_checkpoint('./models_batch/'))\n",
    "\n",
    "    all_vars = tf.trainable_variables()\n",
    "\n",
    "    # open a loss file to record the loss value\n",
    "    loss_fd = open('loss_batch.txt', 'w')\n",
    "    img2idx = {}\n",
    "    for idx, img in enumerate(train_imgs_names):\n",
    "        img2idx[img] = idx\n",
    "\n",
    "    # plt draw the loss curve\n",
    "    # refer from: http://stackoverflow.com/questions/11874767/real-time-plotting-in-while-loop-with-matplotlib\n",
    "    loss_to_draw = []\n",
    "\n",
    "    for epoch in range(0, n_epochs):\n",
    "        loss_to_draw_epoch = []\n",
    "        # disorganize the order\n",
    "        random.shuffle(train_imgs_names)\n",
    "\n",
    "        for start, end in zip(range(0, len(train_imgs_names), batch_size),\n",
    "                              range(batch_size, len(train_imgs_names), batch_size)):\n",
    "\n",
    "            start_time = time.time()\n",
    "\n",
    "            img_name = train_imgs_names[start:end]\n",
    "            current_feats_index = map(lambda x: img2idx[x], img_name)\n",
    "            current_feats = np.asarray( map(lambda x: train_feats[x], current_feats_index) )\n",
    "\n",
    "            current_num_distribution = np.asarray( map(lambda x: img2paragraph_modify[x][0], img_name) )\n",
    "            current_captions_matrix = np.asarray( map(lambda x: img2paragraph_modify[x][1], img_name) )\n",
    "\n",
    "            current_captions_masks = np.zeros( (current_captions_matrix.shape[0], current_captions_matrix.shape[1], current_captions_matrix.shape[2]) )/idx2word_batch\n",
    "            # find the non-zero element\n",
    "            nonzeros = np.array( map(lambda each_matrix: np.array( map(lambda x: (x != 2).sum() + 1, each_matrix ) ), current_captions_matrix ) )\n",
    "            for i in range(batch_size):\n",
    "                for ind, row in enumerate(current_captions_masks[i]):\n",
    "                    row[:(nonzeros[i, ind]-1)] = 1\n",
    "\n",
    "            # shape of current_feats: batch_size x 50 x 4096\n",
    "            # shape of current_num_distribution: batch_size x 6\n",
    "            # shape of current_captions_matrix: batch_size x 6 x 50\n",
    "            _, loss_val, loss_sent, loss_word= sess.run(\n",
    "                                [train_op, tf_loss, tf_loss_sent, tf_loss_word],\n",
    "                                feed_dict={\n",
    "                                           tf_feats: current_feats,\n",
    "                                           tf_num_distribution: current_num_distribution,\n",
    "                                           tf_captions_matrix: current_captions_matrix,\n",
    "                                           tf_captions_masks: current_captions_masks\n",
    "                                })\n",
    "\n",
    "            # append loss to list in a epoch\n",
    "            loss_to_draw_epoch.append(loss_val)\n",
    "\n",
    "            # running information\n",
    "            print('idx: ', start, ' Epoch: ', epoch, ' loss: ', loss_val, ' loss_sent: ', loss_sent, ' loss_word: ', loss_word, ' Time cost: ', str((time.time() - start_time)))\n",
    "            loss_fd.write('epoch ' + str(epoch) + ' loss ' + str(loss_val))\n",
    "\n",
    "        # draw loss curve every epoch\n",
    "        loss_to_draw.append(np.mean(loss_to_draw_epoch))\n",
    "        plt_save_dir = './loss_imgs'\n",
    "        plt_save_img_name = str(epoch) + '.png'\n",
    "        plt.plot(range(len(loss_to_draw)), loss_to_draw, color='g')\n",
    "        plt.grid(True)\n",
    "        plt.savefig(os.path.join(plt_save_dir, plt_save_img_name))\n",
    "\n",
    "        if np.mod(epoch, 10) == 0:\n",
    "            print(\"Epoch \", epoch, \" is done. Saving the model ...\")\n",
    "            saver.save(sess, os.path.join(model_path, 'model'), global_step=epoch)\n",
    "    loss_fd.close()\n",
    "\n",
    "\n",
    "def test():\n",
    "    \n",
    "    start_time = time.time()\n",
    "    # change the model path according to your environment\n",
    "    model_path = './model-250'\n",
    "\n",
    "    # It's very important to use Pandas to Series this idx2word dict\n",
    "    # After this operation, we can use list to extract the word at the same time\n",
    "    idx2word = pd.Series(np.load('./data/idx2word_batch.npy').tolist())\n",
    "\n",
    "    test_feats_path = './data/im2p_test_output.h5'\n",
    "    test_output_file = h5py.File(test_feats_path, 'r')\n",
    "    test_feats = test_output_file.get('feats')\n",
    "\n",
    "    test_imgs_full_path_lists = open('./densecap/imgs_test_path.txt').read().splitlines()\n",
    "    test_imgs_names = map(lambda x: os.path.basename(x).split('.')[0], test_imgs_full_path_lists)\n",
    "    \n",
    "    # n_words, batch_size, num_boxes, feats_dim, project_dim, sentRNN_lstm_dim, sentRNN_FC_dim, wordRNN_lstm_dim, S_max, N_max\n",
    "    test_model = RegionPooling_HierarchicalRNN(n_words = len(word2idx),\n",
    "                                               batch_size = batch_size,\n",
    "                                               num_boxes = num_boxes,\n",
    "                                               feats_dim = feats_dim,\n",
    "                                               project_dim = project_dim,\n",
    "                                               sentRNN_lstm_dim = sentRNN_lstm_dim,\n",
    "                                               sentRNN_FC_dim = sentRNN_FC_dim,\n",
    "                                               wordRNN_lstm_dim = wordRNN_lstm_dim,\n",
    "                                               S_max = S_max,\n",
    "                                               N_max = N_max,\n",
    "                                               word_embed_dim = word_embed_dim,\n",
    "                                               bias_init_vector = bias_init_vector)\n",
    "    \n",
    "\n",
    "    tf_feats, tf_generated_paragraph, tf_pred_re, tf_sent_topic_vectors = test_model.generate_model()\n",
    "    sess = tf.InteractiveSession()\n",
    "    #print(\"Before there >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\\n\\n\\n\")\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, model_path)\n",
    "\n",
    "    img2idx = {}\n",
    "    for idx, img in enumerate(test_imgs_names):\n",
    "        img2idx[img] = idx\n",
    "    print(\"IM2idx:::::::::::\",img2idx)\n",
    "\n",
    "    test_fd = open('HRNN_results.txt', 'w')\n",
    "    for idx, img_name in enumerate(test_imgs_names):\n",
    "        print(idx, img_name)\n",
    "        test_fd.write(img_name + '\\n')\n",
    "\n",
    "        each_paragraph = []\n",
    "        current_paragraph = \"\"\n",
    "\n",
    "        current_feats_index = img2idx[img_name]\n",
    "        current_feats = test_feats[current_feats_index]\n",
    "        current_feats = np.reshape(current_feats, [1, 50, 4096])\n",
    "\n",
    "        generated_paragraph_indexes, pred, sent_topic_vectors = sess.run(\n",
    "                                                                         [tf_generated_paragraph, tf_pred_re, tf_sent_topic_vectors],\n",
    "                                                                         feed_dict={\n",
    "                                                                             tf_feats: current_feats\n",
    "                                                                         })\n",
    "\n",
    "        #generated_paragraph = idx2word[generated_paragraph_indexes]\n",
    "        for sent_index in generated_paragraph_indexes:\n",
    "            each_sent = []\n",
    "            for word_index in sent_index:\n",
    "                each_sent.append(idx2word[word_index])\n",
    "            each_paragraph.append(each_sent)\n",
    "\n",
    "        for idx, each_sent in enumerate(each_paragraph):\n",
    "            # if the current sentence is the end sentence of the paragraph\n",
    "            # According to the probability distribution:\n",
    "            # CONTINUE: [1, 0]\n",
    "            # STOP    : [0, 1]\n",
    "            # So, if the first item of pred is less than the T_stop\n",
    "            # the generation process is break\n",
    "            if pred[idx][0][0] <= T_stop:\n",
    "                break\n",
    "            current_sent = ''\n",
    "            for each_word in each_sent:\n",
    "                current_sent += each_word + ' '\n",
    "            current_sent = current_sent.replace('<eos> ', '')\n",
    "            current_sent = current_sent.replace('<pad> ', '')\n",
    "            current_sent = current_sent + '.'\n",
    "            current_sent = current_sent.replace(' .', '.')\n",
    "            current_sent = current_sent.replace(' ,', ',')\n",
    "            current_paragraph +=current_sent\n",
    "            if idx != len(each_paragraph) - 1:\n",
    "                current_paragraph += ' '\n",
    "\n",
    "        test_fd.write(current_paragraph + '\\n')\n",
    "    test_fd.close()\n",
    "    print(\"Time cost: \" + str(time.time()-start_time))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val images names ['2385757', '2389264', '2407325', '2337141', '2345813', '2386694', '2368581', '2397809', '185', '2387911', '2343682', '2409165', '2357129', '498337', '2326621', '2320205', '2412782', '2352466', '2414097', '2381526', '2390981', '2375715', '2405096', '2402848', '2346948', '2387882', '2352243', '2337581', '2330919', '2380444', '2377611', '2354637', '2401996', '2391375', '2415510', '2400079', '2366081', '2360527', '2365490', '2336229', '2367475', '2373691', '2376356', '2368952', '2369653', '2355012', '2400892', '2369971', '2414408', '2394007', '2415978', '2405444', '2401822', '2387651', '2350622', '2318603', '2383962', '2377677', '2380510', '2392746', '2412229', '2392926', '2362383', '2398991', '2367805', '2360264', '2400991', '2354133', '2374619', '2413514', '2387844', '2349764', '2403572', '2372008', '2388111', '2366871', '2385053', '2400907', '2379868', '2389754', '2389615', '2365263', '2414350', '2409171', '4025', '2352062', '2378864', '2367278', '2319025', '2335091', '2344088', '2351081', '2408962', '2319561', '2400672', '2390977', '2390288', '2415083', '2396514', '2411955', '2346205', '2395037', '2346498', '2383586', '2346342', '2367250', '2348629', '2413978', '2355118', '2390609', '724', '2347956', '2389121', '2381396', '2394635', '2374001', '2319411', '2356016', '2390189', '2339582', '2362277', '2325410', '2354590', '2367096', '2317592', '2384171', '2397387', '2376251', '2352643', '2354147', '2410352', '2332365', '2323588', '2504', '2346067', '2373700', '2395463', '2388627', '2344272', '2365691', '2364664', '2410717', '2371950', '2402941', '2391252', '2342646', '2352971', '2370165', '2395630', '2317591', '2378059', '2355709', '2336832', '2391722', '2330764', '2346660', '2409862', '2373155', '2393166', '2355642', '2346977', '2377832', '2343586', '2388431', '2381416', '2408088', '2343537', '2390500', '2319274', '2358132', '2336159', '2386365', '2395072', '2405965', '2390610', '2392765', '2363501', '2322189', '2400936', '2354380', '2385725', '2414772', '2387049', '2339305', '2355163', '2411856', '2325076', '2391875', '2369075', '2404587', '2371926', '2395640', '2329779', '2402384', '2348939', '2361855', '2395696', '2391663', '2370517', '2412037', '2347202', '2353100', '2372426', '2405703', '2370496', '2375612', '2338355', '2342324', '2400462', '2396112', '2400862', '2379832', '2333467', '2359089', '2398544', '2328805', '2345621', '2397165', '2391502', '2403224', '2356812', '2389832', '2399569', '2414157', '2392065', '2353938', '2374300', '2411459', '2390394', '2354220', '2380380', '2363108', '2342406', '2334854', '2410297', '2335993', '2411121', '2366378', '2401969', '2387542', '2380071', '2393134', '2315498', '2367637', '2370780', '2317055', '2383295', '2400542', '2402050', '2390890', '2386260', '2377837', '2318159', '2327364', '2409644', '2396452', '2341431', '2381222', '2389464', '2406927', '2371744', '2392787', '2376996', '2340394', '2333581', '2398578', '2348334', '2408546', '2354873', '2384670', '2408339', '2316661', '2318214', '2369502', '2344294', '2316382', '2389172', '2412334', '2382722', '2373092', '2400913', '2364094', '2415823', '2407666', '2408407', '2353425', '2385066', '2350985', '2396301', '2355676', '2318134', '2368058', '2359892', '2327449', '2342078', '2365122', '2416996', '2365457', '2376076', '2397271', '2317228', '2333227', '2343001', '2335613', '2336076', '2363469', '2387136', '2387602', '538', '2381047', '2341390', '2382974', '2393524', '2389704', '2352288', '2366841', '2366424', '2414990', '2382610', '2359501', '2341126', '2406861', '2381613', '2381116', '2362015', '2364508', '2361773', '2386125', '2323005', '2373886', '2391950', '2323158', '2372145', '2392202', '2362921', '150344', '2415140', '2375237', '2397509', '2405568', '2410491', '2365504', '2372907', '2359383', '2384101', '2330124', '2341434', '2403182', '2393422', '2350678', '2403683', '2334135', '2379332', '2382744', '2400289', '2351156', '2354589', '2366570', '2393002', '2375133', '2372957', '2346843', '2376314', '2357994', '2389532', '2355449', '2387258', '2386636', '2381310', '2331045', '2362731', '2353668', '2390084', '2347783', '2383040', '2341770', '2367808', '2397944', '2389426', '2391869', '2351511', '2406056', '2347102', '2414155', '2396971', '2404188', '2352834', '2397549', '2335504', '2390711', '2408710', '2318479', '2345601', '2323690', '2393174', '3665', '2343876', '2327045', '2414418', '2350160', '2360570', '2416836', '2410413', '2359369', '2407102', '4329', '2317914', '2375681', '2385986', '2357723', '2390201', '2376235', '2366660', '2346881', '2345363', '2358150', '853', '1067', '2391849', '2347742', '2374809', '2363649', '2317707', '2394449', '2407504', '2366958', '815', '2381442', '2386687', '2343486', '2403767', '2342183', '2342649', '2360855', '2384747', '2369741', '2338878', '2350786', '2377187', '2391182', '2345464', '2382463', '2373922', '2364569', '2321507', '2391257', '2390199', '2376974', '2411637', '2414334', '2382975', '2374130', '2407054', '2376001', '2405490', '2391435', '2360573', '2356093', '2317319', '2376855', '2372392', '2399545', '2407631', '2414694', '2387731', '2380250', '2399949', '2381902', '2380617', '2387786', '2343356', '2316729', '2335572', '2342386', '2341113', '2407180', '2372676', '2408452', '2351275', '2327618', '2391545', '2406319', '2369158', '2408036', '2409641', '2318461', '2376879', '2382464', '2358660', '2397224', '2360963', '2356329', '2397683', '2398695', '2318316', '2369188', '2383665', '2351900', '2355768', '2341725', '2335274', '2410913', '2404585', '2383900', '2370017', '2406329', '2375118', '2389880', '2333038', '2345388', '2404202', '2345290', '2412080', '2336489', '2379551', '2352259', '2398246', '2322086', '2412178', '2389036', '2411172', '2377272', '2369601', '2399464', '2351596', '2403297', '2353451', '2394454', '2401819', '2398902', '2394152', '2385541', '2369013', '2385974', '2344284', '2365226', '2322420', '2358081', '1160101', '2316641', '2371907', '2384500', '2344095', '2409873', '2342700', '2368261', '2349084', '2373021', '2339331', '2401288', '2334516', '2344881', '2364452', '2349469', '2374709', '2406451', '2391194', '2394298', '2406376', '2371567', '2335457', '2407029', '2372220', '2357948', '2384021', '2388187', '2331430', '2367019', '2392600', '2370343', '2417500', '2346802', '2564', '2360148', '2407311', '2350000', '2374917', '2354814', '2352832', '2383787', '2387276', '2413861', '2336315', '2357005', '2356221', '2400369', '2349353', '2366432', '2405436', '2407604', '2318841', '2394677', '2384439', '2348689', '2413793', '2384089', '2373382', '2402051', '2330715', '2403512', '2379975', '2411960', '2363971', '2353696', '2384283', '2395949', '2352715', '2333164', '2364799', '2399390', '2341305', '2410276', '2319462', '2333402', '2405554', '2344657', '2328347', '2415027', '2356488', '2367352', '2400108', '2380279', '2396396', '2339028', '2363834', '2366302', '2361964', '2343596', '2382173', '2371493', '2362648', '2381158', '2392123', '2371441', '2393319', '2353865', '2412966', '2402825', '2356671', '2385174', '2375529', '2354544', '2323848', '2382977', '2323209', '2387933', '2341028', '2411576', '2356811', '2387207', '2338341', '2381847', '2396662', '2367762', '2391902', '2350959', '2393496', '2366615', '2344117', '2333815', '2352342', '2387761', '2346469', '2382632', '2337106', '2357507', '2368903', '2400371', '2377573', '2399802', '2374100', '2387464', '2410958', '2316254', '2355876', '2319466', '2387553', '2387137', '2401818', '2413411', '2408481', '2320306', '2343554', '2383435', '2341747', '2411945', '2395545', '2347126', '2383269', '2341187', '2395566', '2361388', '2326048', '2326483', '2342090', '2330104', '2388858', '2414131', '2328024', '2405963', '2321198', '2342281', '2408454', '2407660', '2350676', '2324748', '2364648', '2399525', '2367980', '2369455', '2388006', '2406791', '2338235', '2399119', '2367983', '2398222', '2376293', '2379307', '2318313', '2395464', '2355425', '2344467', '2397941', '2412770', '2378990', '2391984', '2327406', '2376475', '2382690', '2414448', '2322107', '2414173', '2377368', '2398978', '2334930', '2354564', '2394297', '2355839', '2355172', '2388186', '2334825', '2372450', '2377971', '2377053', '2393315', '2370179', '2349895', '2335023', '2366720', '2369272', '2344127', '2393434', '2339219', '2348398', '2413035', '2330639', '2316575', '2343155', '2353872', '2414882', '2373063', '2333717', '2350248', '2383119', '2404930', '2335333', '2380117', '2400689', '2377131', '2353667', '2406595', '2343214', '2410358', '2330044', '2377161', '2347777', '2396968', '2343948', '2408400', '2342794', '2388737', '2401886', '2341672', '2385815', '2407806', '2317356', '2347763', '2341717', '2357770', '2378698', '2392376', '2373225', '1979', '2387187', '2341245', '2399182', '2410305', '2350362', '2354852', '2357069', '2391111', '2396639', '2343276', '2385145', '2319237', '2398879', '2378517', '2320078', '2357038', '2378426', '2409995', '2396407', '2344540', '2403306', '2412032', '2364541', '2379607', '2363110', '2385571', '2367180', '2317743', '2345625', '2324168', '2382524', '2389666', '2342125', '2326672', '2358834', '2416678', '2412679', '2373510', '2409161', '2330294', '2385025', '2382543', '2407293', '2405007', '2319434', '2409065', '2374686', '2415570', '2370645', '2343220', '2385158', '2366216', '2411891', '2396279', '2378386', '2406971', '2364907', '2383068', '2346025', '2401071', '2394536', '2346609', '2366521', '2336144', '2369882', '2362607', '2352327', '2343623', '2348208', '2357580', '2393529', '2347586', '2410663', '2338501', '2414226', '2317379', '2384614', '2368599', '2339675', '2392812', '2404620', '2362539', '2329100', '2343589', '2385552', '2412438', '2415173', '2376132', '2392014', '2346010', '285802', '2366148', '2416104', '2369615', '2367875', '2398860', '2318174', '2372154', '2327125', '2343802', '2364976', '2391397', '2412087', '2382659', '2325536', '2343595', '2397030', '2390728', '2409300', '2375570', '2363267', '2379412', '2318100', '2381952', '2410088', '2390131', '2350025', '2357873', '2332037', '2407133', '2387000', '2412304', '2334742', '2360125', '2351557', '2322620', '2364598', '2409948', '2407622', '2409730', '2395914', '2344000', '2367747', '2355445', '2336558', '2326536', '2336255', '2341700', '2387514', '2341671', '2398110', '2399652', '1981', '2323460', '2390548', '2378480', '2359190', '2388960', '2374610', '2390517', '2374041', '2416167', '2359860', '2345744', '2390516', '2350734', '2320101', '2362840', '2355186', '2374316', '2394482', '2345532', '2403827', '2350800', '2384225', '2408041', '2372039', '2362947', '2396741', '2328602', '2403866', '2408144', '2339689', '2373049', '2396987', '2399359', '2356053', '2371761', '2374295', '2400094', '2368870', '358', '2316710', '2348163', '2413458', '2315631', '2365595', '2409738', '2388139', '2367957', '2335492', '2350434', '2337232', '2317038', '2399813', '2364830', '2358273', '2317457', '2343118', '2325817', '2316887', '285905', '2357254', '1592902', '2387245', '2374212', '2393912', '2412430', '2396351', '2364742', '2326373', '2354526', '2410548', '2374003', '2339208', '2412323', '2347068', '2410762', '2354059', '2357064', '2394457', '2364582', '2388352', '2412817', '2345982', '2331553', '2344607', '2401917', '2335275', '2401129', '2410083', '2411500', '2398309', '2385216', '2381550', '2341363', '2415473', '2402262', '2391262', '2368478', '2344295', '2340547', '2326050', '2339004', '2346626', '2316675', '2372940', '2403736', '2361538', '2335133', '2372110', '2355888', '2408983', '2318897', '2349203', '2402756', '2385127', '1947', '2414540', '2360127', '2403553', '2361204', '2382861', '2340485', '2318856', '2349149', '2357096', '2378937', '2330143', '2390945', '2347477', '2367727', '2382133', '2365803', '2385464', '2352560', '2389937', '2335666', '2359041', '2371361', '2402586', '2400198', '2367081', '2330999', '2407240', '2373013', '2333510', '2341668', '2398193', '2384410', '2394250', '2401035', '2382813', '2369897', '2413842', '2335959', '2381711', '2394115', '2394823', '2342230', '2395930', '2375161', '2371998', '2357321', '2381052', '2386678', '2359589', '2408377', '2319233', '2377371', '2356978', '2322528', '2342127', '2351858', '2394808', '2372836', '2350140', '2330793', '2365464', '2324955', '59', '2401816', '2363606', '2411900', '2387088', '2400434', '2408594', '2379456', '2366127', '2322573', '2365692', '2354909', '2381999', '2400554', '2377456', '2338012', '2400676', '2409993', '2409525', '2329113', '2353014', '2375625', '2413783', '2405811', '2366795', '2374646', '2388879', '2317337', '2327958', '2378809', '2372018', '2397405', '2396019', '2358287', '2393934', '2411311', '2341541', '2326852', '2372176', '2371733', '2352132', '2365664', '2316733', '2365437', '1592372', '2358414', '2409530', '2410977', '2330167', '2370025', '2340772', '2358472', '2341383', '2415196', '2367052', '2349022', '2380268', '2394057', '2331963', '2348762', '2393268', '2382522', '2354169', '2346439', '2360521', '2413923', '2322133', '2318491', '2349530', '2388772', '2362719', '2406548', '2360201', '2351834', '2341514', '2400161', '2394967', '2316555', '2386847', '2412609', '2411640', '2376384', '2349745', '2358575', '2362360', '2319185', '2318300', '2319338', '2341703', '2348223', '2411941', '2404878', '2388166', '2348265', '2393093', '2386134', '2317860', '2358790', '2392630', '2381546', '2410233', '2396340', '2413134', '2384897', '2372279', '2412901', '2392163', '2399340', '2362560', '2415464', '2355350', '2385072', '2387946', '2414959', '2335223', '2396029', '2409922', '2369585', '2340615', '2342329', '2357014', '2359654', '2394493', '2339690', '2367978', '2385255', '2409589', '2404541', '2411332', '2395373', '2340926', '2379898', '2366907', '2392500', '2381777', '2371763', '2323566', '2362679', '2390775', '2326661', '2400330', '2354989', '2359414', '2351277', '2345482', '2347852', '2361424', '2363496', '2374097', '2392960', '2363592', '2338512', '2350783', '2390282', '2361585', '2346556', '2393655', '2381013', '2380114', '2413540', '2410181', '2366603', '2405292', '2352992', '2343612', '2395465', '2395628', '2386850', '2380640', '2368472', '2320897', '2404090', '2344453', '2320614', '2375431', '2354513', '2352072', '2368501', '2382276', '2327911', '2397716', '2354631', '2413575', '2327559', '2389618', '2370697', '2375622', '2403565', '2360115', '2399996', '2402523', '3668', '2342671', '2393354', '2358421', '2403502', '2358670', '2389588', '2401096', '2356526', '2380095', '2346818', '2371255', '2391668', '2385831', '2394596', '2342960', '2374375', '1160254', '2375771', '2382933', '2392920', '2387989', '2324588', '2356074', '2317625', '2344661', '2326911', '2404259', '2361721', '2393279', '2386495', '2400046', '2365043', '2344637', '2357554', '2390759', '2318426', '2386258', '2382545', '2336325', '2392061', '2405822', '2401033', '2353349', '2412624', '2340180', '2397756', '2402816', '2367884', '2323076', '2328094', '2402273', '2369292', '2414256', '2361129', '2379258', '2342678', '2383042', '2375286', '543', '2369707', '2415363', '2351567', '2374164', '2318354', '2337138', '2416386', '2344622', '2408617', '2402485', '2390061', '2403911', '2385461', '2368073', '2325644', '2413346', '2333242', '2409746', '2318784', '2356563', '2350520', '2379974', '2393488', '2401112', '2340853', '2355566', '2338820', '2388446', '2408244', '2403749', '2339735', '2345624', '2381108', '2396179', '2382666', '2377454', '2376391', '2347274', '2384475', '2317896', '2398386', '2343705', '2366203', '2336091', '2318766', '2362130', '2364869', '2388733', '2364463', '2320207', '2399556', '2318740', '2319024', '2320357', '2349108', '2324753', '2357052', '2405788', '2367095', '2355641', '2361523', '2348566', '2369937', '2364778', '2410070', '2349065', '2364313', '2385956', '2330315', '2339046', '2347982', '2389147', '2411813', '2366650', '2316025', '2340085', '2349725', '2402374', '2389782', '2318529', '2353047', '2355690', '2407520', '2340975', '2387531', '2355380', '2361208', '2405966', '2350585', '2334735', '2415776', '2391209', '2364493', '2404485', '2336765', '2390589', '2407074', '2355399', '2396682', '2359934', '2390911', '2352126', '2367144', '2393988', '2319437', '2384790', '2332609', '2363103', '2363314', '2371312', '2402053', '2415913', '2387492', '2355933', '2408558', '2368969', '2341785', '2391385', '2366303', '2338004', '2331437', '2405390', '2318074', '2382213', '2352218', '2356795', '2366679', '2373340', '2410425', '2347072', '2315529', '2414805', '2334296', '2400705', '2348349', '2371338', '2355887', '2336426', '2409509', '2400488', '2347040', '2373504', '2400709', '2388269', '2374858', '2357848', '2351724', '2366552', '2373343', '2366798', '2370562', '2336279', '2390109', '2376359', '2403968', '2377951', '2383724', '2327224', '2345537', '2320748', '2352059', '2356407', '2379040', '2404395', '2335228', '2377815', '2346974', '2398486', '2385876', '2368358', '2328180', '2333993', '2412028', '2320127', '2361275', '2325956', '2344077', '2373936', '2400518', '2410757', '2388557', '2374409', '2346288', '2327758', '2360829', '2367594', '2351200', '2401206', '2347475', '2361164', '2347157', '2322190', '2387508', '2328285', '2332131', '2399529', '2406648', '2414784', '2390009', '2385308', '2411532', '2320131', '2366219', '2385693', '2367729', '2383497', '2399749', '2387687', '2318508', '2366431', '2346215', '2332290', '2396197', '2343376', '2355111', '2315963', '2360432', '2329225', '2414189', '2407349', '2354529', '2378366', '2360200', '2386619', '2358729', '2396069', '2414523', '2402102', '2316860', '2359321', '2390811', '2403746', '2392451', '2370342', '1512', '2351235', '2356083', '2352140', '2397229', '2341890', '2377512', '2411735', '2384658', '1159516', '2360315', '2332658', '2413457', '2415457', '2349652', '2359079', '2324375', '2397605', '2403889', '2389796', '2406696', '2334724', '2407120', '2336668', '2366034', '2384206', '2400680', '2332379', '2392818', '2406733', '2387944', '2402124', '2334363', '2331362', '2388172', '2382849', '2388891', '713323', '2407039', '2386334', '2394687', '2408468', '2350752', '2384387', '2412787', '2409238', '2401910', '2346934', '2342976', '2362198', '2319960', '2384406', '2360405', '2399333', '2323665', '1159963', '2379625', '2405682', '2373934', '2337314', '2346066', '2396448', '2317900', '2407413', '2328394', '2374389', '2382714', '2339245', '2316574', '2363770', '2390523', '2415871', '2390619', '2353681', '2375141', '2365887', '2392468', '2376016', '2339907', '2386673', '2339891', '2414494', '2356926', '2351882', '2355581', '2380553', '2405893', '2375873', '2379970', '2389093', '2349339', '2351669', '2376371', '2382695', '2342062', '2343739', '2403096', '2367587', '2393255', '2363034', '2411245', '2356290', '2333032', '2377054', '2381840', '2383123', '2417815', '2342463', '2346654', '2401911', '2330589', '2359948', '2348627', '2395330', '2325024', '2389270', '2366510', '2373821', '2392869', '2381761', '2415192', '2324220', '2359272', '2386989', '2372029', '2317409', '2382557', '2315640', '2366640', '2401940', '2332111', '2371625', '2398866', '2399393', '2408374', '2372030', '2322341', '2374141', '1592381', '2346658', '2391256', '2397008', '2405504', '2378739', '2384906', '2408673', '2396317', '2388519', '2365979', '2414365', '2388526', '2337365', '2348280', '2360988', '2363427', '2368396', '2405392', '2396760', '2398451', '2400579', '2357859', '2348172', '2388316', '2357856', '2323413', '2400412', '2389597', '2364522', '2413323', '2336219', '2367716', '2362819', '2396757', '2380218', '2358578', '2390032', '2409345', '2410852', '2384053', '2378686', '2400643', '2380143', '2389002', '2320951', '2354093', '2362892', '2408480', '2361074', '2368771', '2409608', '2371432', '2344878', '2318431', '2403123', '2410146', '2342919', '2391895', '2413037', '2415154', '2352917', '2336606', '2409059', '2351967', '2344341', '2402305', '2399436', '2399831', '2344880', '2366243', '2394911', '2340505', '2360192', '2355583', '2412691', '2317478', '2377654', '2393764', '2369389', '2380908', '2359734', '2395479', '2410117', '2345963', '2348675', '2402610', '2362917', '2381263', '2411367', '2350934', '2345484', '2375389', '2338460', '2392194', '2345283', '2357009', '2366433', '2400181', '2338625', '2385349', '2409624', '2325906', '2410111', '2356573', '2408231', '2374654', '2401986', '2406374', '2408091', '2388101', '2366266', '2358240', '2387174', '2364629', '2410500', '2396864', '2349287', '2341175', '2413048', '2352077', '2357255', '2364516', '2393711', '2332173', '2370652', '2379368', '2373368', '2391968', '2363840', '2356915', '2402864', '2347167', '2361726', '2363057', '195', '2389587', '2333962', '2339034', '2358787', '2413030', '2382967', '2356452', '2355462', '2398408', '2350770', '2361025', '2402846', '2341989', '2356957', '2371802', '2340693', '2365687', '2360971', '2383081', '2397831', '2382252', '2345262', '2412694', '2350816', '2356774', '2379797', '2355306', '2348088', '2392089', '2354342', '2414493', '2317122', '2411125', '2348278', '2380488', '2409861', '2414069', '2361015', '1186', '2348951', '2408872', '2358566', '2402174', '2376953', '2416072', '2399932', '2330371', '4526', '2368076', '2334644', '813', '2415207', '2406271', '262', '2400363', '2405126', '2392176', '2373460', '2413099', '2352124', '2317117', '2364952', '2345382', '2387583', '2386515', '2411830', '2344135', '2368524', '2354957', '2395289', '2345826', '2399843', '2360483', '2392613', '2346922', '2340767', '2395698', '2363261', '2341640', '2377290', '2379419', '2317513', '2405522', '2365947', '2371836', '2374929', '2348401', '2343057', '2339483', '2369764', '2393778', '2364711', '2331127', '1143', '2371418', '2403431', '2414878', '2384911', '2346679', '2333255', '2393843', '2322397', '2412964', '2341844', '2400168', '2371705', '2390876', '2336110', '587', '2364458', '2400258', '2387213', '2412725', '2408943', '2413997', '2369025', '3524', '2345255', '2387936', '2348847', '2361143', '2356669', '2351483', '2399453', '2392336', '2399058', '2387509', '2346365', '2387430', '2323101', '2393641', '2403291', '2352426', '2346311', '2334281', '2365395', '2367554', '2379698', '2356542', '2374256', '2378992', '2379893', '2395881', '1159529', '2347316', '2398565', '61515', '2382188', '2371228', '2374191', '2405763', '2373065', '2368332', '2366857', '3974', '2403142', '2403730', '2348008', '2364666', '2383971', '2323317', '2348859', '2351300', '2393837', '2360110', '2366855', '2410116', '2399588', '2404970', '2411064', '2372245', '2396669', '2387797', '2347378', '2330444', '2397123', '2367427', '2374351', '2360943', '2331431', '2338001', '2346419', '2317987', '2399954', '2415472', '2349266', '2384407', '2339402', '2395507', '2410775', '2318528', '2408638', '2358223', '2390577', '2360637', '2403309', '2389127', '2400665', '2345254', '2357568', '2368103', '2376750', '2362276', '2373822', '2318753', '2381335', '2396390', '2407563', '677', '2383588', '2364405', '2363823', '2332047', '2369980', '2411794', '2375098', '2349488', '2412833', '2389931', '2411635', '2321324', '2351789', '2330962', '2325271', '2393810', '2386491', '2351574', '2386840', '2342128', '2360241', '2389642', '2381424', '2378294', '2376247', '2387201', '2320667', '2376728', '2398703', '2369310', '2367175', '2324573', '2344722', '2341732', '2336105', '2388874', '2341737', '2322795', '2378412', '2387370', '2369286', '2318253', '2343618', '2354019', '2402340', '2392557', '2397147', '2353466', '2402479', '2393995', '2329101', '2404918', '2347211', '2345294', '2385897', '2319741', '2414099', '2403177', '2319593', '2402961', '2361554', '2356491', '2392581', '2316889', '2360287', '2371216', '2407026', '2367533', '2378060', '2389282', '2390572', '2397120', '2400392', '2316318', '2375898', '2362311', '2401191', '2405981', '2409153', '2377072', '2355213', '2392583', '2382441', '2317053', '2346718', '2350383', '2406500', '2362770', '2410681', '2374807', '2346501', '2357249', '2359027', '2396374', '2374760', '2399256', '2336180', '2355922', '2356858', '2340777', '2381824', '2377986', '2386257', '2397845', '2389726', '2411629', '2393060', '2345476', '2374490', '2393513', '2409142', '2396465', '2356592', '2398489', '2417769', '2380388', '2386965', '2388564', '2360225', '2377334', '2396670', '2354660', '2357502', '2359450', '2378490', '2362795', '2363104', '2335727', '2390392', '2343913', '2373951', '2377309', '2392460', '2354423', '2359238', '2379480', '2374611', '2384854', '2369580', '2326856', '2332039', '2411249', '2406478', '2328010', '2369213', '2387943', '2385460', '2330218', '2360556', '2341674', '2342464', '2352565', '2341047', '2401918', '2372733', '2371551', '2316697', '2376379', '2368666', '2402546', '1592156', '2388838', '2397307', '2348182', '2413071', '2404732', '2356427', '2407637', '2390263', '2405776', '2353337', '2316602', '2353270', '2408708', '2371198', '2354266', '2369855', '2380368', '2382039', '2358567', '2319011', '2370341', '2370551', '2362808', '2382330', '2408653', '2374473', '2362280', '2398037', '2375753', '2371017', '2370877', '2408209', '2333007', '2405738', '2391153', '2372453', '2358526', '2379303', '2389636', '2342857', '2415640', '2386870', '2363443', '2368634', '2413904', '2383358', '2380024', '2416559', '2336529', '2366500', '2351655', '2345044', '2373588', '2390099', '2373407', '2350763', '2402155', '2363548', '2322813', '2343805', '2391461', '2416656', '2402786', '2394339', '2414255', '2338443', '2382381', '2374927', '2362555', '2402202', '2382046', '2412163', '2354362', '2415082', '2408489', '2406617', '2319554', '2327849', '2378204', '2391047', '2341611', '2342864', '2386983', '2347702', '2322974', '2325414', '2400737', '2404002', '2392261', '2356269', '2358999', '2399577', '2370776', '2316535', '2395996', '2336048', '2376467', '2362203', '2415820', '2383182', '2350730', '2345893', '2375076', '2417944', '2337033', '2359636', '2408359', '2416307', '2398017', '2383168', '2330468', '2405056', '2321253', '2341034', '2317165', '2401897', '2398212', '2387721', '2344595', '2379759', '2393164', '2410345', '2363437', '2386462', '2388504', '2401336', '2321387', '2317132', '2396478', '2416725', '2372569', '2359537', '2322095', '2352174', '2343766', '2380329', '2361489', '2413702', '2321676', '2404393', '2356073', '2376970', '2358293', '2342802', '2402837', '2363661', '2403903', '2319419', '2344370', '2389353', '2397657', '2371546', '2347170', '2404791', '2384368', '2417585', '2392507', '2349036', '2329559', '879', '2414977', '2407808', '2378572', '2415884', '2414803', '2402120', '2320356', '2365699', '2354877', '2389638', '2334130', '2367765', '2379154', '2399862', '2404467', '2353974', '2352853', '2339578', '2406885', '2382775', '2354318', '2353172', '2412056', '2396290', '2336760', '2378276', '2379292', '2406213', '2416937', '2395164', '2356436', '2355527', '2386443', '2397502', '2410270', '2397071', '2415676', '2371756', '2394933', '2377521', '2320584', '2394242', '2385172', '2414806', '2380874', '2371037', '2317584', '2390476', '2356914', '2361991', '2350826', '2406588', '2358311', '2409964', '2403418', '2370296', '2411605', '2323547', '2316204', '2375241', '2349448', '2354115', '2404178', '2372285', '2401764', '2377897', '2328135', '2382879', '2360242', '2350037', '2361415', '2367100', '2411866', '2378223', '2346003', '2359944', '2386442', '2327810', '2345716', '2398747', '2327729', '2379103', '2330650', '2391165', '2403131']\n",
      "Wemb output>>>>>>>>>>>> <tf.Variable 'Wemb_1:0' shape=(9904, 1024) dtype=float32_ref> Tensor(\"Shape_1:0\", shape=(2,), dtype=int32)\n",
      "Start build model:\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n"
     ]
    }
   ],
   "source": [
    "    #start by parts\n",
    "    model_path = './models_batch/'\n",
    "    train_feats_path = './im2p_val_output.h5'\n",
    "    \n",
    "    train_output_file = h5py.File(train_feats_path, 'r')\n",
    "    #print(\"Train feats path--------------------------------------------->>>>>>>>\",train_output_file)\n",
    "    train_feats = train_output_file.get('feats')\n",
    "    #print(\"Train feats->>>>>>>>>>>>>>>>\",train_feats)\n",
    "    train_imgs_full_path_lists = open('./imgs_val_path.txt').read().splitlines()\n",
    "    #print(\"paths>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\",train_imgs_full_path_lists)\n",
    "    train_imgs_names = map(lambda x: os.path.basename(x).split('.')[0], train_imgs_full_path_lists)\n",
    "    print(\"Val images names\",list(train_imgs_names))\n",
    "\n",
    "\n",
    "    # Model Initialization:\n",
    "    # n_words, batch_size, num_boxes, feats_dim, project_dim, sentRNN_lstm_dim, sentRNN_FC_dim, wordRNN_lstm_dim, S_max, N_max\n",
    "    model = RegionPooling_HierarchicalRNN(n_words = len(word2idx),\n",
    "                                          batch_size = batch_size,\n",
    "                                          num_boxes = num_boxes,\n",
    "                                          feats_dim = feats_dim,\n",
    "                                          project_dim = project_dim,\n",
    "                                          sentRNN_lstm_dim = sentRNN_lstm_dim,\n",
    "                                          sentRNN_FC_dim = sentRNN_FC_dim,\n",
    "                                          wordRNN_lstm_dim = wordRNN_lstm_dim,\n",
    "                                          S_max = S_max,\n",
    "                                          N_max = N_max,\n",
    "                                          word_embed_dim = word_embed_dim,\n",
    "                                          bias_init_vector = bias_init_vector)\n",
    "\n",
    "    tf_feats, tf_num_distribution, tf_captions_matrix, tf_captions_masks, tf_loss, tf_loss_sent, tf_loss_word = model.build_model()\n",
    "    sess = tf.InteractiveSession()\n",
    "    \n",
    "    \n",
    "    model_path = './models_batch/'\n",
    "    train_feats_path = './im2p_val_output.h5'\n",
    "    #print(\"THE SESSION\",)\n",
    "    \n",
    "    train_output_file = h5py.File(train_feats_path, 'r')\n",
    "    #print(\"Train feats path--------------------------------------------->>>>>>>>\",train_output_file)\n",
    "    train_feats = train_output_file.get('feats')\n",
    "    #print(\"Train feats->>>>>>>>>>>>>>>>\",train_feats)\n",
    "    train_imgs_full_path_lists = open('./imgs_val_path.txt').read().splitlines()\n",
    "    #print(\"paths>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\",train_imgs_full_path_lists)\n",
    "    train_imgs_names = map(lambda x: os.path.basename(x).split('.')[0], train_imgs_full_path_lists)\n",
    "    #print(\"Train images names\",list(train_imgs_names))\n",
    "    \n",
    "    #print(\"Train images names\",list(train_imgs_names))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning Rate and Loss++++++++++++ (0.0001, <tf.Tensor 'add_1224:0' shape=() dtype=float32>)\n",
      "Adam Done\n",
      "After run\n"
     ]
    }
   ],
   "source": [
    "    #saver = tf.train.Saver(max_to_keep=500, write_version=1)\n",
    "    print(\"Learning Rate and Loss++++++++++++\",(learning_rate,tf_loss))\n",
    "    with tf.variable_scope('optimizer',reuse= tf.AUTO_REUSE):\n",
    "        train_op = tf.train.AdamOptimizer(learning_rate).minimize(tf_loss)\n",
    "    print(\"Adam Done\")\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(\"After run\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done [<tf.Variable 'Wemb:0' shape=(9904, 1024) dtype=float32_ref>, <tf.Variable 'regionPooling_W:0' shape=(4096, 1024) dtype=float32_ref>, <tf.Variable 'regionPooling_b:0' shape=(1024,) dtype=float32_ref>, <tf.Variable 'logistic_Theta_W:0' shape=(512, 2) dtype=float32_ref>, <tf.Variable 'logistic_Theta_b:0' shape=(2,) dtype=float32_ref>, <tf.Variable 'fc1_W:0' shape=(512, 1024) dtype=float32_ref>, <tf.Variable 'fc1_b:0' shape=(1024,) dtype=float32_ref>, <tf.Variable 'fc2_W:0' shape=(1024, 1024) dtype=float32_ref>, <tf.Variable 'fc2_b:0' shape=(1024,) dtype=float32_ref>, <tf.Variable 'embed_word_W:0' shape=(512, 9904) dtype=float32_ref>, <tf.Variable 'embed_word_b:0' shape=(9904,) dtype=float32_ref>, <tf.Variable 'sent_LSTM/basic_lstm_cell/kernel:0' shape=(1536, 2048) dtype=float32_ref>, <tf.Variable 'sent_LSTM/basic_lstm_cell/bias:0' shape=(2048,) dtype=float32_ref>, <tf.Variable 'word_LSTM/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0' shape=(1536, 2048) dtype=float32_ref>, <tf.Variable 'word_LSTM/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0' shape=(2048,) dtype=float32_ref>, <tf.Variable 'word_LSTM/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0' shape=(1024, 2048) dtype=float32_ref>, <tf.Variable 'word_LSTM/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0' shape=(2048,) dtype=float32_ref>, <tf.Variable 'Wemb_1:0' shape=(9904, 1024) dtype=float32_ref>, <tf.Variable 'regionPooling_W_1:0' shape=(4096, 1024) dtype=float32_ref>, <tf.Variable 'regionPooling_b_1:0' shape=(1024,) dtype=float32_ref>, <tf.Variable 'logistic_Theta_W_1:0' shape=(512, 2) dtype=float32_ref>, <tf.Variable 'logistic_Theta_b_1:0' shape=(2,) dtype=float32_ref>, <tf.Variable 'fc1_W_1:0' shape=(512, 1024) dtype=float32_ref>, <tf.Variable 'fc1_b_1:0' shape=(1024,) dtype=float32_ref>, <tf.Variable 'fc2_W_1:0' shape=(1024, 1024) dtype=float32_ref>, <tf.Variable 'fc2_b_1:0' shape=(1024,) dtype=float32_ref>, <tf.Variable 'embed_word_W_1:0' shape=(512, 9904) dtype=float32_ref>, <tf.Variable 'embed_word_b_1:0' shape=(9904,) dtype=float32_ref>]\n"
     ]
    }
   ],
   "source": [
    "all_vars = tf.trainable_variables()\n",
    "print(\"Done\",list(all_vars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train images names ['2385757', '2389264', '2407325', '2337141', '2345813', '2386694', '2368581', '2397809', '185', '2387911', '2343682', '2409165', '2357129', '498337', '2326621', '2320205', '2412782', '2352466', '2414097', '2381526', '2390981', '2375715', '2405096', '2402848', '2346948', '2387882', '2352243', '2337581', '2330919', '2380444', '2377611', '2354637', '2401996', '2391375', '2415510', '2400079', '2366081', '2360527', '2365490', '2336229', '2367475', '2373691', '2376356', '2368952', '2369653', '2355012', '2400892', '2369971', '2414408', '2394007', '2415978', '2405444', '2401822', '2387651', '2350622', '2318603', '2383962', '2377677', '2380510', '2392746', '2412229', '2392926', '2362383', '2398991', '2367805', '2360264', '2400991', '2354133', '2374619', '2413514', '2387844', '2349764', '2403572', '2372008', '2388111', '2366871', '2385053', '2400907', '2379868', '2389754', '2389615', '2365263', '2414350', '2409171', '4025', '2352062', '2378864', '2367278', '2319025', '2335091', '2344088', '2351081', '2408962', '2319561', '2400672', '2390977', '2390288', '2415083', '2396514', '2411955', '2346205', '2395037', '2346498', '2383586', '2346342', '2367250', '2348629', '2413978', '2355118', '2390609', '724', '2347956', '2389121', '2381396', '2394635', '2374001', '2319411', '2356016', '2390189', '2339582', '2362277', '2325410', '2354590', '2367096', '2317592', '2384171', '2397387', '2376251', '2352643', '2354147', '2410352', '2332365', '2323588', '2504', '2346067', '2373700', '2395463', '2388627', '2344272', '2365691', '2364664', '2410717', '2371950', '2402941', '2391252', '2342646', '2352971', '2370165', '2395630', '2317591', '2378059', '2355709', '2336832', '2391722', '2330764', '2346660', '2409862', '2373155', '2393166', '2355642', '2346977', '2377832', '2343586', '2388431', '2381416', '2408088', '2343537', '2390500', '2319274', '2358132', '2336159', '2386365', '2395072', '2405965', '2390610', '2392765', '2363501', '2322189', '2400936', '2354380', '2385725', '2414772', '2387049', '2339305', '2355163', '2411856', '2325076', '2391875', '2369075', '2404587', '2371926', '2395640', '2329779', '2402384', '2348939', '2361855', '2395696', '2391663', '2370517', '2412037', '2347202', '2353100', '2372426', '2405703', '2370496', '2375612', '2338355', '2342324', '2400462', '2396112', '2400862', '2379832', '2333467', '2359089', '2398544', '2328805', '2345621', '2397165', '2391502', '2403224', '2356812', '2389832', '2399569', '2414157', '2392065', '2353938', '2374300', '2411459', '2390394', '2354220', '2380380', '2363108', '2342406', '2334854', '2410297', '2335993', '2411121', '2366378', '2401969', '2387542', '2380071', '2393134', '2315498', '2367637', '2370780', '2317055', '2383295', '2400542', '2402050', '2390890', '2386260', '2377837', '2318159', '2327364', '2409644', '2396452', '2341431', '2381222', '2389464', '2406927', '2371744', '2392787', '2376996', '2340394', '2333581', '2398578', '2348334', '2408546', '2354873', '2384670', '2408339', '2316661', '2318214', '2369502', '2344294', '2316382', '2389172', '2412334', '2382722', '2373092', '2400913', '2364094', '2415823', '2407666', '2408407', '2353425', '2385066', '2350985', '2396301', '2355676', '2318134', '2368058', '2359892', '2327449', '2342078', '2365122', '2416996', '2365457', '2376076', '2397271', '2317228', '2333227', '2343001', '2335613', '2336076', '2363469', '2387136', '2387602', '538', '2381047', '2341390', '2382974', '2393524', '2389704', '2352288', '2366841', '2366424', '2414990', '2382610', '2359501', '2341126', '2406861', '2381613', '2381116', '2362015', '2364508', '2361773', '2386125', '2323005', '2373886', '2391950', '2323158', '2372145', '2392202', '2362921', '150344', '2415140', '2375237', '2397509', '2405568', '2410491', '2365504', '2372907', '2359383', '2384101', '2330124', '2341434', '2403182', '2393422', '2350678', '2403683', '2334135', '2379332', '2382744', '2400289', '2351156', '2354589', '2366570', '2393002', '2375133', '2372957', '2346843', '2376314', '2357994', '2389532', '2355449', '2387258', '2386636', '2381310', '2331045', '2362731', '2353668', '2390084', '2347783', '2383040', '2341770', '2367808', '2397944', '2389426', '2391869', '2351511', '2406056', '2347102', '2414155', '2396971', '2404188', '2352834', '2397549', '2335504', '2390711', '2408710', '2318479', '2345601', '2323690', '2393174', '3665', '2343876', '2327045', '2414418', '2350160', '2360570', '2416836', '2410413', '2359369', '2407102', '4329', '2317914', '2375681', '2385986', '2357723', '2390201', '2376235', '2366660', '2346881', '2345363', '2358150', '853', '1067', '2391849', '2347742', '2374809', '2363649', '2317707', '2394449', '2407504', '2366958', '815', '2381442', '2386687', '2343486', '2403767', '2342183', '2342649', '2360855', '2384747', '2369741', '2338878', '2350786', '2377187', '2391182', '2345464', '2382463', '2373922', '2364569', '2321507', '2391257', '2390199', '2376974', '2411637', '2414334', '2382975', '2374130', '2407054', '2376001', '2405490', '2391435', '2360573', '2356093', '2317319', '2376855', '2372392', '2399545', '2407631', '2414694', '2387731', '2380250', '2399949', '2381902', '2380617', '2387786', '2343356', '2316729', '2335572', '2342386', '2341113', '2407180', '2372676', '2408452', '2351275', '2327618', '2391545', '2406319', '2369158', '2408036', '2409641', '2318461', '2376879', '2382464', '2358660', '2397224', '2360963', '2356329', '2397683', '2398695', '2318316', '2369188', '2383665', '2351900', '2355768', '2341725', '2335274', '2410913', '2404585', '2383900', '2370017', '2406329', '2375118', '2389880', '2333038', '2345388', '2404202', '2345290', '2412080', '2336489', '2379551', '2352259', '2398246', '2322086', '2412178', '2389036', '2411172', '2377272', '2369601', '2399464', '2351596', '2403297', '2353451', '2394454', '2401819', '2398902', '2394152', '2385541', '2369013', '2385974', '2344284', '2365226', '2322420', '2358081', '1160101', '2316641', '2371907', '2384500', '2344095', '2409873', '2342700', '2368261', '2349084', '2373021', '2339331', '2401288', '2334516', '2344881', '2364452', '2349469', '2374709', '2406451', '2391194', '2394298', '2406376', '2371567', '2335457', '2407029', '2372220', '2357948', '2384021', '2388187', '2331430', '2367019', '2392600', '2370343', '2417500', '2346802', '2564', '2360148', '2407311', '2350000', '2374917', '2354814', '2352832', '2383787', '2387276', '2413861', '2336315', '2357005', '2356221', '2400369', '2349353', '2366432', '2405436', '2407604', '2318841', '2394677', '2384439', '2348689', '2413793', '2384089', '2373382', '2402051', '2330715', '2403512', '2379975', '2411960', '2363971', '2353696', '2384283', '2395949', '2352715', '2333164', '2364799', '2399390', '2341305', '2410276', '2319462', '2333402', '2405554', '2344657', '2328347', '2415027', '2356488', '2367352', '2400108', '2380279', '2396396', '2339028', '2363834', '2366302', '2361964', '2343596', '2382173', '2371493', '2362648', '2381158', '2392123', '2371441', '2393319', '2353865', '2412966', '2402825', '2356671', '2385174', '2375529', '2354544', '2323848', '2382977', '2323209', '2387933', '2341028', '2411576', '2356811', '2387207', '2338341', '2381847', '2396662', '2367762', '2391902', '2350959', '2393496', '2366615', '2344117', '2333815', '2352342', '2387761', '2346469', '2382632', '2337106', '2357507', '2368903', '2400371', '2377573', '2399802', '2374100', '2387464', '2410958', '2316254', '2355876', '2319466', '2387553', '2387137', '2401818', '2413411', '2408481', '2320306', '2343554', '2383435', '2341747', '2411945', '2395545', '2347126', '2383269', '2341187', '2395566', '2361388', '2326048', '2326483', '2342090', '2330104', '2388858', '2414131', '2328024', '2405963', '2321198', '2342281', '2408454', '2407660', '2350676', '2324748', '2364648', '2399525', '2367980', '2369455', '2388006', '2406791', '2338235', '2399119', '2367983', '2398222', '2376293', '2379307', '2318313', '2395464', '2355425', '2344467', '2397941', '2412770', '2378990', '2391984', '2327406', '2376475', '2382690', '2414448', '2322107', '2414173', '2377368', '2398978', '2334930', '2354564', '2394297', '2355839', '2355172', '2388186', '2334825', '2372450', '2377971', '2377053', '2393315', '2370179', '2349895', '2335023', '2366720', '2369272', '2344127', '2393434', '2339219', '2348398', '2413035', '2330639', '2316575', '2343155', '2353872', '2414882', '2373063', '2333717', '2350248', '2383119', '2404930', '2335333', '2380117', '2400689', '2377131', '2353667', '2406595', '2343214', '2410358', '2330044', '2377161', '2347777', '2396968', '2343948', '2408400', '2342794', '2388737', '2401886', '2341672', '2385815', '2407806', '2317356', '2347763', '2341717', '2357770', '2378698', '2392376', '2373225', '1979', '2387187', '2341245', '2399182', '2410305', '2350362', '2354852', '2357069', '2391111', '2396639', '2343276', '2385145', '2319237', '2398879', '2378517', '2320078', '2357038', '2378426', '2409995', '2396407', '2344540', '2403306', '2412032', '2364541', '2379607', '2363110', '2385571', '2367180', '2317743', '2345625', '2324168', '2382524', '2389666', '2342125', '2326672', '2358834', '2416678', '2412679', '2373510', '2409161', '2330294', '2385025', '2382543', '2407293', '2405007', '2319434', '2409065', '2374686', '2415570', '2370645', '2343220', '2385158', '2366216', '2411891', '2396279', '2378386', '2406971', '2364907', '2383068', '2346025', '2401071', '2394536', '2346609', '2366521', '2336144', '2369882', '2362607', '2352327', '2343623', '2348208', '2357580', '2393529', '2347586', '2410663', '2338501', '2414226', '2317379', '2384614', '2368599', '2339675', '2392812', '2404620', '2362539', '2329100', '2343589', '2385552', '2412438', '2415173', '2376132', '2392014', '2346010', '285802', '2366148', '2416104', '2369615', '2367875', '2398860', '2318174', '2372154', '2327125', '2343802', '2364976', '2391397', '2412087', '2382659', '2325536', '2343595', '2397030', '2390728', '2409300', '2375570', '2363267', '2379412', '2318100', '2381952', '2410088', '2390131', '2350025', '2357873', '2332037', '2407133', '2387000', '2412304', '2334742', '2360125', '2351557', '2322620', '2364598', '2409948', '2407622', '2409730', '2395914', '2344000', '2367747', '2355445', '2336558', '2326536', '2336255', '2341700', '2387514', '2341671', '2398110', '2399652', '1981', '2323460', '2390548', '2378480', '2359190', '2388960', '2374610', '2390517', '2374041', '2416167', '2359860', '2345744', '2390516', '2350734', '2320101', '2362840', '2355186', '2374316', '2394482', '2345532', '2403827', '2350800', '2384225', '2408041', '2372039', '2362947', '2396741', '2328602', '2403866', '2408144', '2339689', '2373049', '2396987', '2399359', '2356053', '2371761', '2374295', '2400094', '2368870', '358', '2316710', '2348163', '2413458', '2315631', '2365595', '2409738', '2388139', '2367957', '2335492', '2350434', '2337232', '2317038', '2399813', '2364830', '2358273', '2317457', '2343118', '2325817', '2316887', '285905', '2357254', '1592902', '2387245', '2374212', '2393912', '2412430', '2396351', '2364742', '2326373', '2354526', '2410548', '2374003', '2339208', '2412323', '2347068', '2410762', '2354059', '2357064', '2394457', '2364582', '2388352', '2412817', '2345982', '2331553', '2344607', '2401917', '2335275', '2401129', '2410083', '2411500', '2398309', '2385216', '2381550', '2341363', '2415473', '2402262', '2391262', '2368478', '2344295', '2340547', '2326050', '2339004', '2346626', '2316675', '2372940', '2403736', '2361538', '2335133', '2372110', '2355888', '2408983', '2318897', '2349203', '2402756', '2385127', '1947', '2414540', '2360127', '2403553', '2361204', '2382861', '2340485', '2318856', '2349149', '2357096', '2378937', '2330143', '2390945', '2347477', '2367727', '2382133', '2365803', '2385464', '2352560', '2389937', '2335666', '2359041', '2371361', '2402586', '2400198', '2367081', '2330999', '2407240', '2373013', '2333510', '2341668', '2398193', '2384410', '2394250', '2401035', '2382813', '2369897', '2413842', '2335959', '2381711', '2394115', '2394823', '2342230', '2395930', '2375161', '2371998', '2357321', '2381052', '2386678', '2359589', '2408377', '2319233', '2377371', '2356978', '2322528', '2342127', '2351858', '2394808', '2372836', '2350140', '2330793', '2365464', '2324955', '59', '2401816', '2363606', '2411900', '2387088', '2400434', '2408594', '2379456', '2366127', '2322573', '2365692', '2354909', '2381999', '2400554', '2377456', '2338012', '2400676', '2409993', '2409525', '2329113', '2353014', '2375625', '2413783', '2405811', '2366795', '2374646', '2388879', '2317337', '2327958', '2378809', '2372018', '2397405', '2396019', '2358287', '2393934', '2411311', '2341541', '2326852', '2372176', '2371733', '2352132', '2365664', '2316733', '2365437', '1592372', '2358414', '2409530', '2410977', '2330167', '2370025', '2340772', '2358472', '2341383', '2415196', '2367052', '2349022', '2380268', '2394057', '2331963', '2348762', '2393268', '2382522', '2354169', '2346439', '2360521', '2413923', '2322133', '2318491', '2349530', '2388772', '2362719', '2406548', '2360201', '2351834', '2341514', '2400161', '2394967', '2316555', '2386847', '2412609', '2411640', '2376384', '2349745', '2358575', '2362360', '2319185', '2318300', '2319338', '2341703', '2348223', '2411941', '2404878', '2388166', '2348265', '2393093', '2386134', '2317860', '2358790', '2392630', '2381546', '2410233', '2396340', '2413134', '2384897', '2372279', '2412901', '2392163', '2399340', '2362560', '2415464', '2355350', '2385072', '2387946', '2414959', '2335223', '2396029', '2409922', '2369585', '2340615', '2342329', '2357014', '2359654', '2394493', '2339690', '2367978', '2385255', '2409589', '2404541', '2411332', '2395373', '2340926', '2379898', '2366907', '2392500', '2381777', '2371763', '2323566', '2362679', '2390775', '2326661', '2400330', '2354989', '2359414', '2351277', '2345482', '2347852', '2361424', '2363496', '2374097', '2392960', '2363592', '2338512', '2350783', '2390282', '2361585', '2346556', '2393655', '2381013', '2380114', '2413540', '2410181', '2366603', '2405292', '2352992', '2343612', '2395465', '2395628', '2386850', '2380640', '2368472', '2320897', '2404090', '2344453', '2320614', '2375431', '2354513', '2352072', '2368501', '2382276', '2327911', '2397716', '2354631', '2413575', '2327559', '2389618', '2370697', '2375622', '2403565', '2360115', '2399996', '2402523', '3668', '2342671', '2393354', '2358421', '2403502', '2358670', '2389588', '2401096', '2356526', '2380095', '2346818', '2371255', '2391668', '2385831', '2394596', '2342960', '2374375', '1160254', '2375771', '2382933', '2392920', '2387989', '2324588', '2356074', '2317625', '2344661', '2326911', '2404259', '2361721', '2393279', '2386495', '2400046', '2365043', '2344637', '2357554', '2390759', '2318426', '2386258', '2382545', '2336325', '2392061', '2405822', '2401033', '2353349', '2412624', '2340180', '2397756', '2402816', '2367884', '2323076', '2328094', '2402273', '2369292', '2414256', '2361129', '2379258', '2342678', '2383042', '2375286', '543', '2369707', '2415363', '2351567', '2374164', '2318354', '2337138', '2416386', '2344622', '2408617', '2402485', '2390061', '2403911', '2385461', '2368073', '2325644', '2413346', '2333242', '2409746', '2318784', '2356563', '2350520', '2379974', '2393488', '2401112', '2340853', '2355566', '2338820', '2388446', '2408244', '2403749', '2339735', '2345624', '2381108', '2396179', '2382666', '2377454', '2376391', '2347274', '2384475', '2317896', '2398386', '2343705', '2366203', '2336091', '2318766', '2362130', '2364869', '2388733', '2364463', '2320207', '2399556', '2318740', '2319024', '2320357', '2349108', '2324753', '2357052', '2405788', '2367095', '2355641', '2361523', '2348566', '2369937', '2364778', '2410070', '2349065', '2364313', '2385956', '2330315', '2339046', '2347982', '2389147', '2411813', '2366650', '2316025', '2340085', '2349725', '2402374', '2389782', '2318529', '2353047', '2355690', '2407520', '2340975', '2387531', '2355380', '2361208', '2405966', '2350585', '2334735', '2415776', '2391209', '2364493', '2404485', '2336765', '2390589', '2407074', '2355399', '2396682', '2359934', '2390911', '2352126', '2367144', '2393988', '2319437', '2384790', '2332609', '2363103', '2363314', '2371312', '2402053', '2415913', '2387492', '2355933', '2408558', '2368969', '2341785', '2391385', '2366303', '2338004', '2331437', '2405390', '2318074', '2382213', '2352218', '2356795', '2366679', '2373340', '2410425', '2347072', '2315529', '2414805', '2334296', '2400705', '2348349', '2371338', '2355887', '2336426', '2409509', '2400488', '2347040', '2373504', '2400709', '2388269', '2374858', '2357848', '2351724', '2366552', '2373343', '2366798', '2370562', '2336279', '2390109', '2376359', '2403968', '2377951', '2383724', '2327224', '2345537', '2320748', '2352059', '2356407', '2379040', '2404395', '2335228', '2377815', '2346974', '2398486', '2385876', '2368358', '2328180', '2333993', '2412028', '2320127', '2361275', '2325956', '2344077', '2373936', '2400518', '2410757', '2388557', '2374409', '2346288', '2327758', '2360829', '2367594', '2351200', '2401206', '2347475', '2361164', '2347157', '2322190', '2387508', '2328285', '2332131', '2399529', '2406648', '2414784', '2390009', '2385308', '2411532', '2320131', '2366219', '2385693', '2367729', '2383497', '2399749', '2387687', '2318508', '2366431', '2346215', '2332290', '2396197', '2343376', '2355111', '2315963', '2360432', '2329225', '2414189', '2407349', '2354529', '2378366', '2360200', '2386619', '2358729', '2396069', '2414523', '2402102', '2316860', '2359321', '2390811', '2403746', '2392451', '2370342', '1512', '2351235', '2356083', '2352140', '2397229', '2341890', '2377512', '2411735', '2384658', '1159516', '2360315', '2332658', '2413457', '2415457', '2349652', '2359079', '2324375', '2397605', '2403889', '2389796', '2406696', '2334724', '2407120', '2336668', '2366034', '2384206', '2400680', '2332379', '2392818', '2406733', '2387944', '2402124', '2334363', '2331362', '2388172', '2382849', '2388891', '713323', '2407039', '2386334', '2394687', '2408468', '2350752', '2384387', '2412787', '2409238', '2401910', '2346934', '2342976', '2362198', '2319960', '2384406', '2360405', '2399333', '2323665', '1159963', '2379625', '2405682', '2373934', '2337314', '2346066', '2396448', '2317900', '2407413', '2328394', '2374389', '2382714', '2339245', '2316574', '2363770', '2390523', '2415871', '2390619', '2353681', '2375141', '2365887', '2392468', '2376016', '2339907', '2386673', '2339891', '2414494', '2356926', '2351882', '2355581', '2380553', '2405893', '2375873', '2379970', '2389093', '2349339', '2351669', '2376371', '2382695', '2342062', '2343739', '2403096', '2367587', '2393255', '2363034', '2411245', '2356290', '2333032', '2377054', '2381840', '2383123', '2417815', '2342463', '2346654', '2401911', '2330589', '2359948', '2348627', '2395330', '2325024', '2389270', '2366510', '2373821', '2392869', '2381761', '2415192', '2324220', '2359272', '2386989', '2372029', '2317409', '2382557', '2315640', '2366640', '2401940', '2332111', '2371625', '2398866', '2399393', '2408374', '2372030', '2322341', '2374141', '1592381', '2346658', '2391256', '2397008', '2405504', '2378739', '2384906', '2408673', '2396317', '2388519', '2365979', '2414365', '2388526', '2337365', '2348280', '2360988', '2363427', '2368396', '2405392', '2396760', '2398451', '2400579', '2357859', '2348172', '2388316', '2357856', '2323413', '2400412', '2389597', '2364522', '2413323', '2336219', '2367716', '2362819', '2396757', '2380218', '2358578', '2390032', '2409345', '2410852', '2384053', '2378686', '2400643', '2380143', '2389002', '2320951', '2354093', '2362892', '2408480', '2361074', '2368771', '2409608', '2371432', '2344878', '2318431', '2403123', '2410146', '2342919', '2391895', '2413037', '2415154', '2352917', '2336606', '2409059', '2351967', '2344341', '2402305', '2399436', '2399831', '2344880', '2366243', '2394911', '2340505', '2360192', '2355583', '2412691', '2317478', '2377654', '2393764', '2369389', '2380908', '2359734', '2395479', '2410117', '2345963', '2348675', '2402610', '2362917', '2381263', '2411367', '2350934', '2345484', '2375389', '2338460', '2392194', '2345283', '2357009', '2366433', '2400181', '2338625', '2385349', '2409624', '2325906', '2410111', '2356573', '2408231', '2374654', '2401986', '2406374', '2408091', '2388101', '2366266', '2358240', '2387174', '2364629', '2410500', '2396864', '2349287', '2341175', '2413048', '2352077', '2357255', '2364516', '2393711', '2332173', '2370652', '2379368', '2373368', '2391968', '2363840', '2356915', '2402864', '2347167', '2361726', '2363057', '195', '2389587', '2333962', '2339034', '2358787', '2413030', '2382967', '2356452', '2355462', '2398408', '2350770', '2361025', '2402846', '2341989', '2356957', '2371802', '2340693', '2365687', '2360971', '2383081', '2397831', '2382252', '2345262', '2412694', '2350816', '2356774', '2379797', '2355306', '2348088', '2392089', '2354342', '2414493', '2317122', '2411125', '2348278', '2380488', '2409861', '2414069', '2361015', '1186', '2348951', '2408872', '2358566', '2402174', '2376953', '2416072', '2399932', '2330371', '4526', '2368076', '2334644', '813', '2415207', '2406271', '262', '2400363', '2405126', '2392176', '2373460', '2413099', '2352124', '2317117', '2364952', '2345382', '2387583', '2386515', '2411830', '2344135', '2368524', '2354957', '2395289', '2345826', '2399843', '2360483', '2392613', '2346922', '2340767', '2395698', '2363261', '2341640', '2377290', '2379419', '2317513', '2405522', '2365947', '2371836', '2374929', '2348401', '2343057', '2339483', '2369764', '2393778', '2364711', '2331127', '1143', '2371418', '2403431', '2414878', '2384911', '2346679', '2333255', '2393843', '2322397', '2412964', '2341844', '2400168', '2371705', '2390876', '2336110', '587', '2364458', '2400258', '2387213', '2412725', '2408943', '2413997', '2369025', '3524', '2345255', '2387936', '2348847', '2361143', '2356669', '2351483', '2399453', '2392336', '2399058', '2387509', '2346365', '2387430', '2323101', '2393641', '2403291', '2352426', '2346311', '2334281', '2365395', '2367554', '2379698', '2356542', '2374256', '2378992', '2379893', '2395881', '1159529', '2347316', '2398565', '61515', '2382188', '2371228', '2374191', '2405763', '2373065', '2368332', '2366857', '3974', '2403142', '2403730', '2348008', '2364666', '2383971', '2323317', '2348859', '2351300', '2393837', '2360110', '2366855', '2410116', '2399588', '2404970', '2411064', '2372245', '2396669', '2387797', '2347378', '2330444', '2397123', '2367427', '2374351', '2360943', '2331431', '2338001', '2346419', '2317987', '2399954', '2415472', '2349266', '2384407', '2339402', '2395507', '2410775', '2318528', '2408638', '2358223', '2390577', '2360637', '2403309', '2389127', '2400665', '2345254', '2357568', '2368103', '2376750', '2362276', '2373822', '2318753', '2381335', '2396390', '2407563', '677', '2383588', '2364405', '2363823', '2332047', '2369980', '2411794', '2375098', '2349488', '2412833', '2389931', '2411635', '2321324', '2351789', '2330962', '2325271', '2393810', '2386491', '2351574', '2386840', '2342128', '2360241', '2389642', '2381424', '2378294', '2376247', '2387201', '2320667', '2376728', '2398703', '2369310', '2367175', '2324573', '2344722', '2341732', '2336105', '2388874', '2341737', '2322795', '2378412', '2387370', '2369286', '2318253', '2343618', '2354019', '2402340', '2392557', '2397147', '2353466', '2402479', '2393995', '2329101', '2404918', '2347211', '2345294', '2385897', '2319741', '2414099', '2403177', '2319593', '2402961', '2361554', '2356491', '2392581', '2316889', '2360287', '2371216', '2407026', '2367533', '2378060', '2389282', '2390572', '2397120', '2400392', '2316318', '2375898', '2362311', '2401191', '2405981', '2409153', '2377072', '2355213', '2392583', '2382441', '2317053', '2346718', '2350383', '2406500', '2362770', '2410681', '2374807', '2346501', '2357249', '2359027', '2396374', '2374760', '2399256', '2336180', '2355922', '2356858', '2340777', '2381824', '2377986', '2386257', '2397845', '2389726', '2411629', '2393060', '2345476', '2374490', '2393513', '2409142', '2396465', '2356592', '2398489', '2417769', '2380388', '2386965', '2388564', '2360225', '2377334', '2396670', '2354660', '2357502', '2359450', '2378490', '2362795', '2363104', '2335727', '2390392', '2343913', '2373951', '2377309', '2392460', '2354423', '2359238', '2379480', '2374611', '2384854', '2369580', '2326856', '2332039', '2411249', '2406478', '2328010', '2369213', '2387943', '2385460', '2330218', '2360556', '2341674', '2342464', '2352565', '2341047', '2401918', '2372733', '2371551', '2316697', '2376379', '2368666', '2402546', '1592156', '2388838', '2397307', '2348182', '2413071', '2404732', '2356427', '2407637', '2390263', '2405776', '2353337', '2316602', '2353270', '2408708', '2371198', '2354266', '2369855', '2380368', '2382039', '2358567', '2319011', '2370341', '2370551', '2362808', '2382330', '2408653', '2374473', '2362280', '2398037', '2375753', '2371017', '2370877', '2408209', '2333007', '2405738', '2391153', '2372453', '2358526', '2379303', '2389636', '2342857', '2415640', '2386870', '2363443', '2368634', '2413904', '2383358', '2380024', '2416559', '2336529', '2366500', '2351655', '2345044', '2373588', '2390099', '2373407', '2350763', '2402155', '2363548', '2322813', '2343805', '2391461', '2416656', '2402786', '2394339', '2414255', '2338443', '2382381', '2374927', '2362555', '2402202', '2382046', '2412163', '2354362', '2415082', '2408489', '2406617', '2319554', '2327849', '2378204', '2391047', '2341611', '2342864', '2386983', '2347702', '2322974', '2325414', '2400737', '2404002', '2392261', '2356269', '2358999', '2399577', '2370776', '2316535', '2395996', '2336048', '2376467', '2362203', '2415820', '2383182', '2350730', '2345893', '2375076', '2417944', '2337033', '2359636', '2408359', '2416307', '2398017', '2383168', '2330468', '2405056', '2321253', '2341034', '2317165', '2401897', '2398212', '2387721', '2344595', '2379759', '2393164', '2410345', '2363437', '2386462', '2388504', '2401336', '2321387', '2317132', '2396478', '2416725', '2372569', '2359537', '2322095', '2352174', '2343766', '2380329', '2361489', '2413702', '2321676', '2404393', '2356073', '2376970', '2358293', '2342802', '2402837', '2363661', '2403903', '2319419', '2344370', '2389353', '2397657', '2371546', '2347170', '2404791', '2384368', '2417585', '2392507', '2349036', '2329559', '879', '2414977', '2407808', '2378572', '2415884', '2414803', '2402120', '2320356', '2365699', '2354877', '2389638', '2334130', '2367765', '2379154', '2399862', '2404467', '2353974', '2352853', '2339578', '2406885', '2382775', '2354318', '2353172', '2412056', '2396290', '2336760', '2378276', '2379292', '2406213', '2416937', '2395164', '2356436', '2355527', '2386443', '2397502', '2410270', '2397071', '2415676', '2371756', '2394933', '2377521', '2320584', '2394242', '2385172', '2414806', '2380874', '2371037', '2317584', '2390476', '2356914', '2361991', '2350826', '2406588', '2358311', '2409964', '2403418', '2370296', '2411605', '2323547', '2316204', '2375241', '2349448', '2354115', '2404178', '2372285', '2401764', '2377897', '2328135', '2382879', '2360242', '2350037', '2361415', '2367100', '2411866', '2378223', '2346003', '2359944', '2386442', '2327810', '2345716', '2398747', '2327729', '2379103', '2330650', '2391165', '2403131']\n"
     ]
    }
   ],
   "source": [
    "    model_path = './models_batch/'\n",
    "    train_feats_path = './im2p_val_output.h5'\n",
    "    \n",
    "    train_output_file = h5py.File(train_feats_path, 'r')\n",
    "    #print(\"Train feats path--------------------------------------------->>>>>>>>\",train_output_file)\n",
    "    train_feats = train_output_file.get('feats')\n",
    "    #print(\"Train feats->>>>>>>>>>>>>>>>\",train_feats)\n",
    "    train_imgs_full_path_lists = open('./imgs_val_path.txt').read().splitlines()\n",
    "    #print(\"paths>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\",train_imgs_full_path_lists)\n",
    "    train_imgs_names = map(lambda x: os.path.basename(x).split('.')[0], train_imgs_full_path_lists)\n",
    "    #print(\"Train images names\",list(train_imgs_names))\n",
    "    \n",
    "    print(\"Train images names\",list(train_imgs_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_imgs_names------- ['2385757', '2389264', '2407325', '2337141', '2345813', '2386694', '2368581', '2397809', '185', '2387911', '2343682', '2409165', '2357129', '498337', '2326621', '2320205', '2412782', '2352466', '2414097', '2381526', '2390981', '2375715', '2405096', '2402848', '2346948', '2387882', '2352243', '2337581', '2330919', '2380444', '2377611', '2354637', '2401996', '2391375', '2415510', '2400079', '2366081', '2360527', '2365490', '2336229', '2367475', '2373691', '2376356', '2368952', '2369653', '2355012', '2400892', '2369971', '2414408', '2394007', '2415978', '2405444', '2401822', '2387651', '2350622', '2318603', '2383962', '2377677', '2380510', '2392746', '2412229', '2392926', '2362383', '2398991', '2367805', '2360264', '2400991', '2354133', '2374619', '2413514', '2387844', '2349764', '2403572', '2372008', '2388111', '2366871', '2385053', '2400907', '2379868', '2389754', '2389615', '2365263', '2414350', '2409171', '4025', '2352062', '2378864', '2367278', '2319025', '2335091', '2344088', '2351081', '2408962', '2319561', '2400672', '2390977', '2390288', '2415083', '2396514', '2411955', '2346205', '2395037', '2346498', '2383586', '2346342', '2367250', '2348629', '2413978', '2355118', '2390609', '724', '2347956', '2389121', '2381396', '2394635', '2374001', '2319411', '2356016', '2390189', '2339582', '2362277', '2325410', '2354590', '2367096', '2317592', '2384171', '2397387', '2376251', '2352643', '2354147', '2410352', '2332365', '2323588', '2504', '2346067', '2373700', '2395463', '2388627', '2344272', '2365691', '2364664', '2410717', '2371950', '2402941', '2391252', '2342646', '2352971', '2370165', '2395630', '2317591', '2378059', '2355709', '2336832', '2391722', '2330764', '2346660', '2409862', '2373155', '2393166', '2355642', '2346977', '2377832', '2343586', '2388431', '2381416', '2408088', '2343537', '2390500', '2319274', '2358132', '2336159', '2386365', '2395072', '2405965', '2390610', '2392765', '2363501', '2322189', '2400936', '2354380', '2385725', '2414772', '2387049', '2339305', '2355163', '2411856', '2325076', '2391875', '2369075', '2404587', '2371926', '2395640', '2329779', '2402384', '2348939', '2361855', '2395696', '2391663', '2370517', '2412037', '2347202', '2353100', '2372426', '2405703', '2370496', '2375612', '2338355', '2342324', '2400462', '2396112', '2400862', '2379832', '2333467', '2359089', '2398544', '2328805', '2345621', '2397165', '2391502', '2403224', '2356812', '2389832', '2399569', '2414157', '2392065', '2353938', '2374300', '2411459', '2390394', '2354220', '2380380', '2363108', '2342406', '2334854', '2410297', '2335993', '2411121', '2366378', '2401969', '2387542', '2380071', '2393134', '2315498', '2367637', '2370780', '2317055', '2383295', '2400542', '2402050', '2390890', '2386260', '2377837', '2318159', '2327364', '2409644', '2396452', '2341431', '2381222', '2389464', '2406927', '2371744', '2392787', '2376996', '2340394', '2333581', '2398578', '2348334', '2408546', '2354873', '2384670', '2408339', '2316661', '2318214', '2369502', '2344294', '2316382', '2389172', '2412334', '2382722', '2373092', '2400913', '2364094', '2415823', '2407666', '2408407', '2353425', '2385066', '2350985', '2396301', '2355676', '2318134', '2368058', '2359892', '2327449', '2342078', '2365122', '2416996', '2365457', '2376076', '2397271', '2317228', '2333227', '2343001', '2335613', '2336076', '2363469', '2387136', '2387602', '538', '2381047', '2341390', '2382974', '2393524', '2389704', '2352288', '2366841', '2366424', '2414990', '2382610', '2359501', '2341126', '2406861', '2381613', '2381116', '2362015', '2364508', '2361773', '2386125', '2323005', '2373886', '2391950', '2323158', '2372145', '2392202', '2362921', '150344', '2415140', '2375237', '2397509', '2405568', '2410491', '2365504', '2372907', '2359383', '2384101', '2330124', '2341434', '2403182', '2393422', '2350678', '2403683', '2334135', '2379332', '2382744', '2400289', '2351156', '2354589', '2366570', '2393002', '2375133', '2372957', '2346843', '2376314', '2357994', '2389532', '2355449', '2387258', '2386636', '2381310', '2331045', '2362731', '2353668', '2390084', '2347783', '2383040', '2341770', '2367808', '2397944', '2389426', '2391869', '2351511', '2406056', '2347102', '2414155', '2396971', '2404188', '2352834', '2397549', '2335504', '2390711', '2408710', '2318479', '2345601', '2323690', '2393174', '3665', '2343876', '2327045', '2414418', '2350160', '2360570', '2416836', '2410413', '2359369', '2407102', '4329', '2317914', '2375681', '2385986', '2357723', '2390201', '2376235', '2366660', '2346881', '2345363', '2358150', '853', '1067', '2391849', '2347742', '2374809', '2363649', '2317707', '2394449', '2407504', '2366958', '815', '2381442', '2386687', '2343486', '2403767', '2342183', '2342649', '2360855', '2384747', '2369741', '2338878', '2350786', '2377187', '2391182', '2345464', '2382463', '2373922', '2364569', '2321507', '2391257', '2390199', '2376974', '2411637', '2414334', '2382975', '2374130', '2407054', '2376001', '2405490', '2391435', '2360573', '2356093', '2317319', '2376855', '2372392', '2399545', '2407631', '2414694', '2387731', '2380250', '2399949', '2381902', '2380617', '2387786', '2343356', '2316729', '2335572', '2342386', '2341113', '2407180', '2372676', '2408452', '2351275', '2327618', '2391545', '2406319', '2369158', '2408036', '2409641', '2318461', '2376879', '2382464', '2358660', '2397224', '2360963', '2356329', '2397683', '2398695', '2318316', '2369188', '2383665', '2351900', '2355768', '2341725', '2335274', '2410913', '2404585', '2383900', '2370017', '2406329', '2375118', '2389880', '2333038', '2345388', '2404202', '2345290', '2412080', '2336489', '2379551', '2352259', '2398246', '2322086', '2412178', '2389036', '2411172', '2377272', '2369601', '2399464', '2351596', '2403297', '2353451', '2394454', '2401819', '2398902', '2394152', '2385541', '2369013', '2385974', '2344284', '2365226', '2322420', '2358081', '1160101', '2316641', '2371907', '2384500', '2344095', '2409873', '2342700', '2368261', '2349084', '2373021', '2339331', '2401288', '2334516', '2344881', '2364452', '2349469', '2374709', '2406451', '2391194', '2394298', '2406376', '2371567', '2335457', '2407029', '2372220', '2357948', '2384021', '2388187', '2331430', '2367019', '2392600', '2370343', '2417500', '2346802', '2564', '2360148', '2407311', '2350000', '2374917', '2354814', '2352832', '2383787', '2387276', '2413861', '2336315', '2357005', '2356221', '2400369', '2349353', '2366432', '2405436', '2407604', '2318841', '2394677', '2384439', '2348689', '2413793', '2384089', '2373382', '2402051', '2330715', '2403512', '2379975', '2411960', '2363971', '2353696', '2384283', '2395949', '2352715', '2333164', '2364799', '2399390', '2341305', '2410276', '2319462', '2333402', '2405554', '2344657', '2328347', '2415027', '2356488', '2367352', '2400108', '2380279', '2396396', '2339028', '2363834', '2366302', '2361964', '2343596', '2382173', '2371493', '2362648', '2381158', '2392123', '2371441', '2393319', '2353865', '2412966', '2402825', '2356671', '2385174', '2375529', '2354544', '2323848', '2382977', '2323209', '2387933', '2341028', '2411576', '2356811', '2387207', '2338341', '2381847', '2396662', '2367762', '2391902', '2350959', '2393496', '2366615', '2344117', '2333815', '2352342', '2387761', '2346469', '2382632', '2337106', '2357507', '2368903', '2400371', '2377573', '2399802', '2374100', '2387464', '2410958', '2316254', '2355876', '2319466', '2387553', '2387137', '2401818', '2413411', '2408481', '2320306', '2343554', '2383435', '2341747', '2411945', '2395545', '2347126', '2383269', '2341187', '2395566', '2361388', '2326048', '2326483', '2342090', '2330104', '2388858', '2414131', '2328024', '2405963', '2321198', '2342281', '2408454', '2407660', '2350676', '2324748', '2364648', '2399525', '2367980', '2369455', '2388006', '2406791', '2338235', '2399119', '2367983', '2398222', '2376293', '2379307', '2318313', '2395464', '2355425', '2344467', '2397941', '2412770', '2378990', '2391984', '2327406', '2376475', '2382690', '2414448', '2322107', '2414173', '2377368', '2398978', '2334930', '2354564', '2394297', '2355839', '2355172', '2388186', '2334825', '2372450', '2377971', '2377053', '2393315', '2370179', '2349895', '2335023', '2366720', '2369272', '2344127', '2393434', '2339219', '2348398', '2413035', '2330639', '2316575', '2343155', '2353872', '2414882', '2373063', '2333717', '2350248', '2383119', '2404930', '2335333', '2380117', '2400689', '2377131', '2353667', '2406595', '2343214', '2410358', '2330044', '2377161', '2347777', '2396968', '2343948', '2408400', '2342794', '2388737', '2401886', '2341672', '2385815', '2407806', '2317356', '2347763', '2341717', '2357770', '2378698', '2392376', '2373225', '1979', '2387187', '2341245', '2399182', '2410305', '2350362', '2354852', '2357069', '2391111', '2396639', '2343276', '2385145', '2319237', '2398879', '2378517', '2320078', '2357038', '2378426', '2409995', '2396407', '2344540', '2403306', '2412032', '2364541', '2379607', '2363110', '2385571', '2367180', '2317743', '2345625', '2324168', '2382524', '2389666', '2342125', '2326672', '2358834', '2416678', '2412679', '2373510', '2409161', '2330294', '2385025', '2382543', '2407293', '2405007', '2319434', '2409065', '2374686', '2415570', '2370645', '2343220', '2385158', '2366216', '2411891', '2396279', '2378386', '2406971', '2364907', '2383068', '2346025', '2401071', '2394536', '2346609', '2366521', '2336144', '2369882', '2362607', '2352327', '2343623', '2348208', '2357580', '2393529', '2347586', '2410663', '2338501', '2414226', '2317379', '2384614', '2368599', '2339675', '2392812', '2404620', '2362539', '2329100', '2343589', '2385552', '2412438', '2415173', '2376132', '2392014', '2346010', '285802', '2366148', '2416104', '2369615', '2367875', '2398860', '2318174', '2372154', '2327125', '2343802', '2364976', '2391397', '2412087', '2382659', '2325536', '2343595', '2397030', '2390728', '2409300', '2375570', '2363267', '2379412', '2318100', '2381952', '2410088', '2390131', '2350025', '2357873', '2332037', '2407133', '2387000', '2412304', '2334742', '2360125', '2351557', '2322620', '2364598', '2409948', '2407622', '2409730', '2395914', '2344000', '2367747', '2355445', '2336558', '2326536', '2336255', '2341700', '2387514', '2341671', '2398110', '2399652', '1981', '2323460', '2390548', '2378480', '2359190', '2388960', '2374610', '2390517', '2374041', '2416167', '2359860', '2345744', '2390516', '2350734', '2320101', '2362840', '2355186', '2374316', '2394482', '2345532', '2403827', '2350800', '2384225', '2408041', '2372039', '2362947', '2396741', '2328602', '2403866', '2408144', '2339689', '2373049', '2396987', '2399359', '2356053', '2371761', '2374295', '2400094', '2368870', '358', '2316710', '2348163', '2413458', '2315631', '2365595', '2409738', '2388139', '2367957', '2335492', '2350434', '2337232', '2317038', '2399813', '2364830', '2358273', '2317457', '2343118', '2325817', '2316887', '285905', '2357254', '1592902', '2387245', '2374212', '2393912', '2412430', '2396351', '2364742', '2326373', '2354526', '2410548', '2374003', '2339208', '2412323', '2347068', '2410762', '2354059', '2357064', '2394457', '2364582', '2388352', '2412817', '2345982', '2331553', '2344607', '2401917', '2335275', '2401129', '2410083', '2411500', '2398309', '2385216', '2381550', '2341363', '2415473', '2402262', '2391262', '2368478', '2344295', '2340547', '2326050', '2339004', '2346626', '2316675', '2372940', '2403736', '2361538', '2335133', '2372110', '2355888', '2408983', '2318897', '2349203', '2402756', '2385127', '1947', '2414540', '2360127', '2403553', '2361204', '2382861', '2340485', '2318856', '2349149', '2357096', '2378937', '2330143', '2390945', '2347477', '2367727', '2382133', '2365803', '2385464', '2352560', '2389937', '2335666', '2359041', '2371361', '2402586', '2400198', '2367081', '2330999', '2407240', '2373013', '2333510', '2341668', '2398193', '2384410', '2394250', '2401035', '2382813', '2369897', '2413842', '2335959', '2381711', '2394115', '2394823', '2342230', '2395930', '2375161', '2371998', '2357321', '2381052', '2386678', '2359589', '2408377', '2319233', '2377371', '2356978', '2322528', '2342127', '2351858', '2394808', '2372836', '2350140', '2330793', '2365464', '2324955', '59', '2401816', '2363606', '2411900', '2387088', '2400434', '2408594', '2379456', '2366127', '2322573', '2365692', '2354909', '2381999', '2400554', '2377456', '2338012', '2400676', '2409993', '2409525', '2329113', '2353014', '2375625', '2413783', '2405811', '2366795', '2374646', '2388879', '2317337', '2327958', '2378809', '2372018', '2397405', '2396019', '2358287', '2393934', '2411311', '2341541', '2326852', '2372176', '2371733', '2352132', '2365664', '2316733', '2365437', '1592372', '2358414', '2409530', '2410977', '2330167', '2370025', '2340772', '2358472', '2341383', '2415196', '2367052', '2349022', '2380268', '2394057', '2331963', '2348762', '2393268', '2382522', '2354169', '2346439', '2360521', '2413923', '2322133', '2318491', '2349530', '2388772', '2362719', '2406548', '2360201', '2351834', '2341514', '2400161', '2394967', '2316555', '2386847', '2412609', '2411640', '2376384', '2349745', '2358575', '2362360', '2319185', '2318300', '2319338', '2341703', '2348223', '2411941', '2404878', '2388166', '2348265', '2393093', '2386134', '2317860', '2358790', '2392630', '2381546', '2410233', '2396340', '2413134', '2384897', '2372279', '2412901', '2392163', '2399340', '2362560', '2415464', '2355350', '2385072', '2387946', '2414959', '2335223', '2396029', '2409922', '2369585', '2340615', '2342329', '2357014', '2359654', '2394493', '2339690', '2367978', '2385255', '2409589', '2404541', '2411332', '2395373', '2340926', '2379898', '2366907', '2392500', '2381777', '2371763', '2323566', '2362679', '2390775', '2326661', '2400330', '2354989', '2359414', '2351277', '2345482', '2347852', '2361424', '2363496', '2374097', '2392960', '2363592', '2338512', '2350783', '2390282', '2361585', '2346556', '2393655', '2381013', '2380114', '2413540', '2410181', '2366603', '2405292', '2352992', '2343612', '2395465', '2395628', '2386850', '2380640', '2368472', '2320897', '2404090', '2344453', '2320614', '2375431', '2354513', '2352072', '2368501', '2382276', '2327911', '2397716', '2354631', '2413575', '2327559', '2389618', '2370697', '2375622', '2403565', '2360115', '2399996', '2402523', '3668', '2342671', '2393354', '2358421', '2403502', '2358670', '2389588', '2401096', '2356526', '2380095', '2346818', '2371255', '2391668', '2385831', '2394596', '2342960', '2374375', '1160254', '2375771', '2382933', '2392920', '2387989', '2324588', '2356074', '2317625', '2344661', '2326911', '2404259', '2361721', '2393279', '2386495', '2400046', '2365043', '2344637', '2357554', '2390759', '2318426', '2386258', '2382545', '2336325', '2392061', '2405822', '2401033', '2353349', '2412624', '2340180', '2397756', '2402816', '2367884', '2323076', '2328094', '2402273', '2369292', '2414256', '2361129', '2379258', '2342678', '2383042', '2375286', '543', '2369707', '2415363', '2351567', '2374164', '2318354', '2337138', '2416386', '2344622', '2408617', '2402485', '2390061', '2403911', '2385461', '2368073', '2325644', '2413346', '2333242', '2409746', '2318784', '2356563', '2350520', '2379974', '2393488', '2401112', '2340853', '2355566', '2338820', '2388446', '2408244', '2403749', '2339735', '2345624', '2381108', '2396179', '2382666', '2377454', '2376391', '2347274', '2384475', '2317896', '2398386', '2343705', '2366203', '2336091', '2318766', '2362130', '2364869', '2388733', '2364463', '2320207', '2399556', '2318740', '2319024', '2320357', '2349108', '2324753', '2357052', '2405788', '2367095', '2355641', '2361523', '2348566', '2369937', '2364778', '2410070', '2349065', '2364313', '2385956', '2330315', '2339046', '2347982', '2389147', '2411813', '2366650', '2316025', '2340085', '2349725', '2402374', '2389782', '2318529', '2353047', '2355690', '2407520', '2340975', '2387531', '2355380', '2361208', '2405966', '2350585', '2334735', '2415776', '2391209', '2364493', '2404485', '2336765', '2390589', '2407074', '2355399', '2396682', '2359934', '2390911', '2352126', '2367144', '2393988', '2319437', '2384790', '2332609', '2363103', '2363314', '2371312', '2402053', '2415913', '2387492', '2355933', '2408558', '2368969', '2341785', '2391385', '2366303', '2338004', '2331437', '2405390', '2318074', '2382213', '2352218', '2356795', '2366679', '2373340', '2410425', '2347072', '2315529', '2414805', '2334296', '2400705', '2348349', '2371338', '2355887', '2336426', '2409509', '2400488', '2347040', '2373504', '2400709', '2388269', '2374858', '2357848', '2351724', '2366552', '2373343', '2366798', '2370562', '2336279', '2390109', '2376359', '2403968', '2377951', '2383724', '2327224', '2345537', '2320748', '2352059', '2356407', '2379040', '2404395', '2335228', '2377815', '2346974', '2398486', '2385876', '2368358', '2328180', '2333993', '2412028', '2320127', '2361275', '2325956', '2344077', '2373936', '2400518', '2410757', '2388557', '2374409', '2346288', '2327758', '2360829', '2367594', '2351200', '2401206', '2347475', '2361164', '2347157', '2322190', '2387508', '2328285', '2332131', '2399529', '2406648', '2414784', '2390009', '2385308', '2411532', '2320131', '2366219', '2385693', '2367729', '2383497', '2399749', '2387687', '2318508', '2366431', '2346215', '2332290', '2396197', '2343376', '2355111', '2315963', '2360432', '2329225', '2414189', '2407349', '2354529', '2378366', '2360200', '2386619', '2358729', '2396069', '2414523', '2402102', '2316860', '2359321', '2390811', '2403746', '2392451', '2370342', '1512', '2351235', '2356083', '2352140', '2397229', '2341890', '2377512', '2411735', '2384658', '1159516', '2360315', '2332658', '2413457', '2415457', '2349652', '2359079', '2324375', '2397605', '2403889', '2389796', '2406696', '2334724', '2407120', '2336668', '2366034', '2384206', '2400680', '2332379', '2392818', '2406733', '2387944', '2402124', '2334363', '2331362', '2388172', '2382849', '2388891', '713323', '2407039', '2386334', '2394687', '2408468', '2350752', '2384387', '2412787', '2409238', '2401910', '2346934', '2342976', '2362198', '2319960', '2384406', '2360405', '2399333', '2323665', '1159963', '2379625', '2405682', '2373934', '2337314', '2346066', '2396448', '2317900', '2407413', '2328394', '2374389', '2382714', '2339245', '2316574', '2363770', '2390523', '2415871', '2390619', '2353681', '2375141', '2365887', '2392468', '2376016', '2339907', '2386673', '2339891', '2414494', '2356926', '2351882', '2355581', '2380553', '2405893', '2375873', '2379970', '2389093', '2349339', '2351669', '2376371', '2382695', '2342062', '2343739', '2403096', '2367587', '2393255', '2363034', '2411245', '2356290', '2333032', '2377054', '2381840', '2383123', '2417815', '2342463', '2346654', '2401911', '2330589', '2359948', '2348627', '2395330', '2325024', '2389270', '2366510', '2373821', '2392869', '2381761', '2415192', '2324220', '2359272', '2386989', '2372029', '2317409', '2382557', '2315640', '2366640', '2401940', '2332111', '2371625', '2398866', '2399393', '2408374', '2372030', '2322341', '2374141', '1592381', '2346658', '2391256', '2397008', '2405504', '2378739', '2384906', '2408673', '2396317', '2388519', '2365979', '2414365', '2388526', '2337365', '2348280', '2360988', '2363427', '2368396', '2405392', '2396760', '2398451', '2400579', '2357859', '2348172', '2388316', '2357856', '2323413', '2400412', '2389597', '2364522', '2413323', '2336219', '2367716', '2362819', '2396757', '2380218', '2358578', '2390032', '2409345', '2410852', '2384053', '2378686', '2400643', '2380143', '2389002', '2320951', '2354093', '2362892', '2408480', '2361074', '2368771', '2409608', '2371432', '2344878', '2318431', '2403123', '2410146', '2342919', '2391895', '2413037', '2415154', '2352917', '2336606', '2409059', '2351967', '2344341', '2402305', '2399436', '2399831', '2344880', '2366243', '2394911', '2340505', '2360192', '2355583', '2412691', '2317478', '2377654', '2393764', '2369389', '2380908', '2359734', '2395479', '2410117', '2345963', '2348675', '2402610', '2362917', '2381263', '2411367', '2350934', '2345484', '2375389', '2338460', '2392194', '2345283', '2357009', '2366433', '2400181', '2338625', '2385349', '2409624', '2325906', '2410111', '2356573', '2408231', '2374654', '2401986', '2406374', '2408091', '2388101', '2366266', '2358240', '2387174', '2364629', '2410500', '2396864', '2349287', '2341175', '2413048', '2352077', '2357255', '2364516', '2393711', '2332173', '2370652', '2379368', '2373368', '2391968', '2363840', '2356915', '2402864', '2347167', '2361726', '2363057', '195', '2389587', '2333962', '2339034', '2358787', '2413030', '2382967', '2356452', '2355462', '2398408', '2350770', '2361025', '2402846', '2341989', '2356957', '2371802', '2340693', '2365687', '2360971', '2383081', '2397831', '2382252', '2345262', '2412694', '2350816', '2356774', '2379797', '2355306', '2348088', '2392089', '2354342', '2414493', '2317122', '2411125', '2348278', '2380488', '2409861', '2414069', '2361015', '1186', '2348951', '2408872', '2358566', '2402174', '2376953', '2416072', '2399932', '2330371', '4526', '2368076', '2334644', '813', '2415207', '2406271', '262', '2400363', '2405126', '2392176', '2373460', '2413099', '2352124', '2317117', '2364952', '2345382', '2387583', '2386515', '2411830', '2344135', '2368524', '2354957', '2395289', '2345826', '2399843', '2360483', '2392613', '2346922', '2340767', '2395698', '2363261', '2341640', '2377290', '2379419', '2317513', '2405522', '2365947', '2371836', '2374929', '2348401', '2343057', '2339483', '2369764', '2393778', '2364711', '2331127', '1143', '2371418', '2403431', '2414878', '2384911', '2346679', '2333255', '2393843', '2322397', '2412964', '2341844', '2400168', '2371705', '2390876', '2336110', '587', '2364458', '2400258', '2387213', '2412725', '2408943', '2413997', '2369025', '3524', '2345255', '2387936', '2348847', '2361143', '2356669', '2351483', '2399453', '2392336', '2399058', '2387509', '2346365', '2387430', '2323101', '2393641', '2403291', '2352426', '2346311', '2334281', '2365395', '2367554', '2379698', '2356542', '2374256', '2378992', '2379893', '2395881', '1159529', '2347316', '2398565', '61515', '2382188', '2371228', '2374191', '2405763', '2373065', '2368332', '2366857', '3974', '2403142', '2403730', '2348008', '2364666', '2383971', '2323317', '2348859', '2351300', '2393837', '2360110', '2366855', '2410116', '2399588', '2404970', '2411064', '2372245', '2396669', '2387797', '2347378', '2330444', '2397123', '2367427', '2374351', '2360943', '2331431', '2338001', '2346419', '2317987', '2399954', '2415472', '2349266', '2384407', '2339402', '2395507', '2410775', '2318528', '2408638', '2358223', '2390577', '2360637', '2403309', '2389127', '2400665', '2345254', '2357568', '2368103', '2376750', '2362276', '2373822', '2318753', '2381335', '2396390', '2407563', '677', '2383588', '2364405', '2363823', '2332047', '2369980', '2411794', '2375098', '2349488', '2412833', '2389931', '2411635', '2321324', '2351789', '2330962', '2325271', '2393810', '2386491', '2351574', '2386840', '2342128', '2360241', '2389642', '2381424', '2378294', '2376247', '2387201', '2320667', '2376728', '2398703', '2369310', '2367175', '2324573', '2344722', '2341732', '2336105', '2388874', '2341737', '2322795', '2378412', '2387370', '2369286', '2318253', '2343618', '2354019', '2402340', '2392557', '2397147', '2353466', '2402479', '2393995', '2329101', '2404918', '2347211', '2345294', '2385897', '2319741', '2414099', '2403177', '2319593', '2402961', '2361554', '2356491', '2392581', '2316889', '2360287', '2371216', '2407026', '2367533', '2378060', '2389282', '2390572', '2397120', '2400392', '2316318', '2375898', '2362311', '2401191', '2405981', '2409153', '2377072', '2355213', '2392583', '2382441', '2317053', '2346718', '2350383', '2406500', '2362770', '2410681', '2374807', '2346501', '2357249', '2359027', '2396374', '2374760', '2399256', '2336180', '2355922', '2356858', '2340777', '2381824', '2377986', '2386257', '2397845', '2389726', '2411629', '2393060', '2345476', '2374490', '2393513', '2409142', '2396465', '2356592', '2398489', '2417769', '2380388', '2386965', '2388564', '2360225', '2377334', '2396670', '2354660', '2357502', '2359450', '2378490', '2362795', '2363104', '2335727', '2390392', '2343913', '2373951', '2377309', '2392460', '2354423', '2359238', '2379480', '2374611', '2384854', '2369580', '2326856', '2332039', '2411249', '2406478', '2328010', '2369213', '2387943', '2385460', '2330218', '2360556', '2341674', '2342464', '2352565', '2341047', '2401918', '2372733', '2371551', '2316697', '2376379', '2368666', '2402546', '1592156', '2388838', '2397307', '2348182', '2413071', '2404732', '2356427', '2407637', '2390263', '2405776', '2353337', '2316602', '2353270', '2408708', '2371198', '2354266', '2369855', '2380368', '2382039', '2358567', '2319011', '2370341', '2370551', '2362808', '2382330', '2408653', '2374473', '2362280', '2398037', '2375753', '2371017', '2370877', '2408209', '2333007', '2405738', '2391153', '2372453', '2358526', '2379303', '2389636', '2342857', '2415640', '2386870', '2363443', '2368634', '2413904', '2383358', '2380024', '2416559', '2336529', '2366500', '2351655', '2345044', '2373588', '2390099', '2373407', '2350763', '2402155', '2363548', '2322813', '2343805', '2391461', '2416656', '2402786', '2394339', '2414255', '2338443', '2382381', '2374927', '2362555', '2402202', '2382046', '2412163', '2354362', '2415082', '2408489', '2406617', '2319554', '2327849', '2378204', '2391047', '2341611', '2342864', '2386983', '2347702', '2322974', '2325414', '2400737', '2404002', '2392261', '2356269', '2358999', '2399577', '2370776', '2316535', '2395996', '2336048', '2376467', '2362203', '2415820', '2383182', '2350730', '2345893', '2375076', '2417944', '2337033', '2359636', '2408359', '2416307', '2398017', '2383168', '2330468', '2405056', '2321253', '2341034', '2317165', '2401897', '2398212', '2387721', '2344595', '2379759', '2393164', '2410345', '2363437', '2386462', '2388504', '2401336', '2321387', '2317132', '2396478', '2416725', '2372569', '2359537', '2322095', '2352174', '2343766', '2380329', '2361489', '2413702', '2321676', '2404393', '2356073', '2376970', '2358293', '2342802', '2402837', '2363661', '2403903', '2319419', '2344370', '2389353', '2397657', '2371546', '2347170', '2404791', '2384368', '2417585', '2392507', '2349036', '2329559', '879', '2414977', '2407808', '2378572', '2415884', '2414803', '2402120', '2320356', '2365699', '2354877', '2389638', '2334130', '2367765', '2379154', '2399862', '2404467', '2353974', '2352853', '2339578', '2406885', '2382775', '2354318', '2353172', '2412056', '2396290', '2336760', '2378276', '2379292', '2406213', '2416937', '2395164', '2356436', '2355527', '2386443', '2397502', '2410270', '2397071', '2415676', '2371756', '2394933', '2377521', '2320584', '2394242', '2385172', '2414806', '2380874', '2371037', '2317584', '2390476', '2356914', '2361991', '2350826', '2406588', '2358311', '2409964', '2403418', '2370296', '2411605', '2323547', '2316204', '2375241', '2349448', '2354115', '2404178', '2372285', '2401764', '2377897', '2328135', '2382879', '2360242', '2350037', '2361415', '2367100', '2411866', '2378223', '2346003', '2359944', '2386442', '2327810', '2345716', '2398747', '2327729', '2379103', '2330650', '2391165', '2403131']\n",
      "IMG@IDX {'2385757': 0, '2389264': 1, '2407325': 2, '2337141': 3, '2345813': 4, '2386694': 5, '2368581': 6, '2397809': 7, '185': 8, '2387911': 9, '2343682': 10, '2409165': 11, '2357129': 12, '498337': 13, '2326621': 14, '2320205': 15, '2412782': 16, '2352466': 17, '2414097': 18, '2381526': 19, '2390981': 20, '2375715': 21, '2405096': 22, '2402848': 23, '2346948': 24, '2387882': 25, '2352243': 26, '2337581': 27, '2330919': 28, '2380444': 29, '2377611': 30, '2354637': 31, '2401996': 32, '2391375': 33, '2415510': 34, '2400079': 35, '2366081': 36, '2360527': 37, '2365490': 38, '2336229': 39, '2367475': 40, '2373691': 41, '2376356': 42, '2368952': 43, '2369653': 44, '2355012': 45, '2400892': 46, '2369971': 47, '2414408': 48, '2394007': 49, '2415978': 50, '2405444': 51, '2401822': 52, '2387651': 53, '2350622': 54, '2318603': 55, '2383962': 56, '2377677': 57, '2380510': 58, '2392746': 59, '2412229': 60, '2392926': 61, '2362383': 62, '2398991': 63, '2367805': 64, '2360264': 65, '2400991': 66, '2354133': 67, '2374619': 68, '2413514': 69, '2387844': 70, '2349764': 71, '2403572': 72, '2372008': 73, '2388111': 74, '2366871': 75, '2385053': 76, '2400907': 77, '2379868': 78, '2389754': 79, '2389615': 80, '2365263': 81, '2414350': 82, '2409171': 83, '4025': 84, '2352062': 85, '2378864': 86, '2367278': 87, '2319025': 88, '2335091': 89, '2344088': 90, '2351081': 91, '2408962': 92, '2319561': 93, '2400672': 94, '2390977': 95, '2390288': 96, '2415083': 97, '2396514': 98, '2411955': 99, '2346205': 100, '2395037': 101, '2346498': 102, '2383586': 103, '2346342': 104, '2367250': 105, '2348629': 106, '2413978': 107, '2355118': 108, '2390609': 109, '724': 110, '2347956': 111, '2389121': 112, '2381396': 113, '2394635': 114, '2374001': 115, '2319411': 116, '2356016': 117, '2390189': 118, '2339582': 119, '2362277': 120, '2325410': 121, '2354590': 122, '2367096': 123, '2317592': 124, '2384171': 125, '2397387': 126, '2376251': 127, '2352643': 128, '2354147': 129, '2410352': 130, '2332365': 131, '2323588': 132, '2504': 133, '2346067': 134, '2373700': 135, '2395463': 136, '2388627': 137, '2344272': 138, '2365691': 139, '2364664': 140, '2410717': 141, '2371950': 142, '2402941': 143, '2391252': 144, '2342646': 145, '2352971': 146, '2370165': 147, '2395630': 148, '2317591': 149, '2378059': 150, '2355709': 151, '2336832': 152, '2391722': 153, '2330764': 154, '2346660': 155, '2409862': 156, '2373155': 157, '2393166': 158, '2355642': 159, '2346977': 160, '2377832': 161, '2343586': 162, '2388431': 163, '2381416': 164, '2408088': 165, '2343537': 166, '2390500': 167, '2319274': 168, '2358132': 169, '2336159': 170, '2386365': 171, '2395072': 172, '2405965': 173, '2390610': 174, '2392765': 175, '2363501': 176, '2322189': 177, '2400936': 178, '2354380': 179, '2385725': 180, '2414772': 181, '2387049': 182, '2339305': 183, '2355163': 184, '2411856': 185, '2325076': 186, '2391875': 187, '2369075': 188, '2404587': 189, '2371926': 190, '2395640': 191, '2329779': 192, '2402384': 193, '2348939': 194, '2361855': 195, '2395696': 196, '2391663': 197, '2370517': 198, '2412037': 199, '2347202': 200, '2353100': 201, '2372426': 202, '2405703': 203, '2370496': 204, '2375612': 205, '2338355': 206, '2342324': 207, '2400462': 208, '2396112': 209, '2400862': 210, '2379832': 211, '2333467': 212, '2359089': 213, '2398544': 214, '2328805': 215, '2345621': 216, '2397165': 217, '2391502': 218, '2403224': 219, '2356812': 220, '2389832': 221, '2399569': 222, '2414157': 223, '2392065': 224, '2353938': 225, '2374300': 226, '2411459': 227, '2390394': 228, '2354220': 229, '2380380': 230, '2363108': 231, '2342406': 232, '2334854': 233, '2410297': 234, '2335993': 235, '2411121': 236, '2366378': 237, '2401969': 238, '2387542': 239, '2380071': 240, '2393134': 241, '2315498': 242, '2367637': 243, '2370780': 244, '2317055': 245, '2383295': 246, '2400542': 247, '2402050': 248, '2390890': 249, '2386260': 250, '2377837': 251, '2318159': 252, '2327364': 253, '2409644': 254, '2396452': 255, '2341431': 256, '2381222': 257, '2389464': 258, '2406927': 259, '2371744': 260, '2392787': 261, '2376996': 262, '2340394': 263, '2333581': 264, '2398578': 265, '2348334': 266, '2408546': 267, '2354873': 268, '2384670': 269, '2408339': 270, '2316661': 271, '2318214': 272, '2369502': 273, '2344294': 274, '2316382': 275, '2389172': 276, '2412334': 277, '2382722': 278, '2373092': 279, '2400913': 280, '2364094': 281, '2415823': 282, '2407666': 283, '2408407': 284, '2353425': 285, '2385066': 286, '2350985': 287, '2396301': 288, '2355676': 289, '2318134': 290, '2368058': 291, '2359892': 292, '2327449': 293, '2342078': 294, '2365122': 295, '2416996': 296, '2365457': 297, '2376076': 298, '2397271': 299, '2317228': 300, '2333227': 301, '2343001': 302, '2335613': 303, '2336076': 304, '2363469': 305, '2387136': 306, '2387602': 307, '538': 308, '2381047': 309, '2341390': 310, '2382974': 311, '2393524': 312, '2389704': 313, '2352288': 314, '2366841': 315, '2366424': 316, '2414990': 317, '2382610': 318, '2359501': 319, '2341126': 320, '2406861': 321, '2381613': 322, '2381116': 323, '2362015': 324, '2364508': 325, '2361773': 326, '2386125': 327, '2323005': 328, '2373886': 329, '2391950': 330, '2323158': 331, '2372145': 332, '2392202': 333, '2362921': 334, '150344': 335, '2415140': 336, '2375237': 337, '2397509': 338, '2405568': 339, '2410491': 340, '2365504': 341, '2372907': 342, '2359383': 343, '2384101': 344, '2330124': 345, '2341434': 346, '2403182': 347, '2393422': 348, '2350678': 349, '2403683': 350, '2334135': 351, '2379332': 352, '2382744': 353, '2400289': 354, '2351156': 355, '2354589': 356, '2366570': 357, '2393002': 358, '2375133': 359, '2372957': 360, '2346843': 361, '2376314': 362, '2357994': 363, '2389532': 364, '2355449': 365, '2387258': 366, '2386636': 367, '2381310': 368, '2331045': 369, '2362731': 370, '2353668': 371, '2390084': 372, '2347783': 373, '2383040': 374, '2341770': 375, '2367808': 376, '2397944': 377, '2389426': 378, '2391869': 379, '2351511': 380, '2406056': 381, '2347102': 382, '2414155': 383, '2396971': 384, '2404188': 385, '2352834': 386, '2397549': 387, '2335504': 388, '2390711': 389, '2408710': 390, '2318479': 391, '2345601': 392, '2323690': 393, '2393174': 394, '3665': 395, '2343876': 396, '2327045': 397, '2414418': 398, '2350160': 399, '2360570': 400, '2416836': 401, '2410413': 402, '2359369': 403, '2407102': 404, '4329': 405, '2317914': 406, '2375681': 407, '2385986': 408, '2357723': 409, '2390201': 410, '2376235': 411, '2366660': 412, '2346881': 413, '2345363': 414, '2358150': 415, '853': 416, '1067': 417, '2391849': 418, '2347742': 419, '2374809': 420, '2363649': 421, '2317707': 422, '2394449': 423, '2407504': 424, '2366958': 425, '815': 426, '2381442': 427, '2386687': 428, '2343486': 429, '2403767': 430, '2342183': 431, '2342649': 432, '2360855': 433, '2384747': 434, '2369741': 435, '2338878': 436, '2350786': 437, '2377187': 438, '2391182': 439, '2345464': 440, '2382463': 441, '2373922': 442, '2364569': 443, '2321507': 444, '2391257': 445, '2390199': 446, '2376974': 447, '2411637': 448, '2414334': 449, '2382975': 450, '2374130': 451, '2407054': 452, '2376001': 453, '2405490': 454, '2391435': 455, '2360573': 456, '2356093': 457, '2317319': 458, '2376855': 459, '2372392': 460, '2399545': 461, '2407631': 462, '2414694': 463, '2387731': 464, '2380250': 465, '2399949': 466, '2381902': 467, '2380617': 468, '2387786': 469, '2343356': 470, '2316729': 471, '2335572': 472, '2342386': 473, '2341113': 474, '2407180': 475, '2372676': 476, '2408452': 477, '2351275': 478, '2327618': 479, '2391545': 480, '2406319': 481, '2369158': 482, '2408036': 483, '2409641': 484, '2318461': 485, '2376879': 486, '2382464': 487, '2358660': 488, '2397224': 489, '2360963': 490, '2356329': 491, '2397683': 492, '2398695': 493, '2318316': 494, '2369188': 495, '2383665': 496, '2351900': 497, '2355768': 498, '2341725': 499, '2335274': 500, '2410913': 501, '2404585': 502, '2383900': 503, '2370017': 504, '2406329': 505, '2375118': 506, '2389880': 507, '2333038': 508, '2345388': 509, '2404202': 510, '2345290': 511, '2412080': 512, '2336489': 513, '2379551': 514, '2352259': 515, '2398246': 516, '2322086': 517, '2412178': 518, '2389036': 519, '2411172': 520, '2377272': 521, '2369601': 522, '2399464': 523, '2351596': 524, '2403297': 525, '2353451': 526, '2394454': 527, '2401819': 528, '2398902': 529, '2394152': 530, '2385541': 531, '2369013': 532, '2385974': 533, '2344284': 534, '2365226': 535, '2322420': 536, '2358081': 537, '1160101': 538, '2316641': 539, '2371907': 540, '2384500': 541, '2344095': 542, '2409873': 543, '2342700': 544, '2368261': 545, '2349084': 546, '2373021': 547, '2339331': 548, '2401288': 549, '2334516': 550, '2344881': 551, '2364452': 552, '2349469': 553, '2374709': 554, '2406451': 555, '2391194': 556, '2394298': 557, '2406376': 558, '2371567': 559, '2335457': 560, '2407029': 561, '2372220': 562, '2357948': 563, '2384021': 564, '2388187': 565, '2331430': 566, '2367019': 567, '2392600': 568, '2370343': 569, '2417500': 570, '2346802': 571, '2564': 572, '2360148': 573, '2407311': 574, '2350000': 575, '2374917': 576, '2354814': 577, '2352832': 578, '2383787': 579, '2387276': 580, '2413861': 581, '2336315': 582, '2357005': 583, '2356221': 584, '2400369': 585, '2349353': 586, '2366432': 587, '2405436': 588, '2407604': 589, '2318841': 590, '2394677': 591, '2384439': 592, '2348689': 593, '2413793': 594, '2384089': 595, '2373382': 596, '2402051': 597, '2330715': 598, '2403512': 599, '2379975': 600, '2411960': 601, '2363971': 602, '2353696': 603, '2384283': 604, '2395949': 605, '2352715': 606, '2333164': 607, '2364799': 608, '2399390': 609, '2341305': 610, '2410276': 611, '2319462': 612, '2333402': 613, '2405554': 614, '2344657': 615, '2328347': 616, '2415027': 617, '2356488': 618, '2367352': 619, '2400108': 620, '2380279': 621, '2396396': 622, '2339028': 623, '2363834': 624, '2366302': 625, '2361964': 626, '2343596': 627, '2382173': 628, '2371493': 629, '2362648': 630, '2381158': 631, '2392123': 632, '2371441': 633, '2393319': 634, '2353865': 635, '2412966': 636, '2402825': 637, '2356671': 638, '2385174': 639, '2375529': 640, '2354544': 641, '2323848': 642, '2382977': 643, '2323209': 644, '2387933': 645, '2341028': 646, '2411576': 647, '2356811': 648, '2387207': 649, '2338341': 650, '2381847': 651, '2396662': 652, '2367762': 653, '2391902': 654, '2350959': 655, '2393496': 656, '2366615': 657, '2344117': 658, '2333815': 659, '2352342': 660, '2387761': 661, '2346469': 662, '2382632': 663, '2337106': 664, '2357507': 665, '2368903': 666, '2400371': 667, '2377573': 668, '2399802': 669, '2374100': 670, '2387464': 671, '2410958': 672, '2316254': 673, '2355876': 674, '2319466': 675, '2387553': 676, '2387137': 677, '2401818': 678, '2413411': 679, '2408481': 680, '2320306': 681, '2343554': 682, '2383435': 683, '2341747': 684, '2411945': 685, '2395545': 686, '2347126': 687, '2383269': 688, '2341187': 689, '2395566': 690, '2361388': 691, '2326048': 692, '2326483': 693, '2342090': 694, '2330104': 695, '2388858': 696, '2414131': 697, '2328024': 698, '2405963': 699, '2321198': 700, '2342281': 701, '2408454': 702, '2407660': 703, '2350676': 704, '2324748': 705, '2364648': 706, '2399525': 707, '2367980': 708, '2369455': 709, '2388006': 710, '2406791': 711, '2338235': 712, '2399119': 713, '2367983': 714, '2398222': 715, '2376293': 716, '2379307': 717, '2318313': 718, '2395464': 719, '2355425': 720, '2344467': 721, '2397941': 722, '2412770': 723, '2378990': 724, '2391984': 725, '2327406': 726, '2376475': 727, '2382690': 728, '2414448': 729, '2322107': 730, '2414173': 731, '2377368': 732, '2398978': 733, '2334930': 734, '2354564': 735, '2394297': 736, '2355839': 737, '2355172': 738, '2388186': 739, '2334825': 740, '2372450': 741, '2377971': 742, '2377053': 743, '2393315': 744, '2370179': 745, '2349895': 746, '2335023': 747, '2366720': 748, '2369272': 749, '2344127': 750, '2393434': 751, '2339219': 752, '2348398': 753, '2413035': 754, '2330639': 755, '2316575': 756, '2343155': 757, '2353872': 758, '2414882': 759, '2373063': 760, '2333717': 761, '2350248': 762, '2383119': 763, '2404930': 764, '2335333': 765, '2380117': 766, '2400689': 767, '2377131': 768, '2353667': 769, '2406595': 770, '2343214': 771, '2410358': 772, '2330044': 773, '2377161': 774, '2347777': 775, '2396968': 776, '2343948': 777, '2408400': 778, '2342794': 779, '2388737': 780, '2401886': 781, '2341672': 782, '2385815': 783, '2407806': 784, '2317356': 785, '2347763': 786, '2341717': 787, '2357770': 788, '2378698': 789, '2392376': 790, '2373225': 791, '1979': 792, '2387187': 793, '2341245': 794, '2399182': 795, '2410305': 796, '2350362': 797, '2354852': 798, '2357069': 799, '2391111': 800, '2396639': 801, '2343276': 802, '2385145': 803, '2319237': 804, '2398879': 805, '2378517': 806, '2320078': 807, '2357038': 808, '2378426': 809, '2409995': 810, '2396407': 811, '2344540': 812, '2403306': 813, '2412032': 814, '2364541': 815, '2379607': 816, '2363110': 817, '2385571': 818, '2367180': 819, '2317743': 820, '2345625': 821, '2324168': 822, '2382524': 823, '2389666': 824, '2342125': 825, '2326672': 826, '2358834': 827, '2416678': 828, '2412679': 829, '2373510': 830, '2409161': 831, '2330294': 832, '2385025': 833, '2382543': 834, '2407293': 835, '2405007': 836, '2319434': 837, '2409065': 838, '2374686': 839, '2415570': 840, '2370645': 841, '2343220': 842, '2385158': 843, '2366216': 844, '2411891': 845, '2396279': 846, '2378386': 847, '2406971': 848, '2364907': 849, '2383068': 850, '2346025': 851, '2401071': 852, '2394536': 853, '2346609': 854, '2366521': 855, '2336144': 856, '2369882': 857, '2362607': 858, '2352327': 859, '2343623': 860, '2348208': 861, '2357580': 862, '2393529': 863, '2347586': 864, '2410663': 865, '2338501': 866, '2414226': 867, '2317379': 868, '2384614': 869, '2368599': 870, '2339675': 871, '2392812': 872, '2404620': 873, '2362539': 874, '2329100': 875, '2343589': 876, '2385552': 877, '2412438': 878, '2415173': 879, '2376132': 880, '2392014': 881, '2346010': 882, '285802': 883, '2366148': 884, '2416104': 885, '2369615': 886, '2367875': 887, '2398860': 888, '2318174': 889, '2372154': 890, '2327125': 891, '2343802': 892, '2364976': 893, '2391397': 894, '2412087': 895, '2382659': 896, '2325536': 897, '2343595': 898, '2397030': 899, '2390728': 900, '2409300': 901, '2375570': 902, '2363267': 903, '2379412': 904, '2318100': 905, '2381952': 906, '2410088': 907, '2390131': 908, '2350025': 909, '2357873': 910, '2332037': 911, '2407133': 912, '2387000': 913, '2412304': 914, '2334742': 915, '2360125': 916, '2351557': 917, '2322620': 918, '2364598': 919, '2409948': 920, '2407622': 921, '2409730': 922, '2395914': 923, '2344000': 924, '2367747': 925, '2355445': 926, '2336558': 927, '2326536': 928, '2336255': 929, '2341700': 930, '2387514': 931, '2341671': 932, '2398110': 933, '2399652': 934, '1981': 935, '2323460': 936, '2390548': 937, '2378480': 938, '2359190': 939, '2388960': 940, '2374610': 941, '2390517': 942, '2374041': 943, '2416167': 944, '2359860': 945, '2345744': 946, '2390516': 947, '2350734': 948, '2320101': 949, '2362840': 950, '2355186': 951, '2374316': 952, '2394482': 953, '2345532': 954, '2403827': 955, '2350800': 956, '2384225': 957, '2408041': 958, '2372039': 959, '2362947': 960, '2396741': 961, '2328602': 962, '2403866': 963, '2408144': 964, '2339689': 965, '2373049': 966, '2396987': 967, '2399359': 968, '2356053': 969, '2371761': 970, '2374295': 971, '2400094': 972, '2368870': 973, '358': 974, '2316710': 975, '2348163': 976, '2413458': 977, '2315631': 978, '2365595': 979, '2409738': 980, '2388139': 981, '2367957': 982, '2335492': 983, '2350434': 984, '2337232': 985, '2317038': 986, '2399813': 987, '2364830': 988, '2358273': 989, '2317457': 990, '2343118': 991, '2325817': 992, '2316887': 993, '285905': 994, '2357254': 995, '1592902': 996, '2387245': 997, '2374212': 998, '2393912': 999, '2412430': 1000, '2396351': 1001, '2364742': 1002, '2326373': 1003, '2354526': 1004, '2410548': 1005, '2374003': 1006, '2339208': 1007, '2412323': 1008, '2347068': 1009, '2410762': 1010, '2354059': 1011, '2357064': 1012, '2394457': 1013, '2364582': 1014, '2388352': 1015, '2412817': 1016, '2345982': 1017, '2331553': 1018, '2344607': 1019, '2401917': 1020, '2335275': 1021, '2401129': 1022, '2410083': 1023, '2411500': 1024, '2398309': 1025, '2385216': 1026, '2381550': 1027, '2341363': 1028, '2415473': 1029, '2402262': 1030, '2391262': 1031, '2368478': 1032, '2344295': 1033, '2340547': 1034, '2326050': 1035, '2339004': 1036, '2346626': 1037, '2316675': 1038, '2372940': 1039, '2403736': 1040, '2361538': 1041, '2335133': 1042, '2372110': 1043, '2355888': 1044, '2408983': 1045, '2318897': 1046, '2349203': 1047, '2402756': 1048, '2385127': 1049, '1947': 1050, '2414540': 1051, '2360127': 1052, '2403553': 1053, '2361204': 1054, '2382861': 1055, '2340485': 1056, '2318856': 1057, '2349149': 1058, '2357096': 1059, '2378937': 1060, '2330143': 1061, '2390945': 1062, '2347477': 1063, '2367727': 1064, '2382133': 1065, '2365803': 1066, '2385464': 1067, '2352560': 1068, '2389937': 1069, '2335666': 1070, '2359041': 1071, '2371361': 1072, '2402586': 1073, '2400198': 1074, '2367081': 1075, '2330999': 1076, '2407240': 1077, '2373013': 1078, '2333510': 1079, '2341668': 1080, '2398193': 1081, '2384410': 1082, '2394250': 1083, '2401035': 1084, '2382813': 1085, '2369897': 1086, '2413842': 1087, '2335959': 1088, '2381711': 1089, '2394115': 1090, '2394823': 1091, '2342230': 1092, '2395930': 1093, '2375161': 1094, '2371998': 1095, '2357321': 1096, '2381052': 1097, '2386678': 1098, '2359589': 1099, '2408377': 1100, '2319233': 1101, '2377371': 1102, '2356978': 1103, '2322528': 1104, '2342127': 1105, '2351858': 1106, '2394808': 1107, '2372836': 1108, '2350140': 1109, '2330793': 1110, '2365464': 1111, '2324955': 1112, '59': 1113, '2401816': 1114, '2363606': 1115, '2411900': 1116, '2387088': 1117, '2400434': 1118, '2408594': 1119, '2379456': 1120, '2366127': 1121, '2322573': 1122, '2365692': 1123, '2354909': 1124, '2381999': 1125, '2400554': 1126, '2377456': 1127, '2338012': 1128, '2400676': 1129, '2409993': 1130, '2409525': 1131, '2329113': 1132, '2353014': 1133, '2375625': 1134, '2413783': 1135, '2405811': 1136, '2366795': 1137, '2374646': 1138, '2388879': 1139, '2317337': 1140, '2327958': 1141, '2378809': 1142, '2372018': 1143, '2397405': 1144, '2396019': 1145, '2358287': 1146, '2393934': 1147, '2411311': 1148, '2341541': 1149, '2326852': 1150, '2372176': 1151, '2371733': 1152, '2352132': 1153, '2365664': 1154, '2316733': 1155, '2365437': 1156, '1592372': 1157, '2358414': 1158, '2409530': 1159, '2410977': 1160, '2330167': 1161, '2370025': 1162, '2340772': 1163, '2358472': 1164, '2341383': 1165, '2415196': 1166, '2367052': 1167, '2349022': 1168, '2380268': 1169, '2394057': 1170, '2331963': 1171, '2348762': 1172, '2393268': 1173, '2382522': 1174, '2354169': 1175, '2346439': 1176, '2360521': 1177, '2413923': 1178, '2322133': 1179, '2318491': 1180, '2349530': 1181, '2388772': 1182, '2362719': 1183, '2406548': 1184, '2360201': 1185, '2351834': 1186, '2341514': 1187, '2400161': 1188, '2394967': 1189, '2316555': 1190, '2386847': 1191, '2412609': 1192, '2411640': 1193, '2376384': 1194, '2349745': 1195, '2358575': 1196, '2362360': 1197, '2319185': 1198, '2318300': 1199, '2319338': 1200, '2341703': 1201, '2348223': 1202, '2411941': 1203, '2404878': 1204, '2388166': 1205, '2348265': 1206, '2393093': 1207, '2386134': 1208, '2317860': 1209, '2358790': 1210, '2392630': 1211, '2381546': 1212, '2410233': 1213, '2396340': 1214, '2413134': 1215, '2384897': 1216, '2372279': 1217, '2412901': 1218, '2392163': 1219, '2399340': 1220, '2362560': 1221, '2415464': 1222, '2355350': 1223, '2385072': 1224, '2387946': 1225, '2414959': 1226, '2335223': 1227, '2396029': 1228, '2409922': 1229, '2369585': 1230, '2340615': 1231, '2342329': 1232, '2357014': 1233, '2359654': 1234, '2394493': 1235, '2339690': 1236, '2367978': 1237, '2385255': 1238, '2409589': 1239, '2404541': 1240, '2411332': 1241, '2395373': 1242, '2340926': 1243, '2379898': 1244, '2366907': 1245, '2392500': 1246, '2381777': 1247, '2371763': 1248, '2323566': 1249, '2362679': 1250, '2390775': 1251, '2326661': 1252, '2400330': 1253, '2354989': 1254, '2359414': 1255, '2351277': 1256, '2345482': 1257, '2347852': 1258, '2361424': 1259, '2363496': 1260, '2374097': 1261, '2392960': 1262, '2363592': 1263, '2338512': 1264, '2350783': 1265, '2390282': 1266, '2361585': 1267, '2346556': 1268, '2393655': 1269, '2381013': 1270, '2380114': 1271, '2413540': 1272, '2410181': 1273, '2366603': 1274, '2405292': 1275, '2352992': 1276, '2343612': 1277, '2395465': 1278, '2395628': 1279, '2386850': 1280, '2380640': 1281, '2368472': 1282, '2320897': 1283, '2404090': 1284, '2344453': 1285, '2320614': 1286, '2375431': 1287, '2354513': 1288, '2352072': 1289, '2368501': 1290, '2382276': 1291, '2327911': 1292, '2397716': 1293, '2354631': 1294, '2413575': 1295, '2327559': 1296, '2389618': 1297, '2370697': 1298, '2375622': 1299, '2403565': 1300, '2360115': 1301, '2399996': 1302, '2402523': 1303, '3668': 1304, '2342671': 1305, '2393354': 1306, '2358421': 1307, '2403502': 1308, '2358670': 1309, '2389588': 1310, '2401096': 1311, '2356526': 1312, '2380095': 1313, '2346818': 1314, '2371255': 1315, '2391668': 1316, '2385831': 1317, '2394596': 1318, '2342960': 1319, '2374375': 1320, '1160254': 1321, '2375771': 1322, '2382933': 1323, '2392920': 1324, '2387989': 1325, '2324588': 1326, '2356074': 1327, '2317625': 1328, '2344661': 1329, '2326911': 1330, '2404259': 1331, '2361721': 1332, '2393279': 1333, '2386495': 1334, '2400046': 1335, '2365043': 1336, '2344637': 1337, '2357554': 1338, '2390759': 1339, '2318426': 1340, '2386258': 1341, '2382545': 1342, '2336325': 1343, '2392061': 1344, '2405822': 1345, '2401033': 1346, '2353349': 1347, '2412624': 1348, '2340180': 1349, '2397756': 1350, '2402816': 1351, '2367884': 1352, '2323076': 1353, '2328094': 1354, '2402273': 1355, '2369292': 1356, '2414256': 1357, '2361129': 1358, '2379258': 1359, '2342678': 1360, '2383042': 1361, '2375286': 1362, '543': 1363, '2369707': 1364, '2415363': 1365, '2351567': 1366, '2374164': 1367, '2318354': 1368, '2337138': 1369, '2416386': 1370, '2344622': 1371, '2408617': 1372, '2402485': 1373, '2390061': 1374, '2403911': 1375, '2385461': 1376, '2368073': 1377, '2325644': 1378, '2413346': 1379, '2333242': 1380, '2409746': 1381, '2318784': 1382, '2356563': 1383, '2350520': 1384, '2379974': 1385, '2393488': 1386, '2401112': 1387, '2340853': 1388, '2355566': 1389, '2338820': 1390, '2388446': 1391, '2408244': 1392, '2403749': 1393, '2339735': 1394, '2345624': 1395, '2381108': 1396, '2396179': 1397, '2382666': 1398, '2377454': 1399, '2376391': 1400, '2347274': 1401, '2384475': 1402, '2317896': 1403, '2398386': 1404, '2343705': 1405, '2366203': 1406, '2336091': 1407, '2318766': 1408, '2362130': 1409, '2364869': 1410, '2388733': 1411, '2364463': 1412, '2320207': 1413, '2399556': 1414, '2318740': 1415, '2319024': 1416, '2320357': 1417, '2349108': 1418, '2324753': 1419, '2357052': 1420, '2405788': 1421, '2367095': 1422, '2355641': 1423, '2361523': 1424, '2348566': 1425, '2369937': 1426, '2364778': 1427, '2410070': 1428, '2349065': 1429, '2364313': 1430, '2385956': 1431, '2330315': 1432, '2339046': 1433, '2347982': 1434, '2389147': 1435, '2411813': 1436, '2366650': 1437, '2316025': 1438, '2340085': 1439, '2349725': 1440, '2402374': 1441, '2389782': 1442, '2318529': 1443, '2353047': 1444, '2355690': 1445, '2407520': 1446, '2340975': 1447, '2387531': 1448, '2355380': 1449, '2361208': 1450, '2405966': 1451, '2350585': 1452, '2334735': 1453, '2415776': 1454, '2391209': 1455, '2364493': 1456, '2404485': 1457, '2336765': 1458, '2390589': 1459, '2407074': 1460, '2355399': 1461, '2396682': 1462, '2359934': 1463, '2390911': 1464, '2352126': 1465, '2367144': 1466, '2393988': 1467, '2319437': 1468, '2384790': 1469, '2332609': 1470, '2363103': 1471, '2363314': 1472, '2371312': 1473, '2402053': 1474, '2415913': 1475, '2387492': 1476, '2355933': 1477, '2408558': 1478, '2368969': 1479, '2341785': 1480, '2391385': 1481, '2366303': 1482, '2338004': 1483, '2331437': 1484, '2405390': 1485, '2318074': 1486, '2382213': 1487, '2352218': 1488, '2356795': 1489, '2366679': 1490, '2373340': 1491, '2410425': 1492, '2347072': 1493, '2315529': 1494, '2414805': 1495, '2334296': 1496, '2400705': 1497, '2348349': 1498, '2371338': 1499, '2355887': 1500, '2336426': 1501, '2409509': 1502, '2400488': 1503, '2347040': 1504, '2373504': 1505, '2400709': 1506, '2388269': 1507, '2374858': 1508, '2357848': 1509, '2351724': 1510, '2366552': 1511, '2373343': 1512, '2366798': 1513, '2370562': 1514, '2336279': 1515, '2390109': 1516, '2376359': 1517, '2403968': 1518, '2377951': 1519, '2383724': 1520, '2327224': 1521, '2345537': 1522, '2320748': 1523, '2352059': 1524, '2356407': 1525, '2379040': 1526, '2404395': 1527, '2335228': 1528, '2377815': 1529, '2346974': 1530, '2398486': 1531, '2385876': 1532, '2368358': 1533, '2328180': 1534, '2333993': 1535, '2412028': 1536, '2320127': 1537, '2361275': 1538, '2325956': 1539, '2344077': 1540, '2373936': 1541, '2400518': 1542, '2410757': 1543, '2388557': 1544, '2374409': 1545, '2346288': 1546, '2327758': 1547, '2360829': 1548, '2367594': 1549, '2351200': 1550, '2401206': 1551, '2347475': 1552, '2361164': 1553, '2347157': 1554, '2322190': 1555, '2387508': 1556, '2328285': 1557, '2332131': 1558, '2399529': 1559, '2406648': 1560, '2414784': 1561, '2390009': 1562, '2385308': 1563, '2411532': 1564, '2320131': 1565, '2366219': 1566, '2385693': 1567, '2367729': 1568, '2383497': 1569, '2399749': 1570, '2387687': 1571, '2318508': 1572, '2366431': 1573, '2346215': 1574, '2332290': 1575, '2396197': 1576, '2343376': 1577, '2355111': 1578, '2315963': 1579, '2360432': 1580, '2329225': 1581, '2414189': 1582, '2407349': 1583, '2354529': 1584, '2378366': 1585, '2360200': 1586, '2386619': 1587, '2358729': 1588, '2396069': 1589, '2414523': 1590, '2402102': 1591, '2316860': 1592, '2359321': 1593, '2390811': 1594, '2403746': 1595, '2392451': 1596, '2370342': 1597, '1512': 1598, '2351235': 1599, '2356083': 1600, '2352140': 1601, '2397229': 1602, '2341890': 1603, '2377512': 1604, '2411735': 1605, '2384658': 1606, '1159516': 1607, '2360315': 1608, '2332658': 1609, '2413457': 1610, '2415457': 1611, '2349652': 1612, '2359079': 1613, '2324375': 1614, '2397605': 1615, '2403889': 1616, '2389796': 1617, '2406696': 1618, '2334724': 1619, '2407120': 1620, '2336668': 1621, '2366034': 1622, '2384206': 1623, '2400680': 1624, '2332379': 1625, '2392818': 1626, '2406733': 1627, '2387944': 1628, '2402124': 1629, '2334363': 1630, '2331362': 1631, '2388172': 1632, '2382849': 1633, '2388891': 1634, '713323': 1635, '2407039': 1636, '2386334': 1637, '2394687': 1638, '2408468': 1639, '2350752': 1640, '2384387': 1641, '2412787': 1642, '2409238': 1643, '2401910': 1644, '2346934': 1645, '2342976': 1646, '2362198': 1647, '2319960': 1648, '2384406': 1649, '2360405': 1650, '2399333': 1651, '2323665': 1652, '1159963': 1653, '2379625': 1654, '2405682': 1655, '2373934': 1656, '2337314': 1657, '2346066': 1658, '2396448': 1659, '2317900': 1660, '2407413': 1661, '2328394': 1662, '2374389': 1663, '2382714': 1664, '2339245': 1665, '2316574': 1666, '2363770': 1667, '2390523': 1668, '2415871': 1669, '2390619': 1670, '2353681': 1671, '2375141': 1672, '2365887': 1673, '2392468': 1674, '2376016': 1675, '2339907': 1676, '2386673': 1677, '2339891': 1678, '2414494': 1679, '2356926': 1680, '2351882': 1681, '2355581': 1682, '2380553': 1683, '2405893': 1684, '2375873': 1685, '2379970': 1686, '2389093': 1687, '2349339': 1688, '2351669': 1689, '2376371': 1690, '2382695': 1691, '2342062': 1692, '2343739': 1693, '2403096': 1694, '2367587': 1695, '2393255': 1696, '2363034': 1697, '2411245': 1698, '2356290': 1699, '2333032': 1700, '2377054': 1701, '2381840': 1702, '2383123': 1703, '2417815': 1704, '2342463': 1705, '2346654': 1706, '2401911': 1707, '2330589': 1708, '2359948': 1709, '2348627': 1710, '2395330': 1711, '2325024': 1712, '2389270': 1713, '2366510': 1714, '2373821': 1715, '2392869': 1716, '2381761': 1717, '2415192': 1718, '2324220': 1719, '2359272': 1720, '2386989': 1721, '2372029': 1722, '2317409': 1723, '2382557': 1724, '2315640': 1725, '2366640': 1726, '2401940': 1727, '2332111': 1728, '2371625': 1729, '2398866': 1730, '2399393': 1731, '2408374': 1732, '2372030': 1733, '2322341': 1734, '2374141': 1735, '1592381': 1736, '2346658': 1737, '2391256': 1738, '2397008': 1739, '2405504': 1740, '2378739': 1741, '2384906': 1742, '2408673': 1743, '2396317': 1744, '2388519': 1745, '2365979': 1746, '2414365': 1747, '2388526': 1748, '2337365': 1749, '2348280': 1750, '2360988': 1751, '2363427': 1752, '2368396': 1753, '2405392': 1754, '2396760': 1755, '2398451': 1756, '2400579': 1757, '2357859': 1758, '2348172': 1759, '2388316': 1760, '2357856': 1761, '2323413': 1762, '2400412': 1763, '2389597': 1764, '2364522': 1765, '2413323': 1766, '2336219': 1767, '2367716': 1768, '2362819': 1769, '2396757': 1770, '2380218': 1771, '2358578': 1772, '2390032': 1773, '2409345': 1774, '2410852': 1775, '2384053': 1776, '2378686': 1777, '2400643': 1778, '2380143': 1779, '2389002': 1780, '2320951': 1781, '2354093': 1782, '2362892': 1783, '2408480': 1784, '2361074': 1785, '2368771': 1786, '2409608': 1787, '2371432': 1788, '2344878': 1789, '2318431': 1790, '2403123': 1791, '2410146': 1792, '2342919': 1793, '2391895': 1794, '2413037': 1795, '2415154': 1796, '2352917': 1797, '2336606': 1798, '2409059': 1799, '2351967': 1800, '2344341': 1801, '2402305': 1802, '2399436': 1803, '2399831': 1804, '2344880': 1805, '2366243': 1806, '2394911': 1807, '2340505': 1808, '2360192': 1809, '2355583': 1810, '2412691': 1811, '2317478': 1812, '2377654': 1813, '2393764': 1814, '2369389': 1815, '2380908': 1816, '2359734': 1817, '2395479': 1818, '2410117': 1819, '2345963': 1820, '2348675': 1821, '2402610': 1822, '2362917': 1823, '2381263': 1824, '2411367': 1825, '2350934': 1826, '2345484': 1827, '2375389': 1828, '2338460': 1829, '2392194': 1830, '2345283': 1831, '2357009': 1832, '2366433': 1833, '2400181': 1834, '2338625': 1835, '2385349': 1836, '2409624': 1837, '2325906': 1838, '2410111': 1839, '2356573': 1840, '2408231': 1841, '2374654': 1842, '2401986': 1843, '2406374': 1844, '2408091': 1845, '2388101': 1846, '2366266': 1847, '2358240': 1848, '2387174': 1849, '2364629': 1850, '2410500': 1851, '2396864': 1852, '2349287': 1853, '2341175': 1854, '2413048': 1855, '2352077': 1856, '2357255': 1857, '2364516': 1858, '2393711': 1859, '2332173': 1860, '2370652': 1861, '2379368': 1862, '2373368': 1863, '2391968': 1864, '2363840': 1865, '2356915': 1866, '2402864': 1867, '2347167': 1868, '2361726': 1869, '2363057': 1870, '195': 1871, '2389587': 1872, '2333962': 1873, '2339034': 1874, '2358787': 1875, '2413030': 1876, '2382967': 1877, '2356452': 1878, '2355462': 1879, '2398408': 1880, '2350770': 1881, '2361025': 1882, '2402846': 1883, '2341989': 1884, '2356957': 1885, '2371802': 1886, '2340693': 1887, '2365687': 1888, '2360971': 1889, '2383081': 1890, '2397831': 1891, '2382252': 1892, '2345262': 1893, '2412694': 1894, '2350816': 1895, '2356774': 1896, '2379797': 1897, '2355306': 1898, '2348088': 1899, '2392089': 1900, '2354342': 1901, '2414493': 1902, '2317122': 1903, '2411125': 1904, '2348278': 1905, '2380488': 1906, '2409861': 1907, '2414069': 1908, '2361015': 1909, '1186': 1910, '2348951': 1911, '2408872': 1912, '2358566': 1913, '2402174': 1914, '2376953': 1915, '2416072': 1916, '2399932': 1917, '2330371': 1918, '4526': 1919, '2368076': 1920, '2334644': 1921, '813': 1922, '2415207': 1923, '2406271': 1924, '262': 1925, '2400363': 1926, '2405126': 1927, '2392176': 1928, '2373460': 1929, '2413099': 1930, '2352124': 1931, '2317117': 1932, '2364952': 1933, '2345382': 1934, '2387583': 1935, '2386515': 1936, '2411830': 1937, '2344135': 1938, '2368524': 1939, '2354957': 1940, '2395289': 1941, '2345826': 1942, '2399843': 1943, '2360483': 1944, '2392613': 1945, '2346922': 1946, '2340767': 1947, '2395698': 1948, '2363261': 1949, '2341640': 1950, '2377290': 1951, '2379419': 1952, '2317513': 1953, '2405522': 1954, '2365947': 1955, '2371836': 1956, '2374929': 1957, '2348401': 1958, '2343057': 1959, '2339483': 1960, '2369764': 1961, '2393778': 1962, '2364711': 1963, '2331127': 1964, '1143': 1965, '2371418': 1966, '2403431': 1967, '2414878': 1968, '2384911': 1969, '2346679': 1970, '2333255': 1971, '2393843': 1972, '2322397': 1973, '2412964': 1974, '2341844': 1975, '2400168': 1976, '2371705': 1977, '2390876': 1978, '2336110': 1979, '587': 1980, '2364458': 1981, '2400258': 1982, '2387213': 1983, '2412725': 1984, '2408943': 1985, '2413997': 1986, '2369025': 1987, '3524': 1988, '2345255': 1989, '2387936': 1990, '2348847': 1991, '2361143': 1992, '2356669': 1993, '2351483': 1994, '2399453': 1995, '2392336': 1996, '2399058': 1997, '2387509': 1998, '2346365': 1999, '2387430': 2000, '2323101': 2001, '2393641': 2002, '2403291': 2003, '2352426': 2004, '2346311': 2005, '2334281': 2006, '2365395': 2007, '2367554': 2008, '2379698': 2009, '2356542': 2010, '2374256': 2011, '2378992': 2012, '2379893': 2013, '2395881': 2014, '1159529': 2015, '2347316': 2016, '2398565': 2017, '61515': 2018, '2382188': 2019, '2371228': 2020, '2374191': 2021, '2405763': 2022, '2373065': 2023, '2368332': 2024, '2366857': 2025, '3974': 2026, '2403142': 2027, '2403730': 2028, '2348008': 2029, '2364666': 2030, '2383971': 2031, '2323317': 2032, '2348859': 2033, '2351300': 2034, '2393837': 2035, '2360110': 2036, '2366855': 2037, '2410116': 2038, '2399588': 2039, '2404970': 2040, '2411064': 2041, '2372245': 2042, '2396669': 2043, '2387797': 2044, '2347378': 2045, '2330444': 2046, '2397123': 2047, '2367427': 2048, '2374351': 2049, '2360943': 2050, '2331431': 2051, '2338001': 2052, '2346419': 2053, '2317987': 2054, '2399954': 2055, '2415472': 2056, '2349266': 2057, '2384407': 2058, '2339402': 2059, '2395507': 2060, '2410775': 2061, '2318528': 2062, '2408638': 2063, '2358223': 2064, '2390577': 2065, '2360637': 2066, '2403309': 2067, '2389127': 2068, '2400665': 2069, '2345254': 2070, '2357568': 2071, '2368103': 2072, '2376750': 2073, '2362276': 2074, '2373822': 2075, '2318753': 2076, '2381335': 2077, '2396390': 2078, '2407563': 2079, '677': 2080, '2383588': 2081, '2364405': 2082, '2363823': 2083, '2332047': 2084, '2369980': 2085, '2411794': 2086, '2375098': 2087, '2349488': 2088, '2412833': 2089, '2389931': 2090, '2411635': 2091, '2321324': 2092, '2351789': 2093, '2330962': 2094, '2325271': 2095, '2393810': 2096, '2386491': 2097, '2351574': 2098, '2386840': 2099, '2342128': 2100, '2360241': 2101, '2389642': 2102, '2381424': 2103, '2378294': 2104, '2376247': 2105, '2387201': 2106, '2320667': 2107, '2376728': 2108, '2398703': 2109, '2369310': 2110, '2367175': 2111, '2324573': 2112, '2344722': 2113, '2341732': 2114, '2336105': 2115, '2388874': 2116, '2341737': 2117, '2322795': 2118, '2378412': 2119, '2387370': 2120, '2369286': 2121, '2318253': 2122, '2343618': 2123, '2354019': 2124, '2402340': 2125, '2392557': 2126, '2397147': 2127, '2353466': 2128, '2402479': 2129, '2393995': 2130, '2329101': 2131, '2404918': 2132, '2347211': 2133, '2345294': 2134, '2385897': 2135, '2319741': 2136, '2414099': 2137, '2403177': 2138, '2319593': 2139, '2402961': 2140, '2361554': 2141, '2356491': 2142, '2392581': 2143, '2316889': 2144, '2360287': 2145, '2371216': 2146, '2407026': 2147, '2367533': 2148, '2378060': 2149, '2389282': 2150, '2390572': 2151, '2397120': 2152, '2400392': 2153, '2316318': 2154, '2375898': 2155, '2362311': 2156, '2401191': 2157, '2405981': 2158, '2409153': 2159, '2377072': 2160, '2355213': 2161, '2392583': 2162, '2382441': 2163, '2317053': 2164, '2346718': 2165, '2350383': 2166, '2406500': 2167, '2362770': 2168, '2410681': 2169, '2374807': 2170, '2346501': 2171, '2357249': 2172, '2359027': 2173, '2396374': 2174, '2374760': 2175, '2399256': 2176, '2336180': 2177, '2355922': 2178, '2356858': 2179, '2340777': 2180, '2381824': 2181, '2377986': 2182, '2386257': 2183, '2397845': 2184, '2389726': 2185, '2411629': 2186, '2393060': 2187, '2345476': 2188, '2374490': 2189, '2393513': 2190, '2409142': 2191, '2396465': 2192, '2356592': 2193, '2398489': 2194, '2417769': 2195, '2380388': 2196, '2386965': 2197, '2388564': 2198, '2360225': 2199, '2377334': 2200, '2396670': 2201, '2354660': 2202, '2357502': 2203, '2359450': 2204, '2378490': 2205, '2362795': 2206, '2363104': 2207, '2335727': 2208, '2390392': 2209, '2343913': 2210, '2373951': 2211, '2377309': 2212, '2392460': 2213, '2354423': 2214, '2359238': 2215, '2379480': 2216, '2374611': 2217, '2384854': 2218, '2369580': 2219, '2326856': 2220, '2332039': 2221, '2411249': 2222, '2406478': 2223, '2328010': 2224, '2369213': 2225, '2387943': 2226, '2385460': 2227, '2330218': 2228, '2360556': 2229, '2341674': 2230, '2342464': 2231, '2352565': 2232, '2341047': 2233, '2401918': 2234, '2372733': 2235, '2371551': 2236, '2316697': 2237, '2376379': 2238, '2368666': 2239, '2402546': 2240, '1592156': 2241, '2388838': 2242, '2397307': 2243, '2348182': 2244, '2413071': 2245, '2404732': 2246, '2356427': 2247, '2407637': 2248, '2390263': 2249, '2405776': 2250, '2353337': 2251, '2316602': 2252, '2353270': 2253, '2408708': 2254, '2371198': 2255, '2354266': 2256, '2369855': 2257, '2380368': 2258, '2382039': 2259, '2358567': 2260, '2319011': 2261, '2370341': 2262, '2370551': 2263, '2362808': 2264, '2382330': 2265, '2408653': 2266, '2374473': 2267, '2362280': 2268, '2398037': 2269, '2375753': 2270, '2371017': 2271, '2370877': 2272, '2408209': 2273, '2333007': 2274, '2405738': 2275, '2391153': 2276, '2372453': 2277, '2358526': 2278, '2379303': 2279, '2389636': 2280, '2342857': 2281, '2415640': 2282, '2386870': 2283, '2363443': 2284, '2368634': 2285, '2413904': 2286, '2383358': 2287, '2380024': 2288, '2416559': 2289, '2336529': 2290, '2366500': 2291, '2351655': 2292, '2345044': 2293, '2373588': 2294, '2390099': 2295, '2373407': 2296, '2350763': 2297, '2402155': 2298, '2363548': 2299, '2322813': 2300, '2343805': 2301, '2391461': 2302, '2416656': 2303, '2402786': 2304, '2394339': 2305, '2414255': 2306, '2338443': 2307, '2382381': 2308, '2374927': 2309, '2362555': 2310, '2402202': 2311, '2382046': 2312, '2412163': 2313, '2354362': 2314, '2415082': 2315, '2408489': 2316, '2406617': 2317, '2319554': 2318, '2327849': 2319, '2378204': 2320, '2391047': 2321, '2341611': 2322, '2342864': 2323, '2386983': 2324, '2347702': 2325, '2322974': 2326, '2325414': 2327, '2400737': 2328, '2404002': 2329, '2392261': 2330, '2356269': 2331, '2358999': 2332, '2399577': 2333, '2370776': 2334, '2316535': 2335, '2395996': 2336, '2336048': 2337, '2376467': 2338, '2362203': 2339, '2415820': 2340, '2383182': 2341, '2350730': 2342, '2345893': 2343, '2375076': 2344, '2417944': 2345, '2337033': 2346, '2359636': 2347, '2408359': 2348, '2416307': 2349, '2398017': 2350, '2383168': 2351, '2330468': 2352, '2405056': 2353, '2321253': 2354, '2341034': 2355, '2317165': 2356, '2401897': 2357, '2398212': 2358, '2387721': 2359, '2344595': 2360, '2379759': 2361, '2393164': 2362, '2410345': 2363, '2363437': 2364, '2386462': 2365, '2388504': 2366, '2401336': 2367, '2321387': 2368, '2317132': 2369, '2396478': 2370, '2416725': 2371, '2372569': 2372, '2359537': 2373, '2322095': 2374, '2352174': 2375, '2343766': 2376, '2380329': 2377, '2361489': 2378, '2413702': 2379, '2321676': 2380, '2404393': 2381, '2356073': 2382, '2376970': 2383, '2358293': 2384, '2342802': 2385, '2402837': 2386, '2363661': 2387, '2403903': 2388, '2319419': 2389, '2344370': 2390, '2389353': 2391, '2397657': 2392, '2371546': 2393, '2347170': 2394, '2404791': 2395, '2384368': 2396, '2417585': 2397, '2392507': 2398, '2349036': 2399, '2329559': 2400, '879': 2401, '2414977': 2402, '2407808': 2403, '2378572': 2404, '2415884': 2405, '2414803': 2406, '2402120': 2407, '2320356': 2408, '2365699': 2409, '2354877': 2410, '2389638': 2411, '2334130': 2412, '2367765': 2413, '2379154': 2414, '2399862': 2415, '2404467': 2416, '2353974': 2417, '2352853': 2418, '2339578': 2419, '2406885': 2420, '2382775': 2421, '2354318': 2422, '2353172': 2423, '2412056': 2424, '2396290': 2425, '2336760': 2426, '2378276': 2427, '2379292': 2428, '2406213': 2429, '2416937': 2430, '2395164': 2431, '2356436': 2432, '2355527': 2433, '2386443': 2434, '2397502': 2435, '2410270': 2436, '2397071': 2437, '2415676': 2438, '2371756': 2439, '2394933': 2440, '2377521': 2441, '2320584': 2442, '2394242': 2443, '2385172': 2444, '2414806': 2445, '2380874': 2446, '2371037': 2447, '2317584': 2448, '2390476': 2449, '2356914': 2450, '2361991': 2451, '2350826': 2452, '2406588': 2453, '2358311': 2454, '2409964': 2455, '2403418': 2456, '2370296': 2457, '2411605': 2458, '2323547': 2459, '2316204': 2460, '2375241': 2461, '2349448': 2462, '2354115': 2463, '2404178': 2464, '2372285': 2465, '2401764': 2466, '2377897': 2467, '2328135': 2468, '2382879': 2469, '2360242': 2470, '2350037': 2471, '2361415': 2472, '2367100': 2473, '2411866': 2474, '2378223': 2475, '2346003': 2476, '2359944': 2477, '2386442': 2478, '2327810': 2479, '2345716': 2480, '2398747': 2481, '2327729': 2482, '2379103': 2483, '2330650': 2484, '2391165': 2485, '2403131': 2486}\n"
     ]
    }
   ],
   "source": [
    "    model_path = './models_batch/'\n",
    "    train_feats_path = './im2p_val_output.h5'\n",
    "    \n",
    "    train_output_file = h5py.File(train_feats_path, 'r')\n",
    "    #print(\"Train feats path--------------------------------------------->>>>>>>>\",train_output_file)\n",
    "    train_feats = train_output_file.get('feats')\n",
    "    #print(\"Train feats->>>>>>>>>>>>>>>>\",train_feats)\n",
    "    train_imgs_full_path_lists = open('./imgs_val_path.txt').read().splitlines()\n",
    "    #print(\"paths>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\",train_imgs_full_path_lists)\n",
    "    train_imgs_names = list(map(lambda x: os.path.basename(x).split('.')[0], train_imgs_full_path_lists))\n",
    "    #print(\"Train images names\",list(train_imgs_names))\n",
    "    \n",
    "    #print(\"Train images names\",list(train_imgs_names))\n",
    "    \n",
    "#     loss_fd = open('loss_batch.txt', 'w')\n",
    "    img2idx = {}\n",
    "    print(\"train_imgs_names-------\",train_imgs_names)\n",
    "    for idx, img in enumerate(train_imgs_names):\n",
    "        img2idx[img] = idx\n",
    "    print(\"IMG@IDX\",img2idx)\n",
    "\n",
    "    # plt draw the loss curve\n",
    "    # refer from: http://stackoverflow.com/questions/11874767/real-time-plotting-in-while-loop-with-matplotlib\n",
    "    loss_to_draw = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Cannot interpret feed_dict key as Tensor: The name 'save/Const:0' refers to a Tensor which does not exist. The operation, 'save/Const', does not exist in the graph.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1112\u001b[0m             subfeed_t = self.graph.as_graph_element(\n\u001b[0;32m-> 1113\u001b[0;31m                 subfeed, allow_tensor=True, allow_operation=False)\n\u001b[0m\u001b[1;32m   1114\u001b[0m           \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mas_graph_element\u001b[0;34m(self, obj, allow_tensor, allow_operation)\u001b[0m\n\u001b[1;32m   3795\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3796\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_as_graph_element_locked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_operation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3797\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_as_graph_element_locked\u001b[0;34m(self, obj, allow_tensor, allow_operation)\u001b[0m\n\u001b[1;32m   3837\u001b[0m                          \u001b[0;34m\"exist. The operation, %s, does not exist in the \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3838\u001b[0;31m                          \"graph.\" % (repr(name), repr(op_name)))\n\u001b[0m\u001b[1;32m   3839\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: \"The name 'save/Const:0' refers to a Tensor which does not exist. The operation, 'save/Const', does not exist in the graph.\"",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-9cdc99e33b48>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#tf.reset_default_graph()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0msaver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_meta_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./models_batch/model-100.meta'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./models_batch/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m101\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0mloss_to_draw_epoch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, sess, save_path)\u001b[0m\n\u001b[1;32m   1284\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1285\u001b[0m         sess.run(self.saver_def.restore_op_name,\n\u001b[0;32m-> 1286\u001b[0;31m                  {self.saver_def.filename_tensor_name: save_path})\n\u001b[0m\u001b[1;32m   1287\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNotFoundError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1288\u001b[0m       \u001b[0;31m# There are three common conditions that might cause this error:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1114\u001b[0m           \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1115\u001b[0m             raise TypeError(\n\u001b[0;32m-> 1116\u001b[0;31m                 'Cannot interpret feed_dict key as Tensor: ' + e.args[0])\n\u001b[0m\u001b[1;32m   1117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1118\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubfeed_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Cannot interpret feed_dict key as Tensor: The name 'save/Const:0' refers to a Tensor which does not exist. The operation, 'save/Const', does not exist in the graph."
     ]
    }
   ],
   "source": [
    "#sess = tf.InteractiveSession()\n",
    "#tf.reset_default_graph()\n",
    "saver = tf.train.import_meta_graph('./models_batch/model-100.meta')\n",
    "saver.restore(sess, tf.train.latest_checkpoint('./models_batch/'))\n",
    "for epoch in range(101, n_epochs):\n",
    "        loss_to_draw_epoch = []\n",
    "        #saver = tf.train.import_meta_graph('./models_batch/model-0.meta')\n",
    "        #saver.restore(sess, tf.train.latest_checkpoint('./models_batch/'))\n",
    "        # disorganize the order\n",
    "        #print(\"train images\",train_imgs_names,type(train_imgs_names))\n",
    "        #train_imgs_names=list(train_imgs_names)\n",
    "        random.shuffle(train_imgs_names)\n",
    "\n",
    "        for start, end in zip(range(0, len(train_imgs_names), batch_size),\n",
    "                              range(batch_size, len(train_imgs_names), batch_size)):\n",
    "            loss_fd = open('loss_batch.txt', 'a')\n",
    "            start_time = time.time()\n",
    "            print(\"Start and end\",(start,end))\n",
    "            img_name = train_imgs_names[start:end]\n",
    "            current_feats_index = map(lambda x: img2idx[x], img_name)\n",
    "            current_feats = np.asarray( list(map(lambda x: train_feats[x], current_feats_index) ))\n",
    "\n",
    "\n",
    "            current_num_distribution = np.asarray(list(map(lambda x: img2paragraph_modify[x][0], img_name)))\n",
    "            current_captions_matrix = np.asarray(list(map(lambda x: img2paragraph_modify[x][1], img_name)))\n",
    "\n",
    "            #print(\"Blah Blah\",current_captions_matrix,type(current_captions_matrix))\n",
    "            \n",
    "            \n",
    "            #print(\"Current captions matrix >>>>>>>>>>>>>>>>>>\",current_captions_matrix,type(current_captions_matrix))\n",
    "            #print(\"kwiojwejiowe\",(current_captions_matrix.shape)[0])\n",
    "            \n",
    "            current_captions_masks = np.zeros( (current_captions_matrix.shape[0], current_captions_matrix.shape[1], current_captions_matrix.shape[2]) )\n",
    "            #done by me current_captions_masks = np.zeros( (current_captions_matrix.shape[0], current_captions_matrix.shape[1], current_captions_matrix.shape[2]) )/idx2word_batch\n",
    "            # find the non-zero element\n",
    "            #done by me nonzeros = np.asarray(map(lambda each_matrix: np.asarray( map(lambda x: (x != 2).sum() + 1, each_matrix ) ), current_captions_matrix ) )\n",
    "            nonzeros = np.array(list(map(lambda each_matrix: np.array( list(map(lambda x: (x != 2).sum() + 1, each_matrix ) )), current_captions_matrix ) ))\n",
    "\n",
    "            #print(\"NONZEROS\",nonzeros,nonzeros.shape)\n",
    "            for i in range(batch_size):\n",
    "                for ind, row in enumerate(current_captions_masks[i]):\n",
    "                    row[:(nonzeros[i, ind]-1)] = 1\n",
    "\n",
    "            # shape of current_feats: batch_size x 50 x 4096\n",
    "            # shape of current_num_distribution: batch_size x 6\n",
    "            # shape of current_captions_matrix: batch_size x 6 x 50\n",
    "            #print(\"Done w the loop\")\n",
    "            #print(\"train op, tf loss ,tf_loss_sent,tf_loss_word\",[train_op, tf_loss, tf_loss_sent, tf_loss_word])\n",
    "            _, loss_val, loss_sent, loss_word= sess.run(\n",
    "                                [train_op, tf_loss, tf_loss_sent, tf_loss_word],\n",
    "                                feed_dict={\n",
    "                                           tf_feats: current_feats,\n",
    "                                           tf_num_distribution: current_num_distribution,\n",
    "                                           tf_captions_matrix: current_captions_matrix,\n",
    "                                           tf_captions_masks: current_captions_masks\n",
    "                                })\n",
    "\n",
    "            # append loss to list in a epoch\n",
    "            loss_to_draw_epoch.append(loss_val)\n",
    "\n",
    "            # running information\n",
    "            print('idx: ', start, ' Epoch: ', epoch, ' loss: ', loss_val, ' loss_sent: ', loss_sent, ' loss_word: ', loss_word, ' Time cost: ', str((time.time() - start_time)))\n",
    "            \n",
    "            loss_fd.write('start: '+ str(start) +' end: ' + str(end) +' epoch ' + str(epoch) + ' loss ' + str(loss_val) + '\\n')\n",
    "            loss_fd.close()\n",
    "\n",
    "        # draw loss curve every epoch\n",
    "        loss_to_draw.append(np.mean(loss_to_draw_epoch))\n",
    "        plt_save_dir = './loss_imgs'\n",
    "        plt_save_img_name = str(epoch) + '.png'\n",
    "        plt.plot(range(len(loss_to_draw)), loss_to_draw, color='g')\n",
    "        plt.grid(True)\n",
    "        plt.savefig(os.path.join(plt_save_dir, plt_save_img_name))\n",
    "\n",
    "        if np.mod(epoch, 1) == 0:\n",
    "            print(\"Epoch \", epoch, \" is done. Saving the model ...\")\n",
    "            saver.save(sess, os.path.join(model_path, 'model'), global_step=epoch)\n",
    "#loss_fd.close()\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing word counts and creating vocab based on word count threshold 2\n",
      "filtered words from 18418 to 9900\n",
      "Train Images Names ['2385757', '2389264', '2407325', '2337141', '2345813', '2386694', '2368581', '2397809', '185', '2387911', '2343682', '2409165', '2357129', '498337', '2326621', '2320205', '2412782', '2352466', '2414097', '2381526', '2390981', '2375715', '2405096', '2402848', '2346948', '2387882', '2352243', '2337581', '2330919', '2380444', '2377611', '2354637', '2401996', '2391375', '2415510', '2400079', '2366081', '2360527', '2365490', '2336229', '2367475', '2373691', '2376356', '2368952', '2369653', '2355012', '2400892', '2369971', '2414408', '2394007', '2415978', '2405444', '2401822', '2387651', '2350622', '2318603', '2383962', '2377677', '2380510', '2392746', '2412229', '2392926', '2362383', '2398991', '2367805', '2360264', '2400991', '2354133', '2374619', '2413514', '2387844', '2349764', '2403572', '2372008', '2388111', '2366871', '2385053', '2400907', '2379868', '2389754', '2389615', '2365263', '2414350', '2409171', '4025', '2352062', '2378864', '2367278', '2319025', '2335091', '2344088', '2351081', '2408962', '2319561', '2400672', '2390977', '2390288', '2415083', '2396514', '2411955', '2346205', '2395037', '2346498', '2383586', '2346342', '2367250', '2348629', '2413978', '2355118', '2390609', '724', '2347956', '2389121', '2381396', '2394635', '2374001', '2319411', '2356016', '2390189', '2339582', '2362277', '2325410', '2354590', '2367096', '2317592', '2384171', '2397387', '2376251', '2352643', '2354147', '2410352', '2332365', '2323588', '2504', '2346067', '2373700', '2395463', '2388627', '2344272', '2365691', '2364664', '2410717', '2371950', '2402941', '2391252', '2342646', '2352971', '2370165', '2395630', '2317591', '2378059', '2355709', '2336832', '2391722', '2330764', '2346660', '2409862', '2373155', '2393166', '2355642', '2346977', '2377832', '2343586', '2388431', '2381416', '2408088', '2343537', '2390500', '2319274', '2358132', '2336159', '2386365', '2395072', '2405965', '2390610', '2392765', '2363501', '2322189', '2400936', '2354380', '2385725', '2414772', '2387049', '2339305', '2355163', '2411856', '2325076', '2391875', '2369075', '2404587', '2371926', '2395640', '2329779', '2402384', '2348939', '2361855', '2395696', '2391663', '2370517', '2412037', '2347202', '2353100', '2372426', '2405703', '2370496', '2375612', '2338355', '2342324', '2400462', '2396112', '2400862', '2379832', '2333467', '2359089', '2398544', '2328805', '2345621', '2397165', '2391502', '2403224', '2356812', '2389832', '2399569', '2414157', '2392065', '2353938', '2374300', '2411459', '2390394', '2354220', '2380380', '2363108', '2342406', '2334854', '2410297', '2335993', '2411121', '2366378', '2401969', '2387542', '2380071', '2393134', '2315498', '2367637', '2370780', '2317055', '2383295', '2400542', '2402050', '2390890', '2386260', '2377837', '2318159', '2327364', '2409644', '2396452', '2341431', '2381222', '2389464', '2406927', '2371744', '2392787', '2376996', '2340394', '2333581', '2398578', '2348334', '2408546', '2354873', '2384670', '2408339', '2316661', '2318214', '2369502', '2344294', '2316382', '2389172', '2412334', '2382722', '2373092', '2400913', '2364094', '2415823', '2407666', '2408407', '2353425', '2385066', '2350985', '2396301', '2355676', '2318134', '2368058', '2359892', '2327449', '2342078', '2365122', '2416996', '2365457', '2376076', '2397271', '2317228', '2333227', '2343001', '2335613', '2336076', '2363469', '2387136', '2387602', '538', '2381047', '2341390', '2382974', '2393524', '2389704', '2352288', '2366841', '2366424', '2414990', '2382610', '2359501', '2341126', '2406861', '2381613', '2381116', '2362015', '2364508', '2361773', '2386125', '2323005', '2373886', '2391950', '2323158', '2372145', '2392202', '2362921', '150344', '2415140', '2375237', '2397509', '2405568', '2410491', '2365504', '2372907', '2359383', '2384101', '2330124', '2341434', '2403182', '2393422', '2350678', '2403683', '2334135', '2379332', '2382744', '2400289', '2351156', '2354589', '2366570', '2393002', '2375133', '2372957', '2346843', '2376314', '2357994', '2389532', '2355449', '2387258', '2386636', '2381310', '2331045', '2362731', '2353668', '2390084', '2347783', '2383040', '2341770', '2367808', '2397944', '2389426', '2391869', '2351511', '2406056', '2347102', '2414155', '2396971', '2404188', '2352834', '2397549', '2335504', '2390711', '2408710', '2318479', '2345601', '2323690', '2393174', '3665', '2343876', '2327045', '2414418', '2350160', '2360570', '2416836', '2410413', '2359369', '2407102', '4329', '2317914', '2375681', '2385986', '2357723', '2390201', '2376235', '2366660', '2346881', '2345363', '2358150', '853', '1067', '2391849', '2347742', '2374809', '2363649', '2317707', '2394449', '2407504', '2366958', '815', '2381442', '2386687', '2343486', '2403767', '2342183', '2342649', '2360855', '2384747', '2369741', '2338878', '2350786', '2377187', '2391182', '2345464', '2382463', '2373922', '2364569', '2321507', '2391257', '2390199', '2376974', '2411637', '2414334', '2382975', '2374130', '2407054', '2376001', '2405490', '2391435', '2360573', '2356093', '2317319', '2376855', '2372392', '2399545', '2407631', '2414694', '2387731', '2380250', '2399949', '2381902', '2380617', '2387786', '2343356', '2316729', '2335572', '2342386', '2341113', '2407180', '2372676', '2408452', '2351275', '2327618', '2391545', '2406319', '2369158', '2408036', '2409641', '2318461', '2376879', '2382464', '2358660', '2397224', '2360963', '2356329', '2397683', '2398695', '2318316', '2369188', '2383665', '2351900', '2355768', '2341725', '2335274', '2410913', '2404585', '2383900', '2370017', '2406329', '2375118', '2389880', '2333038', '2345388', '2404202', '2345290', '2412080', '2336489', '2379551', '2352259', '2398246', '2322086', '2412178', '2389036', '2411172', '2377272', '2369601', '2399464', '2351596', '2403297', '2353451', '2394454', '2401819', '2398902', '2394152', '2385541', '2369013', '2385974', '2344284', '2365226', '2322420', '2358081', '1160101', '2316641', '2371907', '2384500', '2344095', '2409873', '2342700', '2368261', '2349084', '2373021', '2339331', '2401288', '2334516', '2344881', '2364452', '2349469', '2374709', '2406451', '2391194', '2394298', '2406376', '2371567', '2335457', '2407029', '2372220', '2357948', '2384021', '2388187', '2331430', '2367019', '2392600', '2370343', '2417500', '2346802', '2564', '2360148', '2407311', '2350000', '2374917', '2354814', '2352832', '2383787', '2387276', '2413861', '2336315', '2357005', '2356221', '2400369', '2349353', '2366432', '2405436', '2407604', '2318841', '2394677', '2384439', '2348689', '2413793', '2384089', '2373382', '2402051', '2330715', '2403512', '2379975', '2411960', '2363971', '2353696', '2384283', '2395949', '2352715', '2333164', '2364799', '2399390', '2341305', '2410276', '2319462', '2333402', '2405554', '2344657', '2328347', '2415027', '2356488', '2367352', '2400108', '2380279', '2396396', '2339028', '2363834', '2366302', '2361964', '2343596', '2382173', '2371493', '2362648', '2381158', '2392123', '2371441', '2393319', '2353865', '2412966', '2402825', '2356671', '2385174', '2375529', '2354544', '2323848', '2382977', '2323209', '2387933', '2341028', '2411576', '2356811', '2387207', '2338341', '2381847', '2396662', '2367762', '2391902', '2350959', '2393496', '2366615', '2344117', '2333815', '2352342', '2387761', '2346469', '2382632', '2337106', '2357507', '2368903', '2400371', '2377573', '2399802', '2374100', '2387464', '2410958', '2316254', '2355876', '2319466', '2387553', '2387137', '2401818', '2413411', '2408481', '2320306', '2343554', '2383435', '2341747', '2411945', '2395545', '2347126', '2383269', '2341187', '2395566', '2361388', '2326048', '2326483', '2342090', '2330104', '2388858', '2414131', '2328024', '2405963', '2321198', '2342281', '2408454', '2407660', '2350676', '2324748', '2364648', '2399525', '2367980', '2369455', '2388006', '2406791', '2338235', '2399119', '2367983', '2398222', '2376293', '2379307', '2318313', '2395464', '2355425', '2344467', '2397941', '2412770', '2378990', '2391984', '2327406', '2376475', '2382690', '2414448', '2322107', '2414173', '2377368', '2398978', '2334930', '2354564', '2394297', '2355839', '2355172', '2388186', '2334825', '2372450', '2377971', '2377053', '2393315', '2370179', '2349895', '2335023', '2366720', '2369272', '2344127', '2393434', '2339219', '2348398', '2413035', '2330639', '2316575', '2343155', '2353872', '2414882', '2373063', '2333717', '2350248', '2383119', '2404930', '2335333', '2380117', '2400689', '2377131', '2353667', '2406595', '2343214', '2410358', '2330044', '2377161', '2347777', '2396968', '2343948', '2408400', '2342794', '2388737', '2401886', '2341672', '2385815', '2407806', '2317356', '2347763', '2341717', '2357770', '2378698', '2392376', '2373225', '1979', '2387187', '2341245', '2399182', '2410305', '2350362', '2354852', '2357069', '2391111', '2396639', '2343276', '2385145', '2319237', '2398879', '2378517', '2320078', '2357038', '2378426', '2409995', '2396407', '2344540', '2403306', '2412032', '2364541', '2379607', '2363110', '2385571', '2367180', '2317743', '2345625', '2324168', '2382524', '2389666', '2342125', '2326672', '2358834', '2416678', '2412679', '2373510', '2409161', '2330294', '2385025', '2382543', '2407293', '2405007', '2319434', '2409065', '2374686', '2415570', '2370645', '2343220', '2385158', '2366216', '2411891', '2396279', '2378386', '2406971', '2364907', '2383068', '2346025', '2401071', '2394536', '2346609', '2366521', '2336144', '2369882', '2362607', '2352327', '2343623', '2348208', '2357580', '2393529', '2347586', '2410663', '2338501', '2414226', '2317379', '2384614', '2368599', '2339675', '2392812', '2404620', '2362539', '2329100', '2343589', '2385552', '2412438', '2415173', '2376132', '2392014', '2346010', '285802', '2366148', '2416104', '2369615', '2367875', '2398860', '2318174', '2372154', '2327125', '2343802', '2364976', '2391397', '2412087', '2382659', '2325536', '2343595', '2397030', '2390728', '2409300', '2375570', '2363267', '2379412', '2318100', '2381952', '2410088', '2390131', '2350025', '2357873', '2332037', '2407133', '2387000', '2412304', '2334742', '2360125', '2351557', '2322620', '2364598', '2409948', '2407622', '2409730', '2395914', '2344000', '2367747', '2355445', '2336558', '2326536', '2336255', '2341700', '2387514', '2341671', '2398110', '2399652', '1981', '2323460', '2390548', '2378480', '2359190', '2388960', '2374610', '2390517', '2374041', '2416167', '2359860', '2345744', '2390516', '2350734', '2320101', '2362840', '2355186', '2374316', '2394482', '2345532', '2403827', '2350800', '2384225', '2408041', '2372039', '2362947', '2396741', '2328602', '2403866', '2408144', '2339689', '2373049', '2396987', '2399359', '2356053', '2371761', '2374295', '2400094', '2368870', '358', '2316710', '2348163', '2413458', '2315631', '2365595', '2409738', '2388139', '2367957', '2335492', '2350434', '2337232', '2317038', '2399813', '2364830', '2358273', '2317457', '2343118', '2325817', '2316887', '285905', '2357254', '1592902', '2387245', '2374212', '2393912', '2412430', '2396351', '2364742', '2326373', '2354526', '2410548', '2374003', '2339208', '2412323', '2347068', '2410762', '2354059', '2357064', '2394457', '2364582', '2388352', '2412817', '2345982', '2331553', '2344607', '2401917', '2335275', '2401129', '2410083', '2411500', '2398309', '2385216', '2381550', '2341363', '2415473', '2402262', '2391262', '2368478', '2344295', '2340547', '2326050', '2339004', '2346626', '2316675', '2372940', '2403736', '2361538', '2335133', '2372110', '2355888', '2408983', '2318897', '2349203', '2402756', '2385127', '1947', '2414540', '2360127', '2403553', '2361204', '2382861', '2340485', '2318856', '2349149', '2357096', '2378937', '2330143', '2390945', '2347477', '2367727', '2382133', '2365803', '2385464', '2352560', '2389937', '2335666', '2359041', '2371361', '2402586', '2400198', '2367081', '2330999', '2407240', '2373013', '2333510', '2341668', '2398193', '2384410', '2394250', '2401035', '2382813', '2369897', '2413842', '2335959', '2381711', '2394115', '2394823', '2342230', '2395930', '2375161', '2371998', '2357321', '2381052', '2386678', '2359589', '2408377', '2319233', '2377371', '2356978', '2322528', '2342127', '2351858', '2394808', '2372836', '2350140', '2330793', '2365464', '2324955', '59', '2401816', '2363606', '2411900', '2387088', '2400434', '2408594', '2379456', '2366127', '2322573', '2365692', '2354909', '2381999', '2400554', '2377456', '2338012', '2400676', '2409993', '2409525', '2329113', '2353014', '2375625', '2413783', '2405811', '2366795', '2374646', '2388879', '2317337', '2327958', '2378809', '2372018', '2397405', '2396019', '2358287', '2393934', '2411311', '2341541', '2326852', '2372176', '2371733', '2352132', '2365664', '2316733', '2365437', '1592372', '2358414', '2409530', '2410977', '2330167', '2370025', '2340772', '2358472', '2341383', '2415196', '2367052', '2349022', '2380268', '2394057', '2331963', '2348762', '2393268', '2382522', '2354169', '2346439', '2360521', '2413923', '2322133', '2318491', '2349530', '2388772', '2362719', '2406548', '2360201', '2351834', '2341514', '2400161', '2394967', '2316555', '2386847', '2412609', '2411640', '2376384', '2349745', '2358575', '2362360', '2319185', '2318300', '2319338', '2341703', '2348223', '2411941', '2404878', '2388166', '2348265', '2393093', '2386134', '2317860', '2358790', '2392630', '2381546', '2410233', '2396340', '2413134', '2384897', '2372279', '2412901', '2392163', '2399340', '2362560', '2415464', '2355350', '2385072', '2387946', '2414959', '2335223', '2396029', '2409922', '2369585', '2340615', '2342329', '2357014', '2359654', '2394493', '2339690', '2367978', '2385255', '2409589', '2404541', '2411332', '2395373', '2340926', '2379898', '2366907', '2392500', '2381777', '2371763', '2323566', '2362679', '2390775', '2326661', '2400330', '2354989', '2359414', '2351277', '2345482', '2347852', '2361424', '2363496', '2374097', '2392960', '2363592', '2338512', '2350783', '2390282', '2361585', '2346556', '2393655', '2381013', '2380114', '2413540', '2410181', '2366603', '2405292', '2352992', '2343612', '2395465', '2395628', '2386850', '2380640', '2368472', '2320897', '2404090', '2344453', '2320614', '2375431', '2354513', '2352072', '2368501', '2382276', '2327911', '2397716', '2354631', '2413575', '2327559', '2389618', '2370697', '2375622', '2403565', '2360115', '2399996', '2402523', '3668', '2342671', '2393354', '2358421', '2403502', '2358670', '2389588', '2401096', '2356526', '2380095', '2346818', '2371255', '2391668', '2385831', '2394596', '2342960', '2374375', '1160254', '2375771', '2382933', '2392920', '2387989', '2324588', '2356074', '2317625', '2344661', '2326911', '2404259', '2361721', '2393279', '2386495', '2400046', '2365043', '2344637', '2357554', '2390759', '2318426', '2386258', '2382545', '2336325', '2392061', '2405822', '2401033', '2353349', '2412624', '2340180', '2397756', '2402816', '2367884', '2323076', '2328094', '2402273', '2369292', '2414256', '2361129', '2379258', '2342678', '2383042', '2375286', '543', '2369707', '2415363', '2351567', '2374164', '2318354', '2337138', '2416386', '2344622', '2408617', '2402485', '2390061', '2403911', '2385461', '2368073', '2325644', '2413346', '2333242', '2409746', '2318784', '2356563', '2350520', '2379974', '2393488', '2401112', '2340853', '2355566', '2338820', '2388446', '2408244', '2403749', '2339735', '2345624', '2381108', '2396179', '2382666', '2377454', '2376391', '2347274', '2384475', '2317896', '2398386', '2343705', '2366203', '2336091', '2318766', '2362130', '2364869', '2388733', '2364463', '2320207', '2399556', '2318740', '2319024', '2320357', '2349108', '2324753', '2357052', '2405788', '2367095', '2355641', '2361523', '2348566', '2369937', '2364778', '2410070', '2349065', '2364313', '2385956', '2330315', '2339046', '2347982', '2389147', '2411813', '2366650', '2316025', '2340085', '2349725', '2402374', '2389782', '2318529', '2353047', '2355690', '2407520', '2340975', '2387531', '2355380', '2361208', '2405966', '2350585', '2334735', '2415776', '2391209', '2364493', '2404485', '2336765', '2390589', '2407074', '2355399', '2396682', '2359934', '2390911', '2352126', '2367144', '2393988', '2319437', '2384790', '2332609', '2363103', '2363314', '2371312', '2402053', '2415913', '2387492', '2355933', '2408558', '2368969', '2341785', '2391385', '2366303', '2338004', '2331437', '2405390', '2318074', '2382213', '2352218', '2356795', '2366679', '2373340', '2410425', '2347072', '2315529', '2414805', '2334296', '2400705', '2348349', '2371338', '2355887', '2336426', '2409509', '2400488', '2347040', '2373504', '2400709', '2388269', '2374858', '2357848', '2351724', '2366552', '2373343', '2366798', '2370562', '2336279', '2390109', '2376359', '2403968', '2377951', '2383724', '2327224', '2345537', '2320748', '2352059', '2356407', '2379040', '2404395', '2335228', '2377815', '2346974', '2398486', '2385876', '2368358', '2328180', '2333993', '2412028', '2320127', '2361275', '2325956', '2344077', '2373936', '2400518', '2410757', '2388557', '2374409', '2346288', '2327758', '2360829', '2367594', '2351200', '2401206', '2347475', '2361164', '2347157', '2322190', '2387508', '2328285', '2332131', '2399529', '2406648', '2414784', '2390009', '2385308', '2411532', '2320131', '2366219', '2385693', '2367729', '2383497', '2399749', '2387687', '2318508', '2366431', '2346215', '2332290', '2396197', '2343376', '2355111', '2315963', '2360432', '2329225', '2414189', '2407349', '2354529', '2378366', '2360200', '2386619', '2358729', '2396069', '2414523', '2402102', '2316860', '2359321', '2390811', '2403746', '2392451', '2370342', '1512', '2351235', '2356083', '2352140', '2397229', '2341890', '2377512', '2411735', '2384658', '1159516', '2360315', '2332658', '2413457', '2415457', '2349652', '2359079', '2324375', '2397605', '2403889', '2389796', '2406696', '2334724', '2407120', '2336668', '2366034', '2384206', '2400680', '2332379', '2392818', '2406733', '2387944', '2402124', '2334363', '2331362', '2388172', '2382849', '2388891', '713323', '2407039', '2386334', '2394687', '2408468', '2350752', '2384387', '2412787', '2409238', '2401910', '2346934', '2342976', '2362198', '2319960', '2384406', '2360405', '2399333', '2323665', '1159963', '2379625', '2405682', '2373934', '2337314', '2346066', '2396448', '2317900', '2407413', '2328394', '2374389', '2382714', '2339245', '2316574', '2363770', '2390523', '2415871', '2390619', '2353681', '2375141', '2365887', '2392468', '2376016', '2339907', '2386673', '2339891', '2414494', '2356926', '2351882', '2355581', '2380553', '2405893', '2375873', '2379970', '2389093', '2349339', '2351669', '2376371', '2382695', '2342062', '2343739', '2403096', '2367587', '2393255', '2363034', '2411245', '2356290', '2333032', '2377054', '2381840', '2383123', '2417815', '2342463', '2346654', '2401911', '2330589', '2359948', '2348627', '2395330', '2325024', '2389270', '2366510', '2373821', '2392869', '2381761', '2415192', '2324220', '2359272', '2386989', '2372029', '2317409', '2382557', '2315640', '2366640', '2401940', '2332111', '2371625', '2398866', '2399393', '2408374', '2372030', '2322341', '2374141', '1592381', '2346658', '2391256', '2397008', '2405504', '2378739', '2384906', '2408673', '2396317', '2388519', '2365979', '2414365', '2388526', '2337365', '2348280', '2360988', '2363427', '2368396', '2405392', '2396760', '2398451', '2400579', '2357859', '2348172', '2388316', '2357856', '2323413', '2400412', '2389597', '2364522', '2413323', '2336219', '2367716', '2362819', '2396757', '2380218', '2358578', '2390032', '2409345', '2410852', '2384053', '2378686', '2400643', '2380143', '2389002', '2320951', '2354093', '2362892', '2408480', '2361074', '2368771', '2409608', '2371432', '2344878', '2318431', '2403123', '2410146', '2342919', '2391895', '2413037', '2415154', '2352917', '2336606', '2409059', '2351967', '2344341', '2402305', '2399436', '2399831', '2344880', '2366243', '2394911', '2340505', '2360192', '2355583', '2412691', '2317478', '2377654', '2393764', '2369389', '2380908', '2359734', '2395479', '2410117', '2345963', '2348675', '2402610', '2362917', '2381263', '2411367', '2350934', '2345484', '2375389', '2338460', '2392194', '2345283', '2357009', '2366433', '2400181', '2338625', '2385349', '2409624', '2325906', '2410111', '2356573', '2408231', '2374654', '2401986', '2406374', '2408091', '2388101', '2366266', '2358240', '2387174', '2364629', '2410500', '2396864', '2349287', '2341175', '2413048', '2352077', '2357255', '2364516', '2393711', '2332173', '2370652', '2379368', '2373368', '2391968', '2363840', '2356915', '2402864', '2347167', '2361726', '2363057', '195', '2389587', '2333962', '2339034', '2358787', '2413030', '2382967', '2356452', '2355462', '2398408', '2350770', '2361025', '2402846', '2341989', '2356957', '2371802', '2340693', '2365687', '2360971', '2383081', '2397831', '2382252', '2345262', '2412694', '2350816', '2356774', '2379797', '2355306', '2348088', '2392089', '2354342', '2414493', '2317122', '2411125', '2348278', '2380488', '2409861', '2414069', '2361015', '1186', '2348951', '2408872', '2358566', '2402174', '2376953', '2416072', '2399932', '2330371', '4526', '2368076', '2334644', '813', '2415207', '2406271', '262', '2400363', '2405126', '2392176', '2373460', '2413099', '2352124', '2317117', '2364952', '2345382', '2387583', '2386515', '2411830', '2344135', '2368524', '2354957', '2395289', '2345826', '2399843', '2360483', '2392613', '2346922', '2340767', '2395698', '2363261', '2341640', '2377290', '2379419', '2317513', '2405522', '2365947', '2371836', '2374929', '2348401', '2343057', '2339483', '2369764', '2393778', '2364711', '2331127', '1143', '2371418', '2403431', '2414878', '2384911', '2346679', '2333255', '2393843', '2322397', '2412964', '2341844', '2400168', '2371705', '2390876', '2336110', '587', '2364458', '2400258', '2387213', '2412725', '2408943', '2413997', '2369025', '3524', '2345255', '2387936', '2348847', '2361143', '2356669', '2351483', '2399453', '2392336', '2399058', '2387509', '2346365', '2387430', '2323101', '2393641', '2403291', '2352426', '2346311', '2334281', '2365395', '2367554', '2379698', '2356542', '2374256', '2378992', '2379893', '2395881', '1159529', '2347316', '2398565', '61515', '2382188', '2371228', '2374191', '2405763', '2373065', '2368332', '2366857', '3974', '2403142', '2403730', '2348008', '2364666', '2383971', '2323317', '2348859', '2351300', '2393837', '2360110', '2366855', '2410116', '2399588', '2404970', '2411064', '2372245', '2396669', '2387797', '2347378', '2330444', '2397123', '2367427', '2374351', '2360943', '2331431', '2338001', '2346419', '2317987', '2399954', '2415472', '2349266', '2384407', '2339402', '2395507', '2410775', '2318528', '2408638', '2358223', '2390577', '2360637', '2403309', '2389127', '2400665', '2345254', '2357568', '2368103', '2376750', '2362276', '2373822', '2318753', '2381335', '2396390', '2407563', '677', '2383588', '2364405', '2363823', '2332047', '2369980', '2411794', '2375098', '2349488', '2412833', '2389931', '2411635', '2321324', '2351789', '2330962', '2325271', '2393810', '2386491', '2351574', '2386840', '2342128', '2360241', '2389642', '2381424', '2378294', '2376247', '2387201', '2320667', '2376728', '2398703', '2369310', '2367175', '2324573', '2344722', '2341732', '2336105', '2388874', '2341737', '2322795', '2378412', '2387370', '2369286', '2318253', '2343618', '2354019', '2402340', '2392557', '2397147', '2353466', '2402479', '2393995', '2329101', '2404918', '2347211', '2345294', '2385897', '2319741', '2414099', '2403177', '2319593', '2402961', '2361554', '2356491', '2392581', '2316889', '2360287', '2371216', '2407026', '2367533', '2378060', '2389282', '2390572', '2397120', '2400392', '2316318', '2375898', '2362311', '2401191', '2405981', '2409153', '2377072', '2355213', '2392583', '2382441', '2317053', '2346718', '2350383', '2406500', '2362770', '2410681', '2374807', '2346501', '2357249', '2359027', '2396374', '2374760', '2399256', '2336180', '2355922', '2356858', '2340777', '2381824', '2377986', '2386257', '2397845', '2389726', '2411629', '2393060', '2345476', '2374490', '2393513', '2409142', '2396465', '2356592', '2398489', '2417769', '2380388', '2386965', '2388564', '2360225', '2377334', '2396670', '2354660', '2357502', '2359450', '2378490', '2362795', '2363104', '2335727', '2390392', '2343913', '2373951', '2377309', '2392460', '2354423', '2359238', '2379480', '2374611', '2384854', '2369580', '2326856', '2332039', '2411249', '2406478', '2328010', '2369213', '2387943', '2385460', '2330218', '2360556', '2341674', '2342464', '2352565', '2341047', '2401918', '2372733', '2371551', '2316697', '2376379', '2368666', '2402546', '1592156', '2388838', '2397307', '2348182', '2413071', '2404732', '2356427', '2407637', '2390263', '2405776', '2353337', '2316602', '2353270', '2408708', '2371198', '2354266', '2369855', '2380368', '2382039', '2358567', '2319011', '2370341', '2370551', '2362808', '2382330', '2408653', '2374473', '2362280', '2398037', '2375753', '2371017', '2370877', '2408209', '2333007', '2405738', '2391153', '2372453', '2358526', '2379303', '2389636', '2342857', '2415640', '2386870', '2363443', '2368634', '2413904', '2383358', '2380024', '2416559', '2336529', '2366500', '2351655', '2345044', '2373588', '2390099', '2373407', '2350763', '2402155', '2363548', '2322813', '2343805', '2391461', '2416656', '2402786', '2394339', '2414255', '2338443', '2382381', '2374927', '2362555', '2402202', '2382046', '2412163', '2354362', '2415082', '2408489', '2406617', '2319554', '2327849', '2378204', '2391047', '2341611', '2342864', '2386983', '2347702', '2322974', '2325414', '2400737', '2404002', '2392261', '2356269', '2358999', '2399577', '2370776', '2316535', '2395996', '2336048', '2376467', '2362203', '2415820', '2383182', '2350730', '2345893', '2375076', '2417944', '2337033', '2359636', '2408359', '2416307', '2398017', '2383168', '2330468', '2405056', '2321253', '2341034', '2317165', '2401897', '2398212', '2387721', '2344595', '2379759', '2393164', '2410345', '2363437', '2386462', '2388504', '2401336', '2321387', '2317132', '2396478', '2416725', '2372569', '2359537', '2322095', '2352174', '2343766', '2380329', '2361489', '2413702', '2321676', '2404393', '2356073', '2376970', '2358293', '2342802', '2402837', '2363661', '2403903', '2319419', '2344370', '2389353', '2397657', '2371546', '2347170', '2404791', '2384368', '2417585', '2392507', '2349036', '2329559', '879', '2414977', '2407808', '2378572', '2415884', '2414803', '2402120', '2320356', '2365699', '2354877', '2389638', '2334130', '2367765', '2379154', '2399862', '2404467', '2353974', '2352853', '2339578', '2406885', '2382775', '2354318', '2353172', '2412056', '2396290', '2336760', '2378276', '2379292', '2406213', '2416937', '2395164', '2356436', '2355527', '2386443', '2397502', '2410270', '2397071', '2415676', '2371756', '2394933', '2377521', '2320584', '2394242', '2385172', '2414806', '2380874', '2371037', '2317584', '2390476', '2356914', '2361991', '2350826', '2406588', '2358311', '2409964', '2403418', '2370296', '2411605', '2323547', '2316204', '2375241', '2349448', '2354115', '2404178', '2372285', '2401764', '2377897', '2328135', '2382879', '2360242', '2350037', '2361415', '2367100', '2411866', '2378223', '2346003', '2359944', '2386442', '2327810', '2345716', '2398747', '2327729', '2379103', '2330650', '2391165', '2403131']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0711 14:30:03.287303 140399233443584 deprecation.py:323] From <ipython-input-1-ddcb88721579>:68: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "W0711 14:30:03.311513 140399233443584 deprecation.py:323] From <ipython-input-1-ddcb88721579>:86: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "W0711 14:30:03.426924 140399233443584 deprecation.py:506] From /home/student/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0711 14:30:03.448374 140399233443584 deprecation.py:506] From /home/student/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py:738: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wemb output>>>>>>>>>>>> <tf.Variable 'Wemb:0' shape=(9904, 1024) dtype=float32_ref> Tensor(\"Shape:0\", shape=(2,), dtype=int32)\n",
      "Start build model:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0711 14:30:03.778566 140399233443584 deprecation.py:323] From <ipython-input-1-ddcb88721579>:166: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "W0711 14:30:04.040371 140399233443584 deprecation.py:323] From <ipython-input-1-ddcb88721579>:197: sparse_to_dense (from tensorflow.python.ops.sparse_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Done w session\n",
      "Img2idx {'2385757': 0, '2389264': 1, '2407325': 2, '2337141': 3, '2345813': 4, '2386694': 5, '2368581': 6, '2397809': 7, '185': 8, '2387911': 9, '2343682': 10, '2409165': 11, '2357129': 12, '498337': 13, '2326621': 14, '2320205': 15, '2412782': 16, '2352466': 17, '2414097': 18, '2381526': 19, '2390981': 20, '2375715': 21, '2405096': 22, '2402848': 23, '2346948': 24, '2387882': 25, '2352243': 26, '2337581': 27, '2330919': 28, '2380444': 29, '2377611': 30, '2354637': 31, '2401996': 32, '2391375': 33, '2415510': 34, '2400079': 35, '2366081': 36, '2360527': 37, '2365490': 38, '2336229': 39, '2367475': 40, '2373691': 41, '2376356': 42, '2368952': 43, '2369653': 44, '2355012': 45, '2400892': 46, '2369971': 47, '2414408': 48, '2394007': 49, '2415978': 50, '2405444': 51, '2401822': 52, '2387651': 53, '2350622': 54, '2318603': 55, '2383962': 56, '2377677': 57, '2380510': 58, '2392746': 59, '2412229': 60, '2392926': 61, '2362383': 62, '2398991': 63, '2367805': 64, '2360264': 65, '2400991': 66, '2354133': 67, '2374619': 68, '2413514': 69, '2387844': 70, '2349764': 71, '2403572': 72, '2372008': 73, '2388111': 74, '2366871': 75, '2385053': 76, '2400907': 77, '2379868': 78, '2389754': 79, '2389615': 80, '2365263': 81, '2414350': 82, '2409171': 83, '4025': 84, '2352062': 85, '2378864': 86, '2367278': 87, '2319025': 88, '2335091': 89, '2344088': 90, '2351081': 91, '2408962': 92, '2319561': 93, '2400672': 94, '2390977': 95, '2390288': 96, '2415083': 97, '2396514': 98, '2411955': 99, '2346205': 100, '2395037': 101, '2346498': 102, '2383586': 103, '2346342': 104, '2367250': 105, '2348629': 106, '2413978': 107, '2355118': 108, '2390609': 109, '724': 110, '2347956': 111, '2389121': 112, '2381396': 113, '2394635': 114, '2374001': 115, '2319411': 116, '2356016': 117, '2390189': 118, '2339582': 119, '2362277': 120, '2325410': 121, '2354590': 122, '2367096': 123, '2317592': 124, '2384171': 125, '2397387': 126, '2376251': 127, '2352643': 128, '2354147': 129, '2410352': 130, '2332365': 131, '2323588': 132, '2504': 133, '2346067': 134, '2373700': 135, '2395463': 136, '2388627': 137, '2344272': 138, '2365691': 139, '2364664': 140, '2410717': 141, '2371950': 142, '2402941': 143, '2391252': 144, '2342646': 145, '2352971': 146, '2370165': 147, '2395630': 148, '2317591': 149, '2378059': 150, '2355709': 151, '2336832': 152, '2391722': 153, '2330764': 154, '2346660': 155, '2409862': 156, '2373155': 157, '2393166': 158, '2355642': 159, '2346977': 160, '2377832': 161, '2343586': 162, '2388431': 163, '2381416': 164, '2408088': 165, '2343537': 166, '2390500': 167, '2319274': 168, '2358132': 169, '2336159': 170, '2386365': 171, '2395072': 172, '2405965': 173, '2390610': 174, '2392765': 175, '2363501': 176, '2322189': 177, '2400936': 178, '2354380': 179, '2385725': 180, '2414772': 181, '2387049': 182, '2339305': 183, '2355163': 184, '2411856': 185, '2325076': 186, '2391875': 187, '2369075': 188, '2404587': 189, '2371926': 190, '2395640': 191, '2329779': 192, '2402384': 193, '2348939': 194, '2361855': 195, '2395696': 196, '2391663': 197, '2370517': 198, '2412037': 199, '2347202': 200, '2353100': 201, '2372426': 202, '2405703': 203, '2370496': 204, '2375612': 205, '2338355': 206, '2342324': 207, '2400462': 208, '2396112': 209, '2400862': 210, '2379832': 211, '2333467': 212, '2359089': 213, '2398544': 214, '2328805': 215, '2345621': 216, '2397165': 217, '2391502': 218, '2403224': 219, '2356812': 220, '2389832': 221, '2399569': 222, '2414157': 223, '2392065': 224, '2353938': 225, '2374300': 226, '2411459': 227, '2390394': 228, '2354220': 229, '2380380': 230, '2363108': 231, '2342406': 232, '2334854': 233, '2410297': 234, '2335993': 235, '2411121': 236, '2366378': 237, '2401969': 238, '2387542': 239, '2380071': 240, '2393134': 241, '2315498': 242, '2367637': 243, '2370780': 244, '2317055': 245, '2383295': 246, '2400542': 247, '2402050': 248, '2390890': 249, '2386260': 250, '2377837': 251, '2318159': 252, '2327364': 253, '2409644': 254, '2396452': 255, '2341431': 256, '2381222': 257, '2389464': 258, '2406927': 259, '2371744': 260, '2392787': 261, '2376996': 262, '2340394': 263, '2333581': 264, '2398578': 265, '2348334': 266, '2408546': 267, '2354873': 268, '2384670': 269, '2408339': 270, '2316661': 271, '2318214': 272, '2369502': 273, '2344294': 274, '2316382': 275, '2389172': 276, '2412334': 277, '2382722': 278, '2373092': 279, '2400913': 280, '2364094': 281, '2415823': 282, '2407666': 283, '2408407': 284, '2353425': 285, '2385066': 286, '2350985': 287, '2396301': 288, '2355676': 289, '2318134': 290, '2368058': 291, '2359892': 292, '2327449': 293, '2342078': 294, '2365122': 295, '2416996': 296, '2365457': 297, '2376076': 298, '2397271': 299, '2317228': 300, '2333227': 301, '2343001': 302, '2335613': 303, '2336076': 304, '2363469': 305, '2387136': 306, '2387602': 307, '538': 308, '2381047': 309, '2341390': 310, '2382974': 311, '2393524': 312, '2389704': 313, '2352288': 314, '2366841': 315, '2366424': 316, '2414990': 317, '2382610': 318, '2359501': 319, '2341126': 320, '2406861': 321, '2381613': 322, '2381116': 323, '2362015': 324, '2364508': 325, '2361773': 326, '2386125': 327, '2323005': 328, '2373886': 329, '2391950': 330, '2323158': 331, '2372145': 332, '2392202': 333, '2362921': 334, '150344': 335, '2415140': 336, '2375237': 337, '2397509': 338, '2405568': 339, '2410491': 340, '2365504': 341, '2372907': 342, '2359383': 343, '2384101': 344, '2330124': 345, '2341434': 346, '2403182': 347, '2393422': 348, '2350678': 349, '2403683': 350, '2334135': 351, '2379332': 352, '2382744': 353, '2400289': 354, '2351156': 355, '2354589': 356, '2366570': 357, '2393002': 358, '2375133': 359, '2372957': 360, '2346843': 361, '2376314': 362, '2357994': 363, '2389532': 364, '2355449': 365, '2387258': 366, '2386636': 367, '2381310': 368, '2331045': 369, '2362731': 370, '2353668': 371, '2390084': 372, '2347783': 373, '2383040': 374, '2341770': 375, '2367808': 376, '2397944': 377, '2389426': 378, '2391869': 379, '2351511': 380, '2406056': 381, '2347102': 382, '2414155': 383, '2396971': 384, '2404188': 385, '2352834': 386, '2397549': 387, '2335504': 388, '2390711': 389, '2408710': 390, '2318479': 391, '2345601': 392, '2323690': 393, '2393174': 394, '3665': 395, '2343876': 396, '2327045': 397, '2414418': 398, '2350160': 399, '2360570': 400, '2416836': 401, '2410413': 402, '2359369': 403, '2407102': 404, '4329': 405, '2317914': 406, '2375681': 407, '2385986': 408, '2357723': 409, '2390201': 410, '2376235': 411, '2366660': 412, '2346881': 413, '2345363': 414, '2358150': 415, '853': 416, '1067': 417, '2391849': 418, '2347742': 419, '2374809': 420, '2363649': 421, '2317707': 422, '2394449': 423, '2407504': 424, '2366958': 425, '815': 426, '2381442': 427, '2386687': 428, '2343486': 429, '2403767': 430, '2342183': 431, '2342649': 432, '2360855': 433, '2384747': 434, '2369741': 435, '2338878': 436, '2350786': 437, '2377187': 438, '2391182': 439, '2345464': 440, '2382463': 441, '2373922': 442, '2364569': 443, '2321507': 444, '2391257': 445, '2390199': 446, '2376974': 447, '2411637': 448, '2414334': 449, '2382975': 450, '2374130': 451, '2407054': 452, '2376001': 453, '2405490': 454, '2391435': 455, '2360573': 456, '2356093': 457, '2317319': 458, '2376855': 459, '2372392': 460, '2399545': 461, '2407631': 462, '2414694': 463, '2387731': 464, '2380250': 465, '2399949': 466, '2381902': 467, '2380617': 468, '2387786': 469, '2343356': 470, '2316729': 471, '2335572': 472, '2342386': 473, '2341113': 474, '2407180': 475, '2372676': 476, '2408452': 477, '2351275': 478, '2327618': 479, '2391545': 480, '2406319': 481, '2369158': 482, '2408036': 483, '2409641': 484, '2318461': 485, '2376879': 486, '2382464': 487, '2358660': 488, '2397224': 489, '2360963': 490, '2356329': 491, '2397683': 492, '2398695': 493, '2318316': 494, '2369188': 495, '2383665': 496, '2351900': 497, '2355768': 498, '2341725': 499, '2335274': 500, '2410913': 501, '2404585': 502, '2383900': 503, '2370017': 504, '2406329': 505, '2375118': 506, '2389880': 507, '2333038': 508, '2345388': 509, '2404202': 510, '2345290': 511, '2412080': 512, '2336489': 513, '2379551': 514, '2352259': 515, '2398246': 516, '2322086': 517, '2412178': 518, '2389036': 519, '2411172': 520, '2377272': 521, '2369601': 522, '2399464': 523, '2351596': 524, '2403297': 525, '2353451': 526, '2394454': 527, '2401819': 528, '2398902': 529, '2394152': 530, '2385541': 531, '2369013': 532, '2385974': 533, '2344284': 534, '2365226': 535, '2322420': 536, '2358081': 537, '1160101': 538, '2316641': 539, '2371907': 540, '2384500': 541, '2344095': 542, '2409873': 543, '2342700': 544, '2368261': 545, '2349084': 546, '2373021': 547, '2339331': 548, '2401288': 549, '2334516': 550, '2344881': 551, '2364452': 552, '2349469': 553, '2374709': 554, '2406451': 555, '2391194': 556, '2394298': 557, '2406376': 558, '2371567': 559, '2335457': 560, '2407029': 561, '2372220': 562, '2357948': 563, '2384021': 564, '2388187': 565, '2331430': 566, '2367019': 567, '2392600': 568, '2370343': 569, '2417500': 570, '2346802': 571, '2564': 572, '2360148': 573, '2407311': 574, '2350000': 575, '2374917': 576, '2354814': 577, '2352832': 578, '2383787': 579, '2387276': 580, '2413861': 581, '2336315': 582, '2357005': 583, '2356221': 584, '2400369': 585, '2349353': 586, '2366432': 587, '2405436': 588, '2407604': 589, '2318841': 590, '2394677': 591, '2384439': 592, '2348689': 593, '2413793': 594, '2384089': 595, '2373382': 596, '2402051': 597, '2330715': 598, '2403512': 599, '2379975': 600, '2411960': 601, '2363971': 602, '2353696': 603, '2384283': 604, '2395949': 605, '2352715': 606, '2333164': 607, '2364799': 608, '2399390': 609, '2341305': 610, '2410276': 611, '2319462': 612, '2333402': 613, '2405554': 614, '2344657': 615, '2328347': 616, '2415027': 617, '2356488': 618, '2367352': 619, '2400108': 620, '2380279': 621, '2396396': 622, '2339028': 623, '2363834': 624, '2366302': 625, '2361964': 626, '2343596': 627, '2382173': 628, '2371493': 629, '2362648': 630, '2381158': 631, '2392123': 632, '2371441': 633, '2393319': 634, '2353865': 635, '2412966': 636, '2402825': 637, '2356671': 638, '2385174': 639, '2375529': 640, '2354544': 641, '2323848': 642, '2382977': 643, '2323209': 644, '2387933': 645, '2341028': 646, '2411576': 647, '2356811': 648, '2387207': 649, '2338341': 650, '2381847': 651, '2396662': 652, '2367762': 653, '2391902': 654, '2350959': 655, '2393496': 656, '2366615': 657, '2344117': 658, '2333815': 659, '2352342': 660, '2387761': 661, '2346469': 662, '2382632': 663, '2337106': 664, '2357507': 665, '2368903': 666, '2400371': 667, '2377573': 668, '2399802': 669, '2374100': 670, '2387464': 671, '2410958': 672, '2316254': 673, '2355876': 674, '2319466': 675, '2387553': 676, '2387137': 677, '2401818': 678, '2413411': 679, '2408481': 680, '2320306': 681, '2343554': 682, '2383435': 683, '2341747': 684, '2411945': 685, '2395545': 686, '2347126': 687, '2383269': 688, '2341187': 689, '2395566': 690, '2361388': 691, '2326048': 692, '2326483': 693, '2342090': 694, '2330104': 695, '2388858': 696, '2414131': 697, '2328024': 698, '2405963': 699, '2321198': 700, '2342281': 701, '2408454': 702, '2407660': 703, '2350676': 704, '2324748': 705, '2364648': 706, '2399525': 707, '2367980': 708, '2369455': 709, '2388006': 710, '2406791': 711, '2338235': 712, '2399119': 713, '2367983': 714, '2398222': 715, '2376293': 716, '2379307': 717, '2318313': 718, '2395464': 719, '2355425': 720, '2344467': 721, '2397941': 722, '2412770': 723, '2378990': 724, '2391984': 725, '2327406': 726, '2376475': 727, '2382690': 728, '2414448': 729, '2322107': 730, '2414173': 731, '2377368': 732, '2398978': 733, '2334930': 734, '2354564': 735, '2394297': 736, '2355839': 737, '2355172': 738, '2388186': 739, '2334825': 740, '2372450': 741, '2377971': 742, '2377053': 743, '2393315': 744, '2370179': 745, '2349895': 746, '2335023': 747, '2366720': 748, '2369272': 749, '2344127': 750, '2393434': 751, '2339219': 752, '2348398': 753, '2413035': 754, '2330639': 755, '2316575': 756, '2343155': 757, '2353872': 758, '2414882': 759, '2373063': 760, '2333717': 761, '2350248': 762, '2383119': 763, '2404930': 764, '2335333': 765, '2380117': 766, '2400689': 767, '2377131': 768, '2353667': 769, '2406595': 770, '2343214': 771, '2410358': 772, '2330044': 773, '2377161': 774, '2347777': 775, '2396968': 776, '2343948': 777, '2408400': 778, '2342794': 779, '2388737': 780, '2401886': 781, '2341672': 782, '2385815': 783, '2407806': 784, '2317356': 785, '2347763': 786, '2341717': 787, '2357770': 788, '2378698': 789, '2392376': 790, '2373225': 791, '1979': 792, '2387187': 793, '2341245': 794, '2399182': 795, '2410305': 796, '2350362': 797, '2354852': 798, '2357069': 799, '2391111': 800, '2396639': 801, '2343276': 802, '2385145': 803, '2319237': 804, '2398879': 805, '2378517': 806, '2320078': 807, '2357038': 808, '2378426': 809, '2409995': 810, '2396407': 811, '2344540': 812, '2403306': 813, '2412032': 814, '2364541': 815, '2379607': 816, '2363110': 817, '2385571': 818, '2367180': 819, '2317743': 820, '2345625': 821, '2324168': 822, '2382524': 823, '2389666': 824, '2342125': 825, '2326672': 826, '2358834': 827, '2416678': 828, '2412679': 829, '2373510': 830, '2409161': 831, '2330294': 832, '2385025': 833, '2382543': 834, '2407293': 835, '2405007': 836, '2319434': 837, '2409065': 838, '2374686': 839, '2415570': 840, '2370645': 841, '2343220': 842, '2385158': 843, '2366216': 844, '2411891': 845, '2396279': 846, '2378386': 847, '2406971': 848, '2364907': 849, '2383068': 850, '2346025': 851, '2401071': 852, '2394536': 853, '2346609': 854, '2366521': 855, '2336144': 856, '2369882': 857, '2362607': 858, '2352327': 859, '2343623': 860, '2348208': 861, '2357580': 862, '2393529': 863, '2347586': 864, '2410663': 865, '2338501': 866, '2414226': 867, '2317379': 868, '2384614': 869, '2368599': 870, '2339675': 871, '2392812': 872, '2404620': 873, '2362539': 874, '2329100': 875, '2343589': 876, '2385552': 877, '2412438': 878, '2415173': 879, '2376132': 880, '2392014': 881, '2346010': 882, '285802': 883, '2366148': 884, '2416104': 885, '2369615': 886, '2367875': 887, '2398860': 888, '2318174': 889, '2372154': 890, '2327125': 891, '2343802': 892, '2364976': 893, '2391397': 894, '2412087': 895, '2382659': 896, '2325536': 897, '2343595': 898, '2397030': 899, '2390728': 900, '2409300': 901, '2375570': 902, '2363267': 903, '2379412': 904, '2318100': 905, '2381952': 906, '2410088': 907, '2390131': 908, '2350025': 909, '2357873': 910, '2332037': 911, '2407133': 912, '2387000': 913, '2412304': 914, '2334742': 915, '2360125': 916, '2351557': 917, '2322620': 918, '2364598': 919, '2409948': 920, '2407622': 921, '2409730': 922, '2395914': 923, '2344000': 924, '2367747': 925, '2355445': 926, '2336558': 927, '2326536': 928, '2336255': 929, '2341700': 930, '2387514': 931, '2341671': 932, '2398110': 933, '2399652': 934, '1981': 935, '2323460': 936, '2390548': 937, '2378480': 938, '2359190': 939, '2388960': 940, '2374610': 941, '2390517': 942, '2374041': 943, '2416167': 944, '2359860': 945, '2345744': 946, '2390516': 947, '2350734': 948, '2320101': 949, '2362840': 950, '2355186': 951, '2374316': 952, '2394482': 953, '2345532': 954, '2403827': 955, '2350800': 956, '2384225': 957, '2408041': 958, '2372039': 959, '2362947': 960, '2396741': 961, '2328602': 962, '2403866': 963, '2408144': 964, '2339689': 965, '2373049': 966, '2396987': 967, '2399359': 968, '2356053': 969, '2371761': 970, '2374295': 971, '2400094': 972, '2368870': 973, '358': 974, '2316710': 975, '2348163': 976, '2413458': 977, '2315631': 978, '2365595': 979, '2409738': 980, '2388139': 981, '2367957': 982, '2335492': 983, '2350434': 984, '2337232': 985, '2317038': 986, '2399813': 987, '2364830': 988, '2358273': 989, '2317457': 990, '2343118': 991, '2325817': 992, '2316887': 993, '285905': 994, '2357254': 995, '1592902': 996, '2387245': 997, '2374212': 998, '2393912': 999, '2412430': 1000, '2396351': 1001, '2364742': 1002, '2326373': 1003, '2354526': 1004, '2410548': 1005, '2374003': 1006, '2339208': 1007, '2412323': 1008, '2347068': 1009, '2410762': 1010, '2354059': 1011, '2357064': 1012, '2394457': 1013, '2364582': 1014, '2388352': 1015, '2412817': 1016, '2345982': 1017, '2331553': 1018, '2344607': 1019, '2401917': 1020, '2335275': 1021, '2401129': 1022, '2410083': 1023, '2411500': 1024, '2398309': 1025, '2385216': 1026, '2381550': 1027, '2341363': 1028, '2415473': 1029, '2402262': 1030, '2391262': 1031, '2368478': 1032, '2344295': 1033, '2340547': 1034, '2326050': 1035, '2339004': 1036, '2346626': 1037, '2316675': 1038, '2372940': 1039, '2403736': 1040, '2361538': 1041, '2335133': 1042, '2372110': 1043, '2355888': 1044, '2408983': 1045, '2318897': 1046, '2349203': 1047, '2402756': 1048, '2385127': 1049, '1947': 1050, '2414540': 1051, '2360127': 1052, '2403553': 1053, '2361204': 1054, '2382861': 1055, '2340485': 1056, '2318856': 1057, '2349149': 1058, '2357096': 1059, '2378937': 1060, '2330143': 1061, '2390945': 1062, '2347477': 1063, '2367727': 1064, '2382133': 1065, '2365803': 1066, '2385464': 1067, '2352560': 1068, '2389937': 1069, '2335666': 1070, '2359041': 1071, '2371361': 1072, '2402586': 1073, '2400198': 1074, '2367081': 1075, '2330999': 1076, '2407240': 1077, '2373013': 1078, '2333510': 1079, '2341668': 1080, '2398193': 1081, '2384410': 1082, '2394250': 1083, '2401035': 1084, '2382813': 1085, '2369897': 1086, '2413842': 1087, '2335959': 1088, '2381711': 1089, '2394115': 1090, '2394823': 1091, '2342230': 1092, '2395930': 1093, '2375161': 1094, '2371998': 1095, '2357321': 1096, '2381052': 1097, '2386678': 1098, '2359589': 1099, '2408377': 1100, '2319233': 1101, '2377371': 1102, '2356978': 1103, '2322528': 1104, '2342127': 1105, '2351858': 1106, '2394808': 1107, '2372836': 1108, '2350140': 1109, '2330793': 1110, '2365464': 1111, '2324955': 1112, '59': 1113, '2401816': 1114, '2363606': 1115, '2411900': 1116, '2387088': 1117, '2400434': 1118, '2408594': 1119, '2379456': 1120, '2366127': 1121, '2322573': 1122, '2365692': 1123, '2354909': 1124, '2381999': 1125, '2400554': 1126, '2377456': 1127, '2338012': 1128, '2400676': 1129, '2409993': 1130, '2409525': 1131, '2329113': 1132, '2353014': 1133, '2375625': 1134, '2413783': 1135, '2405811': 1136, '2366795': 1137, '2374646': 1138, '2388879': 1139, '2317337': 1140, '2327958': 1141, '2378809': 1142, '2372018': 1143, '2397405': 1144, '2396019': 1145, '2358287': 1146, '2393934': 1147, '2411311': 1148, '2341541': 1149, '2326852': 1150, '2372176': 1151, '2371733': 1152, '2352132': 1153, '2365664': 1154, '2316733': 1155, '2365437': 1156, '1592372': 1157, '2358414': 1158, '2409530': 1159, '2410977': 1160, '2330167': 1161, '2370025': 1162, '2340772': 1163, '2358472': 1164, '2341383': 1165, '2415196': 1166, '2367052': 1167, '2349022': 1168, '2380268': 1169, '2394057': 1170, '2331963': 1171, '2348762': 1172, '2393268': 1173, '2382522': 1174, '2354169': 1175, '2346439': 1176, '2360521': 1177, '2413923': 1178, '2322133': 1179, '2318491': 1180, '2349530': 1181, '2388772': 1182, '2362719': 1183, '2406548': 1184, '2360201': 1185, '2351834': 1186, '2341514': 1187, '2400161': 1188, '2394967': 1189, '2316555': 1190, '2386847': 1191, '2412609': 1192, '2411640': 1193, '2376384': 1194, '2349745': 1195, '2358575': 1196, '2362360': 1197, '2319185': 1198, '2318300': 1199, '2319338': 1200, '2341703': 1201, '2348223': 1202, '2411941': 1203, '2404878': 1204, '2388166': 1205, '2348265': 1206, '2393093': 1207, '2386134': 1208, '2317860': 1209, '2358790': 1210, '2392630': 1211, '2381546': 1212, '2410233': 1213, '2396340': 1214, '2413134': 1215, '2384897': 1216, '2372279': 1217, '2412901': 1218, '2392163': 1219, '2399340': 1220, '2362560': 1221, '2415464': 1222, '2355350': 1223, '2385072': 1224, '2387946': 1225, '2414959': 1226, '2335223': 1227, '2396029': 1228, '2409922': 1229, '2369585': 1230, '2340615': 1231, '2342329': 1232, '2357014': 1233, '2359654': 1234, '2394493': 1235, '2339690': 1236, '2367978': 1237, '2385255': 1238, '2409589': 1239, '2404541': 1240, '2411332': 1241, '2395373': 1242, '2340926': 1243, '2379898': 1244, '2366907': 1245, '2392500': 1246, '2381777': 1247, '2371763': 1248, '2323566': 1249, '2362679': 1250, '2390775': 1251, '2326661': 1252, '2400330': 1253, '2354989': 1254, '2359414': 1255, '2351277': 1256, '2345482': 1257, '2347852': 1258, '2361424': 1259, '2363496': 1260, '2374097': 1261, '2392960': 1262, '2363592': 1263, '2338512': 1264, '2350783': 1265, '2390282': 1266, '2361585': 1267, '2346556': 1268, '2393655': 1269, '2381013': 1270, '2380114': 1271, '2413540': 1272, '2410181': 1273, '2366603': 1274, '2405292': 1275, '2352992': 1276, '2343612': 1277, '2395465': 1278, '2395628': 1279, '2386850': 1280, '2380640': 1281, '2368472': 1282, '2320897': 1283, '2404090': 1284, '2344453': 1285, '2320614': 1286, '2375431': 1287, '2354513': 1288, '2352072': 1289, '2368501': 1290, '2382276': 1291, '2327911': 1292, '2397716': 1293, '2354631': 1294, '2413575': 1295, '2327559': 1296, '2389618': 1297, '2370697': 1298, '2375622': 1299, '2403565': 1300, '2360115': 1301, '2399996': 1302, '2402523': 1303, '3668': 1304, '2342671': 1305, '2393354': 1306, '2358421': 1307, '2403502': 1308, '2358670': 1309, '2389588': 1310, '2401096': 1311, '2356526': 1312, '2380095': 1313, '2346818': 1314, '2371255': 1315, '2391668': 1316, '2385831': 1317, '2394596': 1318, '2342960': 1319, '2374375': 1320, '1160254': 1321, '2375771': 1322, '2382933': 1323, '2392920': 1324, '2387989': 1325, '2324588': 1326, '2356074': 1327, '2317625': 1328, '2344661': 1329, '2326911': 1330, '2404259': 1331, '2361721': 1332, '2393279': 1333, '2386495': 1334, '2400046': 1335, '2365043': 1336, '2344637': 1337, '2357554': 1338, '2390759': 1339, '2318426': 1340, '2386258': 1341, '2382545': 1342, '2336325': 1343, '2392061': 1344, '2405822': 1345, '2401033': 1346, '2353349': 1347, '2412624': 1348, '2340180': 1349, '2397756': 1350, '2402816': 1351, '2367884': 1352, '2323076': 1353, '2328094': 1354, '2402273': 1355, '2369292': 1356, '2414256': 1357, '2361129': 1358, '2379258': 1359, '2342678': 1360, '2383042': 1361, '2375286': 1362, '543': 1363, '2369707': 1364, '2415363': 1365, '2351567': 1366, '2374164': 1367, '2318354': 1368, '2337138': 1369, '2416386': 1370, '2344622': 1371, '2408617': 1372, '2402485': 1373, '2390061': 1374, '2403911': 1375, '2385461': 1376, '2368073': 1377, '2325644': 1378, '2413346': 1379, '2333242': 1380, '2409746': 1381, '2318784': 1382, '2356563': 1383, '2350520': 1384, '2379974': 1385, '2393488': 1386, '2401112': 1387, '2340853': 1388, '2355566': 1389, '2338820': 1390, '2388446': 1391, '2408244': 1392, '2403749': 1393, '2339735': 1394, '2345624': 1395, '2381108': 1396, '2396179': 1397, '2382666': 1398, '2377454': 1399, '2376391': 1400, '2347274': 1401, '2384475': 1402, '2317896': 1403, '2398386': 1404, '2343705': 1405, '2366203': 1406, '2336091': 1407, '2318766': 1408, '2362130': 1409, '2364869': 1410, '2388733': 1411, '2364463': 1412, '2320207': 1413, '2399556': 1414, '2318740': 1415, '2319024': 1416, '2320357': 1417, '2349108': 1418, '2324753': 1419, '2357052': 1420, '2405788': 1421, '2367095': 1422, '2355641': 1423, '2361523': 1424, '2348566': 1425, '2369937': 1426, '2364778': 1427, '2410070': 1428, '2349065': 1429, '2364313': 1430, '2385956': 1431, '2330315': 1432, '2339046': 1433, '2347982': 1434, '2389147': 1435, '2411813': 1436, '2366650': 1437, '2316025': 1438, '2340085': 1439, '2349725': 1440, '2402374': 1441, '2389782': 1442, '2318529': 1443, '2353047': 1444, '2355690': 1445, '2407520': 1446, '2340975': 1447, '2387531': 1448, '2355380': 1449, '2361208': 1450, '2405966': 1451, '2350585': 1452, '2334735': 1453, '2415776': 1454, '2391209': 1455, '2364493': 1456, '2404485': 1457, '2336765': 1458, '2390589': 1459, '2407074': 1460, '2355399': 1461, '2396682': 1462, '2359934': 1463, '2390911': 1464, '2352126': 1465, '2367144': 1466, '2393988': 1467, '2319437': 1468, '2384790': 1469, '2332609': 1470, '2363103': 1471, '2363314': 1472, '2371312': 1473, '2402053': 1474, '2415913': 1475, '2387492': 1476, '2355933': 1477, '2408558': 1478, '2368969': 1479, '2341785': 1480, '2391385': 1481, '2366303': 1482, '2338004': 1483, '2331437': 1484, '2405390': 1485, '2318074': 1486, '2382213': 1487, '2352218': 1488, '2356795': 1489, '2366679': 1490, '2373340': 1491, '2410425': 1492, '2347072': 1493, '2315529': 1494, '2414805': 1495, '2334296': 1496, '2400705': 1497, '2348349': 1498, '2371338': 1499, '2355887': 1500, '2336426': 1501, '2409509': 1502, '2400488': 1503, '2347040': 1504, '2373504': 1505, '2400709': 1506, '2388269': 1507, '2374858': 1508, '2357848': 1509, '2351724': 1510, '2366552': 1511, '2373343': 1512, '2366798': 1513, '2370562': 1514, '2336279': 1515, '2390109': 1516, '2376359': 1517, '2403968': 1518, '2377951': 1519, '2383724': 1520, '2327224': 1521, '2345537': 1522, '2320748': 1523, '2352059': 1524, '2356407': 1525, '2379040': 1526, '2404395': 1527, '2335228': 1528, '2377815': 1529, '2346974': 1530, '2398486': 1531, '2385876': 1532, '2368358': 1533, '2328180': 1534, '2333993': 1535, '2412028': 1536, '2320127': 1537, '2361275': 1538, '2325956': 1539, '2344077': 1540, '2373936': 1541, '2400518': 1542, '2410757': 1543, '2388557': 1544, '2374409': 1545, '2346288': 1546, '2327758': 1547, '2360829': 1548, '2367594': 1549, '2351200': 1550, '2401206': 1551, '2347475': 1552, '2361164': 1553, '2347157': 1554, '2322190': 1555, '2387508': 1556, '2328285': 1557, '2332131': 1558, '2399529': 1559, '2406648': 1560, '2414784': 1561, '2390009': 1562, '2385308': 1563, '2411532': 1564, '2320131': 1565, '2366219': 1566, '2385693': 1567, '2367729': 1568, '2383497': 1569, '2399749': 1570, '2387687': 1571, '2318508': 1572, '2366431': 1573, '2346215': 1574, '2332290': 1575, '2396197': 1576, '2343376': 1577, '2355111': 1578, '2315963': 1579, '2360432': 1580, '2329225': 1581, '2414189': 1582, '2407349': 1583, '2354529': 1584, '2378366': 1585, '2360200': 1586, '2386619': 1587, '2358729': 1588, '2396069': 1589, '2414523': 1590, '2402102': 1591, '2316860': 1592, '2359321': 1593, '2390811': 1594, '2403746': 1595, '2392451': 1596, '2370342': 1597, '1512': 1598, '2351235': 1599, '2356083': 1600, '2352140': 1601, '2397229': 1602, '2341890': 1603, '2377512': 1604, '2411735': 1605, '2384658': 1606, '1159516': 1607, '2360315': 1608, '2332658': 1609, '2413457': 1610, '2415457': 1611, '2349652': 1612, '2359079': 1613, '2324375': 1614, '2397605': 1615, '2403889': 1616, '2389796': 1617, '2406696': 1618, '2334724': 1619, '2407120': 1620, '2336668': 1621, '2366034': 1622, '2384206': 1623, '2400680': 1624, '2332379': 1625, '2392818': 1626, '2406733': 1627, '2387944': 1628, '2402124': 1629, '2334363': 1630, '2331362': 1631, '2388172': 1632, '2382849': 1633, '2388891': 1634, '713323': 1635, '2407039': 1636, '2386334': 1637, '2394687': 1638, '2408468': 1639, '2350752': 1640, '2384387': 1641, '2412787': 1642, '2409238': 1643, '2401910': 1644, '2346934': 1645, '2342976': 1646, '2362198': 1647, '2319960': 1648, '2384406': 1649, '2360405': 1650, '2399333': 1651, '2323665': 1652, '1159963': 1653, '2379625': 1654, '2405682': 1655, '2373934': 1656, '2337314': 1657, '2346066': 1658, '2396448': 1659, '2317900': 1660, '2407413': 1661, '2328394': 1662, '2374389': 1663, '2382714': 1664, '2339245': 1665, '2316574': 1666, '2363770': 1667, '2390523': 1668, '2415871': 1669, '2390619': 1670, '2353681': 1671, '2375141': 1672, '2365887': 1673, '2392468': 1674, '2376016': 1675, '2339907': 1676, '2386673': 1677, '2339891': 1678, '2414494': 1679, '2356926': 1680, '2351882': 1681, '2355581': 1682, '2380553': 1683, '2405893': 1684, '2375873': 1685, '2379970': 1686, '2389093': 1687, '2349339': 1688, '2351669': 1689, '2376371': 1690, '2382695': 1691, '2342062': 1692, '2343739': 1693, '2403096': 1694, '2367587': 1695, '2393255': 1696, '2363034': 1697, '2411245': 1698, '2356290': 1699, '2333032': 1700, '2377054': 1701, '2381840': 1702, '2383123': 1703, '2417815': 1704, '2342463': 1705, '2346654': 1706, '2401911': 1707, '2330589': 1708, '2359948': 1709, '2348627': 1710, '2395330': 1711, '2325024': 1712, '2389270': 1713, '2366510': 1714, '2373821': 1715, '2392869': 1716, '2381761': 1717, '2415192': 1718, '2324220': 1719, '2359272': 1720, '2386989': 1721, '2372029': 1722, '2317409': 1723, '2382557': 1724, '2315640': 1725, '2366640': 1726, '2401940': 1727, '2332111': 1728, '2371625': 1729, '2398866': 1730, '2399393': 1731, '2408374': 1732, '2372030': 1733, '2322341': 1734, '2374141': 1735, '1592381': 1736, '2346658': 1737, '2391256': 1738, '2397008': 1739, '2405504': 1740, '2378739': 1741, '2384906': 1742, '2408673': 1743, '2396317': 1744, '2388519': 1745, '2365979': 1746, '2414365': 1747, '2388526': 1748, '2337365': 1749, '2348280': 1750, '2360988': 1751, '2363427': 1752, '2368396': 1753, '2405392': 1754, '2396760': 1755, '2398451': 1756, '2400579': 1757, '2357859': 1758, '2348172': 1759, '2388316': 1760, '2357856': 1761, '2323413': 1762, '2400412': 1763, '2389597': 1764, '2364522': 1765, '2413323': 1766, '2336219': 1767, '2367716': 1768, '2362819': 1769, '2396757': 1770, '2380218': 1771, '2358578': 1772, '2390032': 1773, '2409345': 1774, '2410852': 1775, '2384053': 1776, '2378686': 1777, '2400643': 1778, '2380143': 1779, '2389002': 1780, '2320951': 1781, '2354093': 1782, '2362892': 1783, '2408480': 1784, '2361074': 1785, '2368771': 1786, '2409608': 1787, '2371432': 1788, '2344878': 1789, '2318431': 1790, '2403123': 1791, '2410146': 1792, '2342919': 1793, '2391895': 1794, '2413037': 1795, '2415154': 1796, '2352917': 1797, '2336606': 1798, '2409059': 1799, '2351967': 1800, '2344341': 1801, '2402305': 1802, '2399436': 1803, '2399831': 1804, '2344880': 1805, '2366243': 1806, '2394911': 1807, '2340505': 1808, '2360192': 1809, '2355583': 1810, '2412691': 1811, '2317478': 1812, '2377654': 1813, '2393764': 1814, '2369389': 1815, '2380908': 1816, '2359734': 1817, '2395479': 1818, '2410117': 1819, '2345963': 1820, '2348675': 1821, '2402610': 1822, '2362917': 1823, '2381263': 1824, '2411367': 1825, '2350934': 1826, '2345484': 1827, '2375389': 1828, '2338460': 1829, '2392194': 1830, '2345283': 1831, '2357009': 1832, '2366433': 1833, '2400181': 1834, '2338625': 1835, '2385349': 1836, '2409624': 1837, '2325906': 1838, '2410111': 1839, '2356573': 1840, '2408231': 1841, '2374654': 1842, '2401986': 1843, '2406374': 1844, '2408091': 1845, '2388101': 1846, '2366266': 1847, '2358240': 1848, '2387174': 1849, '2364629': 1850, '2410500': 1851, '2396864': 1852, '2349287': 1853, '2341175': 1854, '2413048': 1855, '2352077': 1856, '2357255': 1857, '2364516': 1858, '2393711': 1859, '2332173': 1860, '2370652': 1861, '2379368': 1862, '2373368': 1863, '2391968': 1864, '2363840': 1865, '2356915': 1866, '2402864': 1867, '2347167': 1868, '2361726': 1869, '2363057': 1870, '195': 1871, '2389587': 1872, '2333962': 1873, '2339034': 1874, '2358787': 1875, '2413030': 1876, '2382967': 1877, '2356452': 1878, '2355462': 1879, '2398408': 1880, '2350770': 1881, '2361025': 1882, '2402846': 1883, '2341989': 1884, '2356957': 1885, '2371802': 1886, '2340693': 1887, '2365687': 1888, '2360971': 1889, '2383081': 1890, '2397831': 1891, '2382252': 1892, '2345262': 1893, '2412694': 1894, '2350816': 1895, '2356774': 1896, '2379797': 1897, '2355306': 1898, '2348088': 1899, '2392089': 1900, '2354342': 1901, '2414493': 1902, '2317122': 1903, '2411125': 1904, '2348278': 1905, '2380488': 1906, '2409861': 1907, '2414069': 1908, '2361015': 1909, '1186': 1910, '2348951': 1911, '2408872': 1912, '2358566': 1913, '2402174': 1914, '2376953': 1915, '2416072': 1916, '2399932': 1917, '2330371': 1918, '4526': 1919, '2368076': 1920, '2334644': 1921, '813': 1922, '2415207': 1923, '2406271': 1924, '262': 1925, '2400363': 1926, '2405126': 1927, '2392176': 1928, '2373460': 1929, '2413099': 1930, '2352124': 1931, '2317117': 1932, '2364952': 1933, '2345382': 1934, '2387583': 1935, '2386515': 1936, '2411830': 1937, '2344135': 1938, '2368524': 1939, '2354957': 1940, '2395289': 1941, '2345826': 1942, '2399843': 1943, '2360483': 1944, '2392613': 1945, '2346922': 1946, '2340767': 1947, '2395698': 1948, '2363261': 1949, '2341640': 1950, '2377290': 1951, '2379419': 1952, '2317513': 1953, '2405522': 1954, '2365947': 1955, '2371836': 1956, '2374929': 1957, '2348401': 1958, '2343057': 1959, '2339483': 1960, '2369764': 1961, '2393778': 1962, '2364711': 1963, '2331127': 1964, '1143': 1965, '2371418': 1966, '2403431': 1967, '2414878': 1968, '2384911': 1969, '2346679': 1970, '2333255': 1971, '2393843': 1972, '2322397': 1973, '2412964': 1974, '2341844': 1975, '2400168': 1976, '2371705': 1977, '2390876': 1978, '2336110': 1979, '587': 1980, '2364458': 1981, '2400258': 1982, '2387213': 1983, '2412725': 1984, '2408943': 1985, '2413997': 1986, '2369025': 1987, '3524': 1988, '2345255': 1989, '2387936': 1990, '2348847': 1991, '2361143': 1992, '2356669': 1993, '2351483': 1994, '2399453': 1995, '2392336': 1996, '2399058': 1997, '2387509': 1998, '2346365': 1999, '2387430': 2000, '2323101': 2001, '2393641': 2002, '2403291': 2003, '2352426': 2004, '2346311': 2005, '2334281': 2006, '2365395': 2007, '2367554': 2008, '2379698': 2009, '2356542': 2010, '2374256': 2011, '2378992': 2012, '2379893': 2013, '2395881': 2014, '1159529': 2015, '2347316': 2016, '2398565': 2017, '61515': 2018, '2382188': 2019, '2371228': 2020, '2374191': 2021, '2405763': 2022, '2373065': 2023, '2368332': 2024, '2366857': 2025, '3974': 2026, '2403142': 2027, '2403730': 2028, '2348008': 2029, '2364666': 2030, '2383971': 2031, '2323317': 2032, '2348859': 2033, '2351300': 2034, '2393837': 2035, '2360110': 2036, '2366855': 2037, '2410116': 2038, '2399588': 2039, '2404970': 2040, '2411064': 2041, '2372245': 2042, '2396669': 2043, '2387797': 2044, '2347378': 2045, '2330444': 2046, '2397123': 2047, '2367427': 2048, '2374351': 2049, '2360943': 2050, '2331431': 2051, '2338001': 2052, '2346419': 2053, '2317987': 2054, '2399954': 2055, '2415472': 2056, '2349266': 2057, '2384407': 2058, '2339402': 2059, '2395507': 2060, '2410775': 2061, '2318528': 2062, '2408638': 2063, '2358223': 2064, '2390577': 2065, '2360637': 2066, '2403309': 2067, '2389127': 2068, '2400665': 2069, '2345254': 2070, '2357568': 2071, '2368103': 2072, '2376750': 2073, '2362276': 2074, '2373822': 2075, '2318753': 2076, '2381335': 2077, '2396390': 2078, '2407563': 2079, '677': 2080, '2383588': 2081, '2364405': 2082, '2363823': 2083, '2332047': 2084, '2369980': 2085, '2411794': 2086, '2375098': 2087, '2349488': 2088, '2412833': 2089, '2389931': 2090, '2411635': 2091, '2321324': 2092, '2351789': 2093, '2330962': 2094, '2325271': 2095, '2393810': 2096, '2386491': 2097, '2351574': 2098, '2386840': 2099, '2342128': 2100, '2360241': 2101, '2389642': 2102, '2381424': 2103, '2378294': 2104, '2376247': 2105, '2387201': 2106, '2320667': 2107, '2376728': 2108, '2398703': 2109, '2369310': 2110, '2367175': 2111, '2324573': 2112, '2344722': 2113, '2341732': 2114, '2336105': 2115, '2388874': 2116, '2341737': 2117, '2322795': 2118, '2378412': 2119, '2387370': 2120, '2369286': 2121, '2318253': 2122, '2343618': 2123, '2354019': 2124, '2402340': 2125, '2392557': 2126, '2397147': 2127, '2353466': 2128, '2402479': 2129, '2393995': 2130, '2329101': 2131, '2404918': 2132, '2347211': 2133, '2345294': 2134, '2385897': 2135, '2319741': 2136, '2414099': 2137, '2403177': 2138, '2319593': 2139, '2402961': 2140, '2361554': 2141, '2356491': 2142, '2392581': 2143, '2316889': 2144, '2360287': 2145, '2371216': 2146, '2407026': 2147, '2367533': 2148, '2378060': 2149, '2389282': 2150, '2390572': 2151, '2397120': 2152, '2400392': 2153, '2316318': 2154, '2375898': 2155, '2362311': 2156, '2401191': 2157, '2405981': 2158, '2409153': 2159, '2377072': 2160, '2355213': 2161, '2392583': 2162, '2382441': 2163, '2317053': 2164, '2346718': 2165, '2350383': 2166, '2406500': 2167, '2362770': 2168, '2410681': 2169, '2374807': 2170, '2346501': 2171, '2357249': 2172, '2359027': 2173, '2396374': 2174, '2374760': 2175, '2399256': 2176, '2336180': 2177, '2355922': 2178, '2356858': 2179, '2340777': 2180, '2381824': 2181, '2377986': 2182, '2386257': 2183, '2397845': 2184, '2389726': 2185, '2411629': 2186, '2393060': 2187, '2345476': 2188, '2374490': 2189, '2393513': 2190, '2409142': 2191, '2396465': 2192, '2356592': 2193, '2398489': 2194, '2417769': 2195, '2380388': 2196, '2386965': 2197, '2388564': 2198, '2360225': 2199, '2377334': 2200, '2396670': 2201, '2354660': 2202, '2357502': 2203, '2359450': 2204, '2378490': 2205, '2362795': 2206, '2363104': 2207, '2335727': 2208, '2390392': 2209, '2343913': 2210, '2373951': 2211, '2377309': 2212, '2392460': 2213, '2354423': 2214, '2359238': 2215, '2379480': 2216, '2374611': 2217, '2384854': 2218, '2369580': 2219, '2326856': 2220, '2332039': 2221, '2411249': 2222, '2406478': 2223, '2328010': 2224, '2369213': 2225, '2387943': 2226, '2385460': 2227, '2330218': 2228, '2360556': 2229, '2341674': 2230, '2342464': 2231, '2352565': 2232, '2341047': 2233, '2401918': 2234, '2372733': 2235, '2371551': 2236, '2316697': 2237, '2376379': 2238, '2368666': 2239, '2402546': 2240, '1592156': 2241, '2388838': 2242, '2397307': 2243, '2348182': 2244, '2413071': 2245, '2404732': 2246, '2356427': 2247, '2407637': 2248, '2390263': 2249, '2405776': 2250, '2353337': 2251, '2316602': 2252, '2353270': 2253, '2408708': 2254, '2371198': 2255, '2354266': 2256, '2369855': 2257, '2380368': 2258, '2382039': 2259, '2358567': 2260, '2319011': 2261, '2370341': 2262, '2370551': 2263, '2362808': 2264, '2382330': 2265, '2408653': 2266, '2374473': 2267, '2362280': 2268, '2398037': 2269, '2375753': 2270, '2371017': 2271, '2370877': 2272, '2408209': 2273, '2333007': 2274, '2405738': 2275, '2391153': 2276, '2372453': 2277, '2358526': 2278, '2379303': 2279, '2389636': 2280, '2342857': 2281, '2415640': 2282, '2386870': 2283, '2363443': 2284, '2368634': 2285, '2413904': 2286, '2383358': 2287, '2380024': 2288, '2416559': 2289, '2336529': 2290, '2366500': 2291, '2351655': 2292, '2345044': 2293, '2373588': 2294, '2390099': 2295, '2373407': 2296, '2350763': 2297, '2402155': 2298, '2363548': 2299, '2322813': 2300, '2343805': 2301, '2391461': 2302, '2416656': 2303, '2402786': 2304, '2394339': 2305, '2414255': 2306, '2338443': 2307, '2382381': 2308, '2374927': 2309, '2362555': 2310, '2402202': 2311, '2382046': 2312, '2412163': 2313, '2354362': 2314, '2415082': 2315, '2408489': 2316, '2406617': 2317, '2319554': 2318, '2327849': 2319, '2378204': 2320, '2391047': 2321, '2341611': 2322, '2342864': 2323, '2386983': 2324, '2347702': 2325, '2322974': 2326, '2325414': 2327, '2400737': 2328, '2404002': 2329, '2392261': 2330, '2356269': 2331, '2358999': 2332, '2399577': 2333, '2370776': 2334, '2316535': 2335, '2395996': 2336, '2336048': 2337, '2376467': 2338, '2362203': 2339, '2415820': 2340, '2383182': 2341, '2350730': 2342, '2345893': 2343, '2375076': 2344, '2417944': 2345, '2337033': 2346, '2359636': 2347, '2408359': 2348, '2416307': 2349, '2398017': 2350, '2383168': 2351, '2330468': 2352, '2405056': 2353, '2321253': 2354, '2341034': 2355, '2317165': 2356, '2401897': 2357, '2398212': 2358, '2387721': 2359, '2344595': 2360, '2379759': 2361, '2393164': 2362, '2410345': 2363, '2363437': 2364, '2386462': 2365, '2388504': 2366, '2401336': 2367, '2321387': 2368, '2317132': 2369, '2396478': 2370, '2416725': 2371, '2372569': 2372, '2359537': 2373, '2322095': 2374, '2352174': 2375, '2343766': 2376, '2380329': 2377, '2361489': 2378, '2413702': 2379, '2321676': 2380, '2404393': 2381, '2356073': 2382, '2376970': 2383, '2358293': 2384, '2342802': 2385, '2402837': 2386, '2363661': 2387, '2403903': 2388, '2319419': 2389, '2344370': 2390, '2389353': 2391, '2397657': 2392, '2371546': 2393, '2347170': 2394, '2404791': 2395, '2384368': 2396, '2417585': 2397, '2392507': 2398, '2349036': 2399, '2329559': 2400, '879': 2401, '2414977': 2402, '2407808': 2403, '2378572': 2404, '2415884': 2405, '2414803': 2406, '2402120': 2407, '2320356': 2408, '2365699': 2409, '2354877': 2410, '2389638': 2411, '2334130': 2412, '2367765': 2413, '2379154': 2414, '2399862': 2415, '2404467': 2416, '2353974': 2417, '2352853': 2418, '2339578': 2419, '2406885': 2420, '2382775': 2421, '2354318': 2422, '2353172': 2423, '2412056': 2424, '2396290': 2425, '2336760': 2426, '2378276': 2427, '2379292': 2428, '2406213': 2429, '2416937': 2430, '2395164': 2431, '2356436': 2432, '2355527': 2433, '2386443': 2434, '2397502': 2435, '2410270': 2436, '2397071': 2437, '2415676': 2438, '2371756': 2439, '2394933': 2440, '2377521': 2441, '2320584': 2442, '2394242': 2443, '2385172': 2444, '2414806': 2445, '2380874': 2446, '2371037': 2447, '2317584': 2448, '2390476': 2449, '2356914': 2450, '2361991': 2451, '2350826': 2452, '2406588': 2453, '2358311': 2454, '2409964': 2455, '2403418': 2456, '2370296': 2457, '2411605': 2458, '2323547': 2459, '2316204': 2460, '2375241': 2461, '2349448': 2462, '2354115': 2463, '2404178': 2464, '2372285': 2465, '2401764': 2466, '2377897': 2467, '2328135': 2468, '2382879': 2469, '2360242': 2470, '2350037': 2471, '2361415': 2472, '2367100': 2473, '2411866': 2474, '2378223': 2475, '2346003': 2476, '2359944': 2477, '2386442': 2478, '2327810': 2479, '2345716': 2480, '2398747': 2481, '2327729': 2482, '2379103': 2483, '2330650': 2484, '2391165': 2485, '2403131': 2486}\n",
      "Before epoch loop->>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0711 14:32:38.357663 140399233443584 deprecation.py:323] From /home/student/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start and end (0, 5)\n",
      "idx:  0  Epoch:  2  loss:  270.91507  loss_sent:  2.7077875  loss_word:  257.37616  Time cost:  125.41162586212158\n",
      "Start and end (5, 10)\n",
      "idx:  5  Epoch:  2  loss:  260.9162  loss_sent:  2.1245575  loss_word:  250.2934  Time cost:  74.912433385849\n",
      "Start and end (10, 15)\n",
      "idx:  10  Epoch:  2  loss:  210.05959  loss_sent:  1.9769504  loss_word:  200.17484  Time cost:  9.07091999053955\n",
      "Start and end (15, 20)\n",
      "idx:  15  Epoch:  2  loss:  238.28853  loss_sent:  4.0089884  loss_word:  218.24356  Time cost:  7.139372110366821\n",
      "Start and end (20, 25)\n",
      "idx:  20  Epoch:  2  loss:  229.59947  loss_sent:  2.3953362  loss_word:  217.62276  Time cost:  7.055136680603027\n",
      "Start and end (25, 30)\n",
      "idx:  25  Epoch:  2  loss:  265.02457  loss_sent:  2.1991463  loss_word:  254.02881  Time cost:  6.837973356246948\n",
      "Start and end (30, 35)\n",
      "idx:  30  Epoch:  2  loss:  230.43161  loss_sent:  1.974234  loss_word:  220.56049  Time cost:  6.759197950363159\n",
      "Start and end (35, 40)\n",
      "idx:  35  Epoch:  2  loss:  165.44879  loss_sent:  2.042425  loss_word:  155.23666  Time cost:  6.651613473892212\n",
      "Start and end (40, 45)\n",
      "idx:  40  Epoch:  2  loss:  369.87973  loss_sent:  2.4401538  loss_word:  357.67892  Time cost:  6.804708957672119\n",
      "Start and end (45, 50)\n",
      "idx:  45  Epoch:  2  loss:  275.1883  loss_sent:  2.3319302  loss_word:  263.52863  Time cost:  6.884775638580322\n",
      "Start and end (50, 55)\n",
      "idx:  50  Epoch:  2  loss:  201.55807  loss_sent:  2.5269012  loss_word:  188.92355  Time cost:  6.864760398864746\n",
      "Start and end (55, 60)\n",
      "idx:  55  Epoch:  2  loss:  262.28568  loss_sent:  2.3461568  loss_word:  250.55492  Time cost:  6.756173133850098\n",
      "Start and end (60, 65)\n",
      "idx:  60  Epoch:  2  loss:  293.10965  loss_sent:  3.1350644  loss_word:  277.4343  Time cost:  7.457807779312134\n",
      "Start and end (65, 70)\n",
      "idx:  65  Epoch:  2  loss:  324.72385  loss_sent:  2.4800975  loss_word:  312.32333  Time cost:  7.288379430770874\n",
      "Start and end (70, 75)\n",
      "idx:  70  Epoch:  2  loss:  204.12665  loss_sent:  3.4216027  loss_word:  187.01863  Time cost:  6.740697860717773\n",
      "Start and end (75, 80)\n",
      "idx:  75  Epoch:  2  loss:  262.75208  loss_sent:  2.6297162  loss_word:  249.60344  Time cost:  6.910131454467773\n",
      "Start and end (80, 85)\n",
      "idx:  80  Epoch:  2  loss:  279.24994  loss_sent:  1.9725925  loss_word:  269.38696  Time cost:  6.7865941524505615\n",
      "Start and end (85, 90)\n",
      "idx:  85  Epoch:  2  loss:  211.12555  loss_sent:  2.6753373  loss_word:  197.74886  Time cost:  6.925106048583984\n",
      "Start and end (90, 95)\n",
      "idx:  90  Epoch:  2  loss:  213.6202  loss_sent:  2.356065  loss_word:  201.83986  Time cost:  6.790602207183838\n",
      "Start and end (95, 100)\n",
      "idx:  95  Epoch:  2  loss:  281.57227  loss_sent:  2.4189558  loss_word:  269.4775  Time cost:  7.006063461303711\n",
      "Start and end (100, 105)\n",
      "idx:  100  Epoch:  2  loss:  277.96332  loss_sent:  2.0732102  loss_word:  267.59723  Time cost:  6.787736654281616\n",
      "Start and end (105, 110)\n",
      "idx:  105  Epoch:  2  loss:  293.1745  loss_sent:  1.9708169  loss_word:  283.32043  Time cost:  6.757674694061279\n",
      "Start and end (110, 115)\n",
      "idx:  110  Epoch:  2  loss:  261.86133  loss_sent:  2.115596  loss_word:  251.28336  Time cost:  6.934680938720703\n",
      "Start and end (115, 120)\n",
      "idx:  115  Epoch:  2  loss:  217.26407  loss_sent:  2.0613515  loss_word:  206.9573  Time cost:  6.775691270828247\n",
      "Start and end (120, 125)\n",
      "idx:  120  Epoch:  2  loss:  299.87546  loss_sent:  2.3578012  loss_word:  288.08646  Time cost:  7.366941452026367\n",
      "Start and end (125, 130)\n",
      "idx:  125  Epoch:  2  loss:  218.7092  loss_sent:  2.0950472  loss_word:  208.23395  Time cost:  6.7726731300354\n",
      "Start and end (130, 135)\n",
      "idx:  130  Epoch:  2  loss:  220.22902  loss_sent:  2.6252246  loss_word:  207.10286  Time cost:  6.887665748596191\n",
      "Start and end (135, 140)\n",
      "idx:  135  Epoch:  2  loss:  279.60577  loss_sent:  3.6721234  loss_word:  261.24515  Time cost:  6.844378709793091\n",
      "Start and end (140, 145)\n",
      "idx:  140  Epoch:  2  loss:  195.15659  loss_sent:  1.8766975  loss_word:  185.77309  Time cost:  6.759380578994751\n",
      "Start and end (145, 150)\n",
      "idx:  145  Epoch:  2  loss:  290.1604  loss_sent:  2.2872834  loss_word:  278.72388  Time cost:  7.001491546630859\n",
      "Start and end (150, 155)\n",
      "idx:  150  Epoch:  2  loss:  302.49432  loss_sent:  2.7928658  loss_word:  288.52994  Time cost:  6.736677169799805\n",
      "Start and end (155, 160)\n",
      "idx:  155  Epoch:  2  loss:  282.67862  loss_sent:  2.039013  loss_word:  272.4836  Time cost:  6.736674785614014\n",
      "Start and end (160, 165)\n",
      "idx:  160  Epoch:  2  loss:  257.38916  loss_sent:  2.6057885  loss_word:  244.36024  Time cost:  6.772110939025879\n",
      "Start and end (165, 170)\n",
      "idx:  165  Epoch:  2  loss:  271.0192  loss_sent:  2.9572167  loss_word:  256.2331  Time cost:  6.698856592178345\n",
      "Start and end (170, 175)\n",
      "idx:  170  Epoch:  2  loss:  199.32368  loss_sent:  1.9578087  loss_word:  189.53465  Time cost:  6.611647129058838\n",
      "Start and end (175, 180)\n",
      "idx:  175  Epoch:  2  loss:  222.85982  loss_sent:  2.3984506  loss_word:  210.86755  Time cost:  6.732031583786011\n",
      "Start and end (180, 185)\n",
      "idx:  180  Epoch:  2  loss:  240.55687  loss_sent:  2.242345  loss_word:  229.34515  Time cost:  6.944058179855347\n",
      "Start and end (185, 190)\n",
      "idx:  185  Epoch:  2  loss:  219.12451  loss_sent:  1.8413223  loss_word:  209.91792  Time cost:  6.820937633514404\n",
      "Start and end (190, 195)\n",
      "idx:  190  Epoch:  2  loss:  252.46783  loss_sent:  2.5030608  loss_word:  239.95253  Time cost:  6.89994215965271\n",
      "Start and end (195, 200)\n",
      "idx:  195  Epoch:  2  loss:  262.70685  loss_sent:  2.5699904  loss_word:  249.85684  Time cost:  7.3958375453948975\n",
      "Start and end (200, 205)\n",
      "idx:  200  Epoch:  2  loss:  274.92505  loss_sent:  2.4799829  loss_word:  262.52515  Time cost:  6.928411960601807\n",
      "Start and end (205, 210)\n",
      "idx:  205  Epoch:  2  loss:  287.57306  loss_sent:  2.0204415  loss_word:  277.47083  Time cost:  7.24457859992981\n",
      "Start and end (210, 215)\n",
      "idx:  210  Epoch:  2  loss:  258.5246  loss_sent:  2.0143387  loss_word:  248.45287  Time cost:  6.710772752761841\n",
      "Start and end (215, 220)\n",
      "idx:  215  Epoch:  2  loss:  295.20404  loss_sent:  2.6186547  loss_word:  282.1108  Time cost:  6.795969247817993\n",
      "Start and end (220, 225)\n",
      "idx:  220  Epoch:  2  loss:  289.24673  loss_sent:  2.445057  loss_word:  277.02142  Time cost:  6.854633331298828\n",
      "Start and end (225, 230)\n",
      "idx:  225  Epoch:  2  loss:  324.47437  loss_sent:  1.8025062  loss_word:  315.46185  Time cost:  6.935649394989014\n",
      "Start and end (230, 235)\n",
      "idx:  230  Epoch:  2  loss:  216.65001  loss_sent:  2.5007765  loss_word:  204.14612  Time cost:  7.329835653305054\n",
      "Start and end (235, 240)\n",
      "idx:  235  Epoch:  2  loss:  267.74243  loss_sent:  2.4830341  loss_word:  255.32726  Time cost:  6.712348222732544\n",
      "Start and end (240, 245)\n",
      "idx:  240  Epoch:  2  loss:  189.44125  loss_sent:  2.9284537  loss_word:  174.79897  Time cost:  6.578287839889526\n",
      "Start and end (245, 250)\n",
      "idx:  245  Epoch:  2  loss:  282.06207  loss_sent:  2.5409431  loss_word:  269.35733  Time cost:  6.876291751861572\n",
      "Start and end (250, 255)\n",
      "idx:  250  Epoch:  2  loss:  253.01768  loss_sent:  2.472273  loss_word:  240.65633  Time cost:  6.758101463317871\n",
      "Start and end (255, 260)\n",
      "idx:  255  Epoch:  2  loss:  218.86998  loss_sent:  2.5858266  loss_word:  205.94084  Time cost:  6.721289873123169\n",
      "Start and end (260, 265)\n",
      "idx:  260  Epoch:  2  loss:  290.8097  loss_sent:  2.9411407  loss_word:  276.104  Time cost:  6.7690110206604\n",
      "Start and end (265, 270)\n",
      "idx:  265  Epoch:  2  loss:  240.42018  loss_sent:  2.1475563  loss_word:  229.6824  Time cost:  7.9444098472595215\n",
      "Start and end (270, 275)\n",
      "idx:  270  Epoch:  2  loss:  179.00401  loss_sent:  2.2209194  loss_word:  167.89941  Time cost:  6.954968452453613\n",
      "Start and end (275, 280)\n",
      "idx:  275  Epoch:  2  loss:  251.08249  loss_sent:  2.0674791  loss_word:  240.7451  Time cost:  6.850019931793213\n",
      "Start and end (280, 285)\n",
      "idx:  280  Epoch:  2  loss:  181.9957  loss_sent:  2.758024  loss_word:  168.2056  Time cost:  7.331959962844849\n",
      "Start and end (285, 290)\n",
      "idx:  285  Epoch:  2  loss:  307.5321  loss_sent:  2.200692  loss_word:  296.52863  Time cost:  6.889260530471802\n",
      "Start and end (290, 295)\n",
      "idx:  290  Epoch:  2  loss:  295.27597  loss_sent:  2.0672436  loss_word:  284.93973  Time cost:  7.397600173950195\n",
      "Start and end (295, 300)\n",
      "idx:  295  Epoch:  2  loss:  295.1821  loss_sent:  3.054058  loss_word:  279.91177  Time cost:  8.526495695114136\n",
      "Start and end (300, 305)\n",
      "idx:  300  Epoch:  2  loss:  192.49611  loss_sent:  2.122007  loss_word:  181.88608  Time cost:  7.982264995574951\n",
      "Start and end (305, 310)\n",
      "idx:  305  Epoch:  2  loss:  233.04507  loss_sent:  2.1367877  loss_word:  222.36113  Time cost:  7.281428813934326\n",
      "Start and end (310, 315)\n",
      "idx:  310  Epoch:  2  loss:  282.24802  loss_sent:  1.9742312  loss_word:  272.37686  Time cost:  7.221936464309692\n",
      "Start and end (315, 320)\n",
      "idx:  315  Epoch:  2  loss:  286.7492  loss_sent:  2.623114  loss_word:  273.63364  Time cost:  7.637181043624878\n",
      "Start and end (320, 325)\n",
      "idx:  320  Epoch:  2  loss:  292.78036  loss_sent:  2.4425263  loss_word:  280.56778  Time cost:  8.891954183578491\n",
      "Start and end (325, 330)\n",
      "idx:  325  Epoch:  2  loss:  331.17758  loss_sent:  2.288746  loss_word:  319.73386  Time cost:  9.128957986831665\n",
      "Start and end (330, 335)\n",
      "idx:  330  Epoch:  2  loss:  269.16858  loss_sent:  2.5682924  loss_word:  256.32706  Time cost:  6.9136643409729\n",
      "Start and end (335, 340)\n",
      "idx:  335  Epoch:  2  loss:  366.87985  loss_sent:  2.0227268  loss_word:  356.7662  Time cost:  6.853558778762817\n",
      "Start and end (340, 345)\n",
      "idx:  340  Epoch:  2  loss:  206.47801  loss_sent:  2.3194528  loss_word:  194.88074  Time cost:  7.533937931060791\n",
      "Start and end (345, 350)\n",
      "idx:  345  Epoch:  2  loss:  201.95624  loss_sent:  2.0301807  loss_word:  191.80531  Time cost:  6.979682922363281\n",
      "Start and end (350, 355)\n",
      "idx:  350  Epoch:  2  loss:  199.32841  loss_sent:  1.839539  loss_word:  190.13072  Time cost:  7.281743288040161\n",
      "Start and end (355, 360)\n",
      "idx:  355  Epoch:  2  loss:  249.74666  loss_sent:  2.4513505  loss_word:  237.48988  Time cost:  7.256957054138184\n",
      "Start and end (360, 365)\n",
      "idx:  360  Epoch:  2  loss:  290.0328  loss_sent:  2.9343145  loss_word:  275.36127  Time cost:  7.043760776519775\n",
      "Start and end (365, 370)\n",
      "idx:  365  Epoch:  2  loss:  273.0782  loss_sent:  2.3062131  loss_word:  261.54712  Time cost:  7.877694129943848\n",
      "Start and end (370, 375)\n",
      "idx:  370  Epoch:  2  loss:  256.64966  loss_sent:  2.4821293  loss_word:  244.23904  Time cost:  7.2458672523498535\n",
      "Start and end (375, 380)\n",
      "idx:  375  Epoch:  2  loss:  249.76158  loss_sent:  1.8418806  loss_word:  240.5522  Time cost:  7.477684497833252\n",
      "Start and end (380, 385)\n",
      "idx:  380  Epoch:  2  loss:  215.84283  loss_sent:  2.5800502  loss_word:  202.94257  Time cost:  8.00604510307312\n",
      "Start and end (385, 390)\n",
      "idx:  385  Epoch:  2  loss:  237.03741  loss_sent:  2.2595644  loss_word:  225.73961  Time cost:  8.397264957427979\n",
      "Start and end (390, 395)\n",
      "idx:  390  Epoch:  2  loss:  270.8482  loss_sent:  2.1159284  loss_word:  260.26852  Time cost:  7.151131868362427\n",
      "Start and end (395, 400)\n",
      "idx:  395  Epoch:  2  loss:  197.18071  loss_sent:  2.2020848  loss_word:  186.17029  Time cost:  7.629374742507935\n",
      "Start and end (400, 405)\n",
      "idx:  400  Epoch:  2  loss:  285.53232  loss_sent:  2.6682653  loss_word:  272.19095  Time cost:  7.246536731719971\n",
      "Start and end (405, 410)\n",
      "idx:  405  Epoch:  2  loss:  224.78348  loss_sent:  2.383237  loss_word:  212.86728  Time cost:  7.09847617149353\n",
      "Start and end (410, 415)\n",
      "idx:  410  Epoch:  2  loss:  287.76755  loss_sent:  2.671877  loss_word:  274.40817  Time cost:  8.671973705291748\n",
      "Start and end (415, 420)\n",
      "idx:  415  Epoch:  2  loss:  189.94202  loss_sent:  2.0027242  loss_word:  179.9284  Time cost:  7.021343469619751\n",
      "Start and end (420, 425)\n",
      "idx:  420  Epoch:  2  loss:  243.92586  loss_sent:  2.2841854  loss_word:  232.50491  Time cost:  7.856757640838623\n",
      "Start and end (425, 430)\n",
      "idx:  425  Epoch:  2  loss:  235.92844  loss_sent:  1.9302204  loss_word:  226.27734  Time cost:  7.186469316482544\n",
      "Start and end (430, 435)\n",
      "idx:  430  Epoch:  2  loss:  204.129  loss_sent:  1.9878284  loss_word:  194.18988  Time cost:  6.835457801818848\n",
      "Start and end (435, 440)\n",
      "idx:  435  Epoch:  2  loss:  319.2085  loss_sent:  2.0846453  loss_word:  308.78528  Time cost:  6.743842363357544\n",
      "Start and end (440, 445)\n",
      "idx:  440  Epoch:  2  loss:  296.12698  loss_sent:  1.9382187  loss_word:  286.43582  Time cost:  6.921665191650391\n",
      "Start and end (445, 450)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ddcb88721579>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    671\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Time cost: \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstart_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 673\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-1-ddcb88721579>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    553\u001b[0m                                            \u001b[0mtf_num_distribution\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcurrent_num_distribution\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    554\u001b[0m                                            \u001b[0mtf_captions_matrix\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcurrent_captions_matrix\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 555\u001b[0;31m                                            \u001b[0mtf_captions_masks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcurrent_captions_masks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    556\u001b[0m                                 })\n\u001b[1;32m    557\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1339\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1427\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1431\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding=utf-8\n",
    "\n",
    "__author__ = \"Xinpeng.Chen\"\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "\n",
    "import h5py\n",
    "import ipdb\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------------------------------------------\n",
    "# Initialization class\n",
    "#  1. Pooling the visual features into a single dense feature\n",
    "#  2. Then, build sentence LSTM, word LSTM\n",
    "# ------------------------------------------------------------------------------------------------------\n",
    "class RegionPooling_HierarchicalRNN():\n",
    "    def __init__(self, n_words,\n",
    "                       batch_size,\n",
    "                       num_boxes,\n",
    "                       feats_dim,\n",
    "                       project_dim,\n",
    "                       sentRNN_lstm_dim,\n",
    "                       sentRNN_FC_dim,\n",
    "                       wordRNN_lstm_dim,\n",
    "                       S_max,\n",
    "                       N_max,\n",
    "                       word_embed_dim,\n",
    "                       bias_init_vector=None):\n",
    "\n",
    "        self.n_words = n_words\n",
    "        self.batch_size = batch_size\n",
    "        self.num_boxes = num_boxes # 50\n",
    "        self.feats_dim = feats_dim # 4096\n",
    "        self.project_dim = project_dim # 1024\n",
    "        self.S_max = S_max # 6\n",
    "        self.N_max = N_max # 50\n",
    "        self.word_embed_dim = word_embed_dim # 1024\n",
    "\n",
    "        self.sentRNN_lstm_dim = sentRNN_lstm_dim # 512 hidden size\n",
    "        self.sentRNN_FC_dim = sentRNN_FC_dim # 1024 in fully connected layer\n",
    "        self.wordRNN_lstm_dim = wordRNN_lstm_dim # 512 hidden size\n",
    "\t\n",
    "\n",
    "\n",
    "        # word embedding, parameters of embedding\n",
    "        # embedding shape: n_words x wordRNN_lstm_dim\n",
    "        with tf.device('/cpu:0'):\n",
    "            self.Wemb = tf.Variable(tf.random_uniform([n_words, word_embed_dim], -0.1, 0.1), name='Wemb')\n",
    "        print(\"Wemb output>>>>>>>>>>>>\",self.Wemb,tf.shape(self.Wemb))\n",
    "        #self.bemb = tf.Variable(tf.zeros([word_embed_dim]), name='bemb')\n",
    "\n",
    "        # regionPooling_W shape: 4096 x 1024\n",
    "        # regionPooling_b shape: 1024\n",
    "        self.regionPooling_W = tf.Variable(tf.random_uniform([feats_dim, project_dim], -0.1, 0.1), name='regionPooling_W')\n",
    "        self.regionPooling_b = tf.Variable(tf.zeros([project_dim]), name='regionPooling_b')\n",
    "\n",
    "        # sentence LSTM\n",
    "        self.sent_LSTM = tf.nn.rnn_cell.BasicLSTMCell(sentRNN_lstm_dim, state_is_tuple=True)\n",
    "\n",
    "        # logistic classifier\n",
    "        self.logistic_Theta_W = tf.Variable(tf.random_uniform([sentRNN_lstm_dim, 2], -0.1, 0.1), name='logistic_Theta_W')\n",
    "        self.logistic_Theta_b = tf.Variable(tf.zeros(2), name='logistic_Theta_b')\n",
    "\n",
    "        # fc1_W: 512 x 1024, fc1_b: 1024\n",
    "        # fc2_W: 1024 x 1024, fc2_b: 1024\n",
    "        self.fc1_W = tf.Variable(tf.random_uniform([sentRNN_lstm_dim, sentRNN_FC_dim], -0.1, 0.1), name='fc1_W')\n",
    "        self.fc1_b = tf.Variable(tf.zeros(sentRNN_FC_dim), name='fc1_b')\n",
    "        self.fc2_W = tf.Variable(tf.random_uniform([sentRNN_FC_dim, 1024], -0.1, 0.1), name='fc2_W')\n",
    "        self.fc2_b = tf.Variable(tf.zeros(1024), name='fc2_b')\n",
    "        def get_a_cell(lstm_size):\n",
    "          lstm = tf.nn.rnn_cell.BasicLSTMCell(lstm_size)\n",
    "          return lstm\n",
    "        # word LSTM\n",
    "        self.word_LSTM = tf.nn.rnn_cell.BasicLSTMCell(wordRNN_lstm_dim, state_is_tuple=True)\n",
    "        #self.word_LSTM = tf.nn.rnn_cell.MultiRNNCell([self.word_LSTM] * 2, state_is_tuple=True)\n",
    "        self.word_LSTM = tf.nn.rnn_cell.MultiRNNCell([get_a_cell(wordRNN_lstm_dim) for i in range(2)], state_is_tuple=True)\n",
    "        #self.word_LSTM2 = tf.nn.rnn_cell.BasicLSTMCell(wordRNN_lstm_dim, state_is_tuple=True)\n",
    "\n",
    "\n",
    "        self.embed_word_W = tf.Variable(tf.random_uniform([wordRNN_lstm_dim, n_words], -0.1,0.1), name='embed_word_W')\n",
    "        if bias_init_vector is not None:\n",
    "            self.embed_word_b = tf.Variable(bias_init_vector.astype(np.float32), name='embed_word_b')\n",
    "        else:\n",
    "            self.embed_word_b = tf.Variable(tf.zeros([n_words]), name='embed_word_b')\n",
    "\n",
    "    def build_model(self):\n",
    "        # receive the feats in the current image\n",
    "        # it's shape is 10 x 50 x 4096\n",
    "        # tmp_feats: 500 x 4096\n",
    "        feats = tf.placeholder(tf.float32, [self.batch_size, self.num_boxes, self.feats_dim])\n",
    "        tmp_feats = tf.reshape(feats, [-1, self.feats_dim])\n",
    "\n",
    "        # project_vec_all: 500 x 4096 * 4096 x 1024 --> 500 x 1024\n",
    "        # project_vec: 10 x 1024\n",
    "        project_vec_all = tf.matmul(tmp_feats, self.regionPooling_W) + self.regionPooling_b\n",
    "        project_vec_all = tf.reshape(project_vec_all, [self.batch_size, 50, self.project_dim])\n",
    "        project_vec = tf.reduce_max(project_vec_all, reduction_indices=1)\n",
    "\n",
    "        # receive the [continue:0, stop:1] lists\n",
    "        # example: [0, 0, 0, 0, 1, 1], it means this paragraph has five sentences\n",
    "        num_distribution = tf.placeholder(tf.int32, [self.batch_size, self.S_max])\n",
    "\n",
    "        # receive the ground truth words, which has been changed to idx use word2idx function\n",
    "        captions = tf.placeholder(tf.int32, [self.batch_size, self.S_max, self.N_max+1])\n",
    "        #print(\"Captions:>>>>>>>>>>>>>>>>>>>>>>>\",captions)\n",
    "        captions_masks = tf.placeholder(tf.float32, [self.batch_size, self.S_max, self.N_max+1])\n",
    "\n",
    "        # ---------------------------------------------------------------------------------------------------------------------\n",
    "        # The method which initialize the state, is refered from below sites:\n",
    "        # 1. http://stackoverflow.com/questions/38241410/tensorflow-remember-lstm-state-for-next-batch-stateful-lstm/38417699\n",
    "        # 2. https://www.tensorflow.org/api_docs/python/rnn_cell/classes_storing_split_rnncell_state#LSTMStateTuple\n",
    "        # 3. https://medium.com/@erikhallstrm/using-the-tensorflow-lstm-api-3-7-5f2b97ca6b73#.u4w9z6h0h\n",
    "        # ---------------------------------------------------------------------------------------------------------------------\n",
    "        sent_state = self.sent_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state = self.word_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state1 = self.word_LSTM1.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state2 = self.word_LSTM2.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #sent_state = tf.zeros([self.batch_size, self.sent_LSTM1.state_size])\n",
    "        #word_state1 = tf.zeros([self.batch_size, self.word_LSTM1.state_size])\n",
    "        #word_state2 = tf.zeros([self.batch_size, self.word_LSTM2.state_size])\n",
    "\n",
    "        probs = []\n",
    "        loss = 0.0\n",
    "        loss_sent = 0.0\n",
    "        loss_word = 0.0\n",
    "        lambda_sent = 5.0\n",
    "        lambda_word = 1.0\n",
    "\n",
    "        print('Start build model:')\n",
    "        #----------------------------------------------------------------------------------------------\n",
    "        # Hierarchical RNN: sentence RNN and words RNN\n",
    "        # The word RNN has the max number, N_max = 50, the number in the papar is 50\n",
    "        #----------------------------------------------------------------------------------------------\n",
    "        for i in range(0, self.S_max):\n",
    "            if i > 0:\n",
    "                tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.variable_scope('sent_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                sent_output, sent_state = self.sent_LSTM(project_vec, sent_state)\n",
    "\n",
    "            with tf.name_scope('fc1'):\n",
    "                hidden1 = tf.nn.relu( tf.matmul(sent_output, self.fc1_W) + self.fc1_b )\n",
    "            with tf.name_scope('fc2'):\n",
    "                sent_topic_vec = tf.nn.relu( tf.matmul(hidden1, self.fc2_W) + self.fc2_b )\n",
    "\n",
    "            # sent_state is a tuple, sent_state = (c, h)\n",
    "            # 'c': shape=(1, 512) dtype=float32, 'h': shape=(1, 512) dtype=float32\n",
    "            # The loss here, I refer from the web which is very helpful for me:\n",
    "            # 1. http://stackoverflow.com/questions/34240703/difference-between-tensorflow-tf-nn-softmax-and-tf-nn-softmax-cross-entropy-with\n",
    "            # 2. http://stackoverflow.com/questions/35277898/tensorflow-for-binary-classification\n",
    "            # 3. http://stackoverflow.com/questions/35226198/is-this-one-hot-encoding-in-tensorflow-fast-or-flawed-for-any-reason\n",
    "            # 4. http://stackoverflow.com/questions/35198528/reshape-y-train-for-binary-text-classification-in-tensorflow\n",
    "            sentRNN_logistic_mu = tf.nn.xw_plus_b( sent_output, self.logistic_Theta_W, self.logistic_Theta_b )\n",
    "            sentRNN_label = tf.stack([ 1 - num_distribution[:, i], num_distribution[:, i] ])\n",
    "            sentRNN_label = tf.transpose(sentRNN_label)\n",
    "            sentRNN_loss = tf.nn.softmax_cross_entropy_with_logits(logits=sentRNN_logistic_mu, labels=sentRNN_label)\n",
    "            sentRNN_loss = tf.reduce_sum(sentRNN_loss)/self.batch_size\n",
    "            loss += sentRNN_loss * lambda_sent\n",
    "            loss_sent += sentRNN_loss\n",
    "\n",
    "            # the begining input of word_LSTM is topic vector, and DON'T compute the loss\n",
    "            # This is follow the paper: Show and Tell\n",
    "            #word_state = self.word_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "            #with tf.variable_scope('word_LSTM'):\n",
    "            #    word_output, word_state = self.word_LSTM(sent_topic_vec)\n",
    "            topic = tf.nn.rnn_cell.LSTMStateTuple(sent_topic_vec[:, 0:512], sent_topic_vec[:, 512:])\n",
    "            word_state = (topic, topic)\n",
    "            for j in range(0, self.N_max):\n",
    "                if j > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                with tf.device('/cpu:0'):\n",
    "                    current_embed = tf.nn.embedding_lookup(self.Wemb, captions[:, i, j])\n",
    "\n",
    "                with tf.variable_scope('word_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                    word_output, word_state = self.word_LSTM(current_embed, word_state)\n",
    "\n",
    "                # How to make one-hot encoder, I refer from this excellent web:\n",
    "                # http://stackoverflow.com/questions/33681517/tensorflow-one-hot-encoder\n",
    "                labels = tf.reshape(captions[:, i, j+1], [-1, 1])\n",
    "                #print(\"Labels and its shape +++++++++++++++\",labels,tf.shape(labels))\n",
    "                indices = tf.reshape(tf.range(0, self.batch_size, 1), [-1, 1])\n",
    "                #print(\"Indices and its shape +++++++++++++++\",indices,tf.shape(indices))\n",
    "                concated = tf.concat([indices, labels],1)\n",
    "                #print(\"Concated+++++++++++++++++++\",concated)\n",
    "                print(\"Success\")\n",
    "                onehot_labels = tf.sparse_to_dense(concated, tf.stack([self.batch_size, self.n_words]), 1.0, 0.0)\n",
    "\n",
    "                # At each timestep the hidden state of the last LSTM layer is used to predict a distribution\n",
    "                # over the words in the vocbulary\n",
    "                logit_words = tf.nn.xw_plus_b(word_output[:], self.embed_word_W, self.embed_word_b)\n",
    "                cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logit_words, labels=onehot_labels)\n",
    "                cross_entropy = cross_entropy * captions_masks[:, i, j]\n",
    "                loss_wordRNN = tf.reduce_sum(cross_entropy) / self.batch_size\n",
    "                loss += loss_wordRNN * lambda_word\n",
    "                loss_word += loss_wordRNN\n",
    "\n",
    "        return feats, num_distribution, captions, captions_masks, loss, loss_sent, loss_word\n",
    "\n",
    "    def generate_model(self):\n",
    "        # feats: 1 x 50 x 4096\n",
    "        feats = tf.placeholder(tf.float32, [1, self.num_boxes, self.feats_dim])\n",
    "        # tmp_feats: 50 x 4096\n",
    "        tmp_feats = tf.reshape(feats, [-1, self.feats_dim])\n",
    "\n",
    "        # project_vec_all: 50 x 4096 * 4096 x 1024 + 1024 --> 50 x 1024\n",
    "        project_vec_all = tf.matmul(tmp_feats, self.regionPooling_W) + self.regionPooling_b\n",
    "        project_vec_all = tf.reshape(project_vec_all, [1, 50, self.project_dim])\n",
    "        project_vec = tf.reduce_max(project_vec_all, reduction_indices=1)\n",
    "\n",
    "        # initialize the sent_LSTM state\n",
    "        sent_state = self.sent_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "\n",
    "        # save the generated paragraph to list, here I named generated_sents\n",
    "        generated_paragraph = []\n",
    "\n",
    "        # pred\n",
    "        pred_re = []\n",
    "\n",
    "        # T_stop: run the sentence RNN forward until the stopping probability p_i (STOP) exceeds a threshold T_stop\n",
    "        T_stop = tf.constant(0.5)\n",
    "\n",
    "        # Start build the generation model\n",
    "        print('Start build the generation model: ')\n",
    "\n",
    "        # sentence RNN\n",
    "        #word_state = self.word_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "        #with tf.variable_scope('word_LSTM'):\n",
    "        #    word_output, word_state = self.word_LSTM(sent_topic_vec, word_state)\n",
    "        for i in range(0, self.S_max):\n",
    "            if i > 0:\n",
    "                tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            # sent_state:\n",
    "            # LSTMStateTuple(c=<tf.Tensor 'sent_LSTM/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "            #                h=<tf.Tensor 'sent_LSTM/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>)\n",
    "            with tf.variable_scope('sent_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                sent_output, sent_state = self.sent_LSTM(project_vec, sent_state)\n",
    "\n",
    "            # self.fc1_W: 512 x 1024, self.fc1_b: 1024\n",
    "            # hidden1: 1 x 1024\n",
    "            # sent_topic_vec: 1 x 1024\n",
    "            with tf.name_scope('fc1'):\n",
    "                hidden1 = tf.nn.relu( tf.matmul(sent_output, self.fc1_W) + self.fc1_b )\n",
    "            with tf.name_scope('fc2'):\n",
    "                sent_topic_vec = tf.nn.relu( tf.matmul(hidden1, self.fc2_W) + self.fc2_b )\n",
    "\n",
    "            sentRNN_logistic_mu = tf.nn.xw_plus_b(sent_output, self.logistic_Theta_W, self.logistic_Theta_b)\n",
    "            pred = tf.nn.softmax(sentRNN_logistic_mu)\n",
    "            pred_re.append(pred)\n",
    "\n",
    "            # save the generated sentence to list, named generated_sent\n",
    "            generated_sent = []\n",
    "\n",
    "            # initialize the word LSTM state\n",
    "            #word_state = self.word_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "            #with tf.variable_scope('word_LSTM'):\n",
    "            #    word_output, word_state = self.word_LSTM(sent_topic_vec, word_state)\n",
    "            topic = tf.nn.rnn_cell.LSTMStateTuple(sent_topic_vec[:, 0:512], sent_topic_vec[:, 512:])\n",
    "            word_state = (topic, topic)\n",
    "            # word RNN, unrolled to N_max time steps\n",
    "            for j in range(0, self.N_max):\n",
    "                if j > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                if j == 0:\n",
    "                    with tf.device('/cpu:0'):\n",
    "                        # get word embedding of BOS (index = 0)\n",
    "                        current_embed = tf.nn.embedding_lookup(self.Wemb, tf.zeros([1], dtype=tf.int64))\n",
    "\n",
    "                with tf.variable_scope('word_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                    word_output, word_state = self.word_LSTM(current_embed, word_state)\n",
    "\n",
    "                # word_state:\n",
    "                # (\n",
    "                #     LSTMStateTuple(c=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell0/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "                #                    h=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell0/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>),\n",
    "                #     LSTMStateTuple(c=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell1/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "                #                    h=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell1/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>)\n",
    "                # )\n",
    "                logit_words = tf.nn.xw_plus_b(word_output, self.embed_word_W, self.embed_word_b)\n",
    "                max_prob_index = tf.argmax(logit_words, 1)[0]\n",
    "                generated_sent.append(max_prob_index)\n",
    "\n",
    "                with tf.device('/cpu:0'):\n",
    "                    current_embed = tf.nn.embedding_lookup(self.Wemb, max_prob_index)\n",
    "                    current_embed = tf.expand_dims(current_embed, 0)\n",
    "\n",
    "            generated_paragraph.append(generated_sent)\n",
    "\n",
    "        return feats, generated_paragraph, pred_re, sent_topic_vec\n",
    "\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "# Preparing Functions\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "def preProBuildWordVocab(sentence_iterator, word_count_threshold=5):\n",
    "    # borrowed this function from NeuralTalk\n",
    "    print('preprocessing word counts and creating vocab based on word count threshold %d' % (word_count_threshold, ))\n",
    "\n",
    "    word_counts = {}\n",
    "    nsents = 0\n",
    "\n",
    "    for sent in sentence_iterator:\n",
    "        nsents += 1\n",
    "        tmp_sent = sent.lower().split(' ')\n",
    "        if '' in tmp_sent:\n",
    "            tmp_sent.remove('')\n",
    "\n",
    "        for w in tmp_sent:\n",
    "           word_counts[w] = word_counts.get(w, 0) + 1\n",
    "\n",
    "    vocab = [w for w in word_counts if word_counts[w] >= word_count_threshold]\n",
    "    print('filtered words from %d to %d' % (len(word_counts), len(vocab)))\n",
    "\n",
    "    ixtoword = {}\n",
    "    ixtoword[0] = '<bos>'\n",
    "    ixtoword[1] = '<eos>'\n",
    "    ixtoword[2] = '<pad>'\n",
    "    ixtoword[3] = '<unk>'\n",
    "\n",
    "    wordtoix = {}\n",
    "    wordtoix['<bos>'] = 0\n",
    "    wordtoix['<eos>'] = 1\n",
    "    wordtoix['<pad>'] = 2\n",
    "    wordtoix['<unk>'] = 3\n",
    "\n",
    "    for idx, w in enumerate(vocab):\n",
    "        wordtoix[w] = idx + 4\n",
    "        ixtoword[idx+4] = w\n",
    "\n",
    "    word_counts['<eos>'] = nsents\n",
    "    word_counts['<bos>'] = nsents\n",
    "    word_counts['<pad>'] = nsents\n",
    "    word_counts['<unk>'] = nsents\n",
    "\n",
    "    bias_init_vector = np.array([1.0 * word_counts[ ixtoword[i] ] for i in ixtoword])\n",
    "    bias_init_vector /= np.sum(bias_init_vector) # normalize to frequencies\n",
    "    bias_init_vector = np.log(bias_init_vector)\n",
    "    bias_init_vector -= np.max(bias_init_vector) # shift to nice numeric range\n",
    "\n",
    "    return wordtoix, ixtoword, bias_init_vector\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Parameters Setting\n",
    "#######################################################################################################\n",
    "batch_size = 5 # Being support batch_size\n",
    "num_boxes = 50 # number of Detected regions in each image\n",
    "feats_dim = 4096 # feature dimensions of each regions\n",
    "project_dim = 1024 # project the features to one vector, which is 1024 dimensions\n",
    "\n",
    "sentRNN_lstm_dim = 512 # the sentence LSTM hidden units\n",
    "sentRNN_FC_dim = 1024 # the fully connected units\n",
    "wordRNN_lstm_dim = 512 # the word LSTM hidden units\n",
    "word_embed_dim = 1024 # the learned embedding vectors for the words\n",
    "\n",
    "S_max = 6\n",
    "N_max = 50\n",
    "T_stop = 0.5\n",
    "\n",
    "n_epochs = 500\n",
    "learning_rate = 0.0001\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Word vocubulary and captions preprocessing stage\n",
    "#######################################################################################################\n",
    "img2paragraph = pickle.load(open('./img2paragraph', 'rb'))\n",
    "all_sentences = []\n",
    "for key, paragraph in img2paragraph.items():\n",
    "    for each_sent in paragraph[1]:\n",
    "        each_sent.replace(',', ' ,')\n",
    "        all_sentences.append(each_sent)\n",
    "word2idx, idx2word, bias_init_vector = preProBuildWordVocab(all_sentences, word_count_threshold=2)\n",
    "np.save('./idx2word_batch', idx2word)\n",
    "\n",
    "img2paragraph_modify = {}\n",
    "for img_name, img_paragraph in img2paragraph.items():\n",
    "    img_paragraph_1 = img_paragraph[1]\n",
    "\n",
    "    # img_paragraph_1 is a list\n",
    "    # it may contain the element: '' or ' ', like this:\n",
    "    # [[\"a man is walking\"], [\"the dog is running\"], [\"\"], [\" \"]]\n",
    "    # so, we should remove them ' ' and '' element\n",
    "    if '' in img_paragraph_1:\n",
    "        img_paragraph_1.remove('')\n",
    "    if ' ' in paragraph[1]:\n",
    "        img_paragraph_1.remove(' ')\n",
    "\n",
    "    # the number sents in each paragraph\n",
    "    # if the sents is bigger than S_max,\n",
    "    # we force the number of sents to be S_max\n",
    "    img_num_sents = len(img_paragraph_1)\n",
    "    if img_num_sents > S_max:\n",
    "        img_num_sents = S_max\n",
    "\n",
    "    # if a paragraph has 4 sentences\n",
    "    # then the img_num_distribution will be like this:\n",
    "    # [0, 0, 0, 1, 1, 1]\n",
    "    img_num_distribution = np.zeros([S_max], dtype=np.int32)\n",
    "    img_num_distribution[img_num_sents-1:] = 1\n",
    "\n",
    "    # we multiply the number 2, because the <pad> is encoded into 2\n",
    "    img_captions_matrix = np.ones([S_max, N_max+1], dtype=np.int32) * 2 # zeros([6, 50])\n",
    "    for idx, img_sent in enumerate(img_paragraph_1):\n",
    "        # the number of sentences is img_num_sents\n",
    "        if idx == img_num_sents:\n",
    "            break\n",
    "\n",
    "        # because we treat the ',' as a word\n",
    "        img_sent = img_sent.replace(',', ' ,')\n",
    "\n",
    "        # Because I have preprocess the paragraph_v1.json file in VScode before,\n",
    "        # and I delete all the 2, 3, 4...bankspaces\n",
    "        # so, actually, the 'elif' code will never run\n",
    "        if img_sent[0] == ' ' and img_sent[1] != ' ':\n",
    "            img_sent = img_sent[1:]\n",
    "        elif img_sent[0] == ' ' and img_sent[1] == ' ' and img_sent[2] != ' ':\n",
    "            img_sent = img_sent[2:]\n",
    "\n",
    "        # Be careful the last part in a sentence, like this:\n",
    "        # '...world.'\n",
    "        # '...world. '\n",
    "        if img_sent[-1] == '.':\n",
    "            img_sent = img_sent[0:-1]\n",
    "        elif img_sent[-1] == ' ' and img_sent[-2] == '.':\n",
    "            img_sent = img_sent[0:-2]\n",
    "\n",
    "        # Last, we add the <bos> and the <eos> in each sentences\n",
    "        img_sent = '<bos> ' + img_sent + ' <eos>'\n",
    "\n",
    "        # translate each word in a sentence into the unique number in word2idx dict\n",
    "        # when we meet the word which is not in the word2idx dict, we use the mark: <unk>\n",
    "        for idy, word in enumerate(img_sent.lower().split(' ')):\n",
    "            # because the biggest number of words in a sentence is N_max, here is 50\n",
    "            if idy == N_max:\n",
    "                break\n",
    "\n",
    "            if word in word2idx:\n",
    "                img_captions_matrix[idx, idy] = word2idx[word]\n",
    "            else:\n",
    "                img_captions_matrix[idx, idy] = word2idx['<unk>']\n",
    "\n",
    "    # Pay attention, the value type 'img_name' here is NUMBER, I change it to STRING type\n",
    "    img2paragraph_modify[str(img_name)] = [img_num_distribution, img_captions_matrix]\n",
    "\n",
    "with open('./img2paragraph_modify_batch', 'wb') as f:\n",
    "    pickle.dump(img2paragraph_modify, f)\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Train, validation and testing stage\n",
    "#######################################################################################################\n",
    "def train():\n",
    "    ##############################################################################\n",
    "    # some preparing work\n",
    "    ##############################################################################\n",
    "    model_path = './models_batch/'\n",
    "    train_feats_path = './im2p_val_output.h5'\n",
    "    train_output_file = h5py.File(train_feats_path, 'r')\n",
    "    train_feats = train_output_file.get('feats')\n",
    "    train_imgs_full_path_lists = open('./imgs_val_path.txt').read().splitlines()\n",
    "    train_imgs_names = list(map(lambda x: os.path.basename(x).split('.')[0], train_imgs_full_path_lists))\n",
    "    print(\"Train Images Names\",train_imgs_names)\n",
    "\n",
    "    # Model Initialization:\n",
    "    # n_words, batch_size, num_boxes, feats_dim, project_dim, sentRNN_lstm_dim, sentRNN_FC_dim, wordRNN_lstm_dim, S_max, N_max\n",
    "    model = RegionPooling_HierarchicalRNN(n_words = len(word2idx),\n",
    "                                          batch_size = batch_size,\n",
    "                                          num_boxes = num_boxes,\n",
    "                                          feats_dim = feats_dim,\n",
    "                                          project_dim = project_dim,\n",
    "                                          sentRNN_lstm_dim = sentRNN_lstm_dim,\n",
    "                                          sentRNN_FC_dim = sentRNN_FC_dim,\n",
    "                                          wordRNN_lstm_dim = wordRNN_lstm_dim,\n",
    "                                          S_max = S_max,\n",
    "                                          N_max = N_max,\n",
    "                                          word_embed_dim = word_embed_dim,\n",
    "                                          bias_init_vector = bias_init_vector)\n",
    "\n",
    "    tf_feats, tf_num_distribution, tf_captions_matrix, tf_captions_masks, tf_loss, tf_loss_sent, tf_loss_word = model.build_model()\n",
    "    sess = tf.Session()\n",
    "\n",
    "    saver = tf.train.Saver(max_to_keep=500, write_version=1)\n",
    "    with tf.variable_scope('optimizer',reuse= tf.AUTO_REUSE):\n",
    "        train_op = tf.train.AdamOptimizer(learning_rate).minimize(tf_loss)\n",
    "    tf.global_variables_initializer().run(session=sess)\n",
    "    print(\"Done w session\")\n",
    "\n",
    "    # when you want to train the model from the front model\n",
    "    #new_saver = tf.train.Saver(max_to_keep=500)\n",
    "    #new_saver = tf.train.import_meta_graph('./models_batch/model-92.meta')\n",
    "    #new_saver.restore(sess, tf.train.latest_checkpoint('./models_batch/'))\n",
    "\n",
    "    all_vars = tf.trainable_variables()\n",
    "\n",
    "    # open a loss file to record the loss value\n",
    "    #loss_fd = open('loss_batch.txt', 'a')\n",
    "    img2idx = {}\n",
    "    for idx, img in enumerate(train_imgs_names):\n",
    "        img2idx[img] = idx\n",
    "    print(\"Img2idx\",img2idx)\n",
    "\n",
    "    # plt draw the loss curve\n",
    "    # refer from: http://stackoverflow.com/questions/11874767/real-time-plotting-in-while-loop-with-matplotlib\n",
    "    loss_to_draw = []\n",
    "    print(\"Before epoch loop->>>>>>>>>>>>>>>>>>>>>>>>>>>>>\")\n",
    "    saver = tf.train.import_meta_graph('./models_batch/model-1.meta')\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('./models_batch/'))\n",
    "    for epoch in range(2, n_epochs):\n",
    "        loss_to_draw_epoch = []\n",
    "        # disorganize the order\n",
    "        random.shuffle(train_imgs_names)\n",
    "\n",
    "        for start, end in zip(range(0, len(train_imgs_names), batch_size),\n",
    "                              range(batch_size, len(train_imgs_names), batch_size)):\n",
    "            loss_fd = open('loss_batch.txt', 'a')\n",
    "\n",
    "            start_time = time.time()\n",
    "            print(\"Start and end\",(start,end))\n",
    "            img_name = train_imgs_names[start:end]\n",
    "            current_feats_index = map(lambda x: img2idx[x], img_name)\n",
    "            current_feats = np.asarray( list(map(lambda x: train_feats[x], current_feats_index) ))\n",
    "\n",
    "            current_num_distribution = np.asarray( list(map(lambda x: img2paragraph_modify[x][0], img_name) ))\n",
    "            current_captions_matrix = np.asarray( list(map(lambda x: img2paragraph_modify[x][1], img_name) ))\n",
    "\n",
    "            current_captions_masks = np.zeros( (current_captions_matrix.shape[0], current_captions_matrix.shape[1], current_captions_matrix.shape[2]) )\n",
    "            # find the non-zero element\n",
    "            nonzeros = np.array( list(map(lambda each_matrix: np.array( list(map(lambda x: (x != 2).sum() + 1, each_matrix ) )), current_captions_matrix ) ))\n",
    "            for i in range(batch_size):\n",
    "                for ind, row in enumerate(current_captions_masks[i]):\n",
    "                    row[:(nonzeros[i, ind]-1)] = 1\n",
    "\n",
    "            # shape of current_feats: batch_size x 50 x 4096\n",
    "            # shape of current_num_distribution: batch_size x 6\n",
    "            # shape of current_captions_matrix: batch_size x 6 x 50\n",
    "            _, loss_val, loss_sent, loss_word= sess.run(\n",
    "                                [train_op, tf_loss, tf_loss_sent, tf_loss_word],\n",
    "                                feed_dict={\n",
    "                                           tf_feats: current_feats,\n",
    "                                           tf_num_distribution: current_num_distribution,\n",
    "                                           tf_captions_matrix: current_captions_matrix,\n",
    "                                           tf_captions_masks: current_captions_masks\n",
    "                                })\n",
    "\n",
    "            # append loss to list in a epoch\n",
    "            loss_to_draw_epoch.append(loss_val)\n",
    "\n",
    "            # running information\n",
    "            print('idx: ', start, ' Epoch: ', epoch, ' loss: ', loss_val, ' loss_sent: ', loss_sent, ' loss_word: ', loss_word, ' Time cost: ', str((time.time() - start_time)))\n",
    "            loss_fd.write('start: '+ str(start) +' end: ' + str(end) +' epoch ' + str(epoch) + ' loss ' + str(loss_val) + '\\n')\n",
    "            loss_fd.close()\n",
    "        # draw loss curve every epoch\n",
    "        loss_to_draw.append(np.mean(loss_to_draw_epoch))\n",
    "        plt_save_dir = './loss_imgs'\n",
    "        plt_save_img_name = str(epoch) + '.png'\n",
    "        plt.plot(range(len(loss_to_draw)), loss_to_draw, color='g')\n",
    "        plt.grid(True)\n",
    "        plt.savefig(os.path.join(plt_save_dir, plt_save_img_name))\n",
    "\n",
    "        if np.mod(epoch, 1) == 0:\n",
    "            print(\"Epoch \", epoch, \" is done. Saving the model ...\")\n",
    "            saver.save(sess, os.path.join(model_path, 'model'), global_step=epoch)\n",
    "    #loss_fd.close()\n",
    "\n",
    "\n",
    "def test():\n",
    "    \n",
    "    start_time = time.time()\n",
    "    # change the model path according to your environment\n",
    "    model_path = './model-250'\n",
    "\n",
    "    # It's very important to use Pandas to Series this idx2word dict\n",
    "    # After this operation, we can use list to extract the word at the same time\n",
    "    idx2word = pd.Series(np.load('./data/idx2word_batch.npy').tolist())\n",
    "\n",
    "    test_feats_path = './data/im2p_test_output.h5'\n",
    "    test_output_file = h5py.File(test_feats_path, 'r')\n",
    "    test_feats = test_output_file.get('feats')\n",
    "\n",
    "    test_imgs_full_path_lists = open('./densecap/imgs_test_path.txt').read().splitlines()\n",
    "    test_imgs_names = map(lambda x: os.path.basename(x).split('.')[0], test_imgs_full_path_lists)\n",
    "    \n",
    "    # n_words, batch_size, num_boxes, feats_dim, project_dim, sentRNN_lstm_dim, sentRNN_FC_dim, wordRNN_lstm_dim, S_max, N_max\n",
    "    test_model = RegionPooling_HierarchicalRNN(n_words = len(word2idx),\n",
    "                                               batch_size = batch_size,\n",
    "                                               num_boxes = num_boxes,\n",
    "                                               feats_dim = feats_dim,\n",
    "                                               project_dim = project_dim,\n",
    "                                               sentRNN_lstm_dim = sentRNN_lstm_dim,\n",
    "                                               sentRNN_FC_dim = sentRNN_FC_dim,\n",
    "                                               wordRNN_lstm_dim = wordRNN_lstm_dim,\n",
    "                                               S_max = S_max,\n",
    "                                               N_max = N_max,\n",
    "                                               word_embed_dim = word_embed_dim,\n",
    "                                               bias_init_vector = bias_init_vector)\n",
    "    \n",
    "\n",
    "    tf_feats, tf_generated_paragraph, tf_pred_re, tf_sent_topic_vectors = test_model.generate_model()\n",
    "    sess = tf.InteractiveSession()\n",
    "    #print(\"Before there >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\\n\\n\\n\")\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, model_path)\n",
    "\n",
    "    img2idx = {}\n",
    "    for idx, img in enumerate(test_imgs_names):\n",
    "        img2idx[img] = idx\n",
    "    print(\"IM2idx:::::::::::\",img2idx)\n",
    "\n",
    "    test_fd = open('HRNN_results.txt', 'w')\n",
    "    for idx, img_name in enumerate(test_imgs_names):\n",
    "        print(idx, img_name)\n",
    "        test_fd.write(img_name + '\\n')\n",
    "\n",
    "        each_paragraph = []\n",
    "        current_paragraph = \"\"\n",
    "\n",
    "        current_feats_index = img2idx[img_name]\n",
    "        current_feats = test_feats[current_feats_index]\n",
    "        current_feats = np.reshape(current_feats, [1, 50, 4096])\n",
    "\n",
    "        generated_paragraph_indexes, pred, sent_topic_vectors = sess.run(\n",
    "                                                                         [tf_generated_paragraph, tf_pred_re, tf_sent_topic_vectors],\n",
    "                                                                         feed_dict={\n",
    "                                                                             tf_feats: current_feats\n",
    "                                                                         })\n",
    "\n",
    "        #generated_paragraph = idx2word[generated_paragraph_indexes]\n",
    "        for sent_index in generated_paragraph_indexes:\n",
    "            each_sent = []\n",
    "            for word_index in sent_index:\n",
    "                each_sent.append(idx2word[word_index])\n",
    "            each_paragraph.append(each_sent)\n",
    "\n",
    "        for idx, each_sent in enumerate(each_paragraph):\n",
    "            # if the current sentence is the end sentence of the paragraph\n",
    "            # According to the probability distribution:\n",
    "            # CONTINUE: [1, 0]\n",
    "            # STOP    : [0, 1]\n",
    "            # So, if the first item of pred is less than the T_stop\n",
    "            # the generation process is break\n",
    "            if pred[idx][0][0] <= T_stop:\n",
    "                break\n",
    "            current_sent = ''\n",
    "            for each_word in each_sent:\n",
    "                current_sent += each_word + ' '\n",
    "            current_sent = current_sent.replace('<eos> ', '')\n",
    "            current_sent = current_sent.replace('<pad> ', '')\n",
    "            current_sent = current_sent + '.'\n",
    "            current_sent = current_sent.replace(' .', '.')\n",
    "            current_sent = current_sent.replace(' ,', ',')\n",
    "            current_paragraph +=current_sent\n",
    "            if idx != len(each_paragraph) - 1:\n",
    "                current_paragraph += ' '\n",
    "\n",
    "        test_fd.write(current_paragraph + '\\n')\n",
    "    test_fd.close()\n",
    "    print(\"Time cost: \" + str(time.time()-start_time))\n",
    "\n",
    "train()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing word counts and creating vocab based on word count threshold 2\n",
      "filtered words from 18418 to 9900\n",
      "Train Images Names ['2336489', '2386636', '2317584', '2349895', '2348349', '2393268', '2338512', '2319338', '2318897', '2382861', '2359450', '2393810', '2379103', '2400913', '2381116', '2379607', '2372836', '1592156', '2388737', '2316733', '2328394', '2391668', '2341747', '2319741', '2355380', '2353667', '2316710', '2393529', '1160254', '1159529', '2392926', '2399813', '2335023', '2400892', '2367765', '2375715', '2395072', '2349745', '2384897', '2406617', '2373936', '2318431', '2410717', '2353681', '2414772', '2346311', '2369025', '2367716', '2342700', '2355768', '2361773', '2352565', '2356407', '2393837', '2351900', '2378739', '2394493', '2334825', '2346609', '2355839', '2406971', '2327224', '2407039', '2377971', '2367554', '2368478', '2327958', '2366303', '2360963', '2414977', '2416656', '2378809', '2407520', '2366857', '2374646', '2327849', '2376953', '2373691', '1143', '2397944', '2360432', '2354513', '2342794', '2398110', '2339907', '2407806', '2411794', '2400094', '2404585', '2366266', '2319960', '2405522', '2393641', '2413048', '2348859', '2378864', '2401206', '2359079', '2344467', '2388269', '2339331', '2351557', '2382610', '2377309', '2406885', '2381442', '2320357', '2384407', '2356016', '2393655', '2414448', '2395330', '2370017', '2367081', '2363770', '2387731', '2327045', '2342281', '2356452', '2399436', '2367278', '2393255', '2389726', '2357005', '2393315', '2353974', '2332047', '2378386', '2365395', '2413514', '2405893', '2334135', '2346881', '2374191', '2376879', '2412229', '2397071', '2345532', '2387797', '2350585', '2386125', '2343001', '2414097', '2364666', '2380218', '2403512', '2342127', '677', '2396741', '2330715', '1592902', '2386334', '2317900', '2397657', '2369764', '2317356', '2398451', '2353100', '2395289', '2383971', '2330468', '2322189', '2354423', '2395996', '2387049', '2387000', '2396760', '2352342', '2376855', '2416104', '2328135', '2402485', '2408339', '2330793', '2414131', '2372569', '2368058', '2398193', '2324168', '2369286', '2402340', '2362921', '2345482', '2359944', '2414990', '2315498', '2400369', '2404090', '2368969', '2401288', '2322795', '2412037', '2383787', '2341126', '3974', '2359414', '2359892', '2355887', '2379898', '2411891', '2362560', '2385464', '2354529', '2330371', '2340772', '2397509', '2363834', '2367100', '2409862', '2409644', '2373013', '2362819', '3665', '2371493', '2414882', '2403911', '2333007', '2327911', '2379412', '2356812', '2385552', '2354133', '2388352', '2397716', '2374295', '2357064', '2350730', '2325410', '2333717', '2383123', '2384406', '2414408', '2383295', '285905', '2336076', '2397405', '2367095', '538', '2372940', '2325956', '2336279', '2347211', '2415820', '2401918', '2390711', '2367533', '2347068', '2318740', '2348088', '2348163', '2368358', '2316602', '2416559', '2401917', '2336426', '2345624', '2334296', '2407180', '2374141', '2412787', '2385876', '2381546', '2320667', '2385145', '2404878', '2388564', '2390577', '2382276', '2366431', '2367594', '2381613', '2355425', '2353451', '2372907', '2376996', '2346658', '2343595', '2383168', '2385127', '2357255', '2352218', '2357568', '2335133', '2349764', '2346469', '2399802', '2407604', '2351567', '2386257', '2389353', '2341717', '2400412', '2326483', '2362276', '2411576', '2352971', '2403968', '2384387', '2359089', '2388557', '2366034', '2362203', '2400363', '2335727', '2397165', '2400258', '2381999', '2358150', '2362539', '2337138', '2367019', '2330764', '2370517', '2366871', '2380114', '2413978', '2319237', '2355445', '2342671', '3668', '2379975', '2400371', '2412304', '2360521', '2370341', '2325644', '2333038', '2402051', '2323076', '2328285', '2402262', '2399652', '2388627', '2402384', '2400434', '879', '2403224', '2375161', '2399588', '2369272', '2409059', '2399333', '2405822', '2417585', '2387245', '2320951', '2350000', '2368599', '2366510', '2390201', '2387936', '2374610', '2368396', '2393934', '2341732', '2392460', '2352715', '2369389', '2380388', '2391875', '2384206', '2391968', '2393711', '2366615', '2323460', '2414959', '2389636', '2396317', '2320131', '2318134', '2323690', '2343705', '2363606', '2384171', '2360570', '2415871', '2402479', '2407622', '2330444', '2390977', '2333032', '2354631', '2399119', '2366841', '2341737', '2366521', '2319233', '2320748', '2390523', '2347202', '2388960', '2415207', '2350959', '2363427', '2415823', '2410663', '2403736', '2372245', '2324588', '2344657', '2370780', '2405392', '2363469', '2335274', '2402120', '2369075', '2369741', '2336529', '2408558', '2367475', '2382557', '2387430', '2337141', '2409608', '2361964', '2407102', '2332131', '2386687', '2393912', '2335959', '2335333', '2396301', '2358526', '2406374', '2386840', '2374389', '2341989', '2354115', '2356978', '2362795', '2384911', '2389931', '2367352', '2412770', '2382133', '2409345', '2363971', '2374611', '2401191', '2398695', '2375237', '2330650', '2344272', '2396662', '2365464', '2389121', '2407504', '2343682', '2367587', '2327729', '2372154', '2342183', '2352077', '2381840', '2341541', '2391262', '2374929', '2353696', '2345294', '2378992', '2390609', '2377837', '2346501', '2408374', '2358472', '2388519', '2373343', '2356427', '2403767', '2366500', '2392089', '2352126', '2397502', '2325536', '2410425', '2391895', '2348762', '2365437', '2373407', '2375118', '2317132', '2405554', '2360264', '2350622', '2370562', '2364541', '2357009', '4025', '2365122', '2386443', '2390263', '2392787', '2351655', '2368076', '2321387', '2365887', '2382695', '2343486', '2411245', '2401996', '2413035', '2339690', '2388772', '2370025', '2389796', '2374490', '2327449', '2413071', '2356526', '2328010', '2406478', '2390610', '2398309', '2358729', '2364522', '2354220', '2402305', '2381222', '2348689', '2396019', '2347126', '2337314', '2390811', '2381310', '2368501', '2347852', '2349084', '2349725', '2415570', '2388504', '2387542', '2375431', '2387602', '2387844', '2379303', '2323547', '2356915', '1981', '2411735', '185', '2350786', '2331437', '2322190', '2409525', '2389147', '2399256', '2354909', '2330044', '2392336', '2349339', '2330589', '2393434', '2397941', '2383900', '2392630', '2406271', '2404587', '2334130', '2366907', '2368952', '2385158', '2317379', '2352174', '2376974', '2408091', '2318766', '2334854', '2403746', '2376728', '2352834', '2401911', '2402816', '2367808', '2402523', '2395373', '2335613', '2343276', '2369585', '2377187', '2414350', '2363314', '2316641', '2382046', '2414334', '2392061', '2397271', '2412334', '2351275', '2356488', '2355676', '2414155', '2412163', '2332658', '2348280', '2371705', '2363496', '2416072', '2371802', '2383724', '2326852', '2414494', '2415154', '2386515', '2317337', '2362892', '2332039', '2362947', '2413575', '2404620', '2322974', '2402941', '2389832', '2415640', '2362015', '2388186', '2396639', '2392920', '2366216', '2320584', '2385308', '2388858', '2402273', '2390061', '2319024', '2330104', '2336219', '2353047', '2346948', '2406791', '2392746', '2383182', '2354989', '2396279', '2318316', '2411172', '2350037', '2402102', '2404395', '2412624', '2408962', '2316575', '2324955', '2364976', '2328805', '2376384', '2380250', '2411856', '2316555', '2361415', '2378276', '2331431', '2370776', '2368073', '2385974', '2391385', '2344095', '2319466', '2396971', '2387492', '2387882', '2341671', '2359537', '2387201', '1979', '2384089', '2361726', '2399529', '2394449', '2374130', '2362770', '2390516', '2415140', '2374917', '2356811', '2356083', '2363261', '2347475', '2379292', '2375286', '2402846', '2355462', '2379970', '2328347', '2323101', '2372676', '2378426', '2403730', '2359654', '2346934', '2359321', '2351300', '713323', '2408231', '4329', '2358273', '2318508', '2357580', '2401033', '2377611', '2356914', '2326661', '2339402', '2375681', '2358421', '2406595', '2372110', '2366679', '2382545', '2363649', '2340485', '2408489', '2339735', '2377054', '2387943', '1160101', '2342090', '2376379', '2346067', '2364405', '2371567', '2341611', '2343214', '2322095', '2403903', '2414523', '2386462', '2355888', '2385831', '2408088', '2404188', '2354380', '2390199', '2390572', '2391435', '2383119', '2387174', '2390009', '2389937', '2381824', '2391153', '2361388', '2349287', '2357948', '2398860', '2382464', '2351235', '2395464', '2362719', '2395640', '2360192', '2340085', '2367762', '2356491', '2344127', '2379868', '2408400', '2360971', '2412323', '2362383', '2378412', '2358670', '2372220', '2408036', '2340853', '2391663', '2375141', '2326911', '2370645', '2378572', '1186', '2388139', '2383040', '2390109', '2413346', '2318074', '2360829', '2401764', '2384225', '2315631', '2357129', '2409300', '2373049', '2409641', '2322341', '2337232', '2346288', '2329779', '2343913', '2382967', '2349022', '2371037', '2376314', '2390589', '2401112', '2353668', '2400991', '2355163', '2344607', '2359383', '2366081', '2415913', '2318100', '2368634', '2360988', '2339675', '2393319', '2357069', '2381952', '2347378', '2366720', '2399340', '2389264', '2373504', '2405126', '2404202', '2317987', '2351081', '2386694', '2394808', '2350826', '2408710', '2365947', '2348223', '2387276', '2353872', '2355350', '2399577', '2348182', '2371950', '2369310', '2341113', '2358311', '2357321', '2342802', '2387687', '2350248', '2400168', '2360242', '2409509', '2409624', '2339219', '2412609', '2399453', '2360241', '2407054', '2412782', '2389093', '2345813', '2402961', '2367747', '2374927', '2380095', '2352288', '2332290', '2356221', '2358787', '2381052', '2376293', '2382463', '2389704', '2409171', '2398902', '2335666', '2409161', '2398978', '2398703', '2364778', '2350734', '2317914', '2377131', '2382744', '2324748', '2396987', '2415464', '2325271', '2396374', '2400862', '2361143', '195', '2379893', '2339028', '2349530', '2400737', '2318603', '2352327', '2416725', '2377654', '2362679', '2394687', '2386134', '2336105', '2364452', '2390876', '2373588', '2374100', '2385757', '2414806', '2326048', '2365699', '2371216', '2357994', '815', '2393988', '2360115', '2413458', '2406696', '2356563', '2392261', '2325906', '2391950', '2391165', '2356074', '2417769', '2350434', '2389615', '2371836', '2394596', '2414255', '2404259', '2381424', '2405965', '2372008', '2327618', '2346802', '2379698', '2346556', '2371312', '2350160', '2386850', '2373092', '2375622', '2411459', '2389464', '2354957', '2380380', '2358567', '2393354', '2415082', '2413793', '2356957', '2363034', '2399393', '2377371', '2407660', '2326373', '2320101', '2416996', '2367637', '2383435', '2356093', '2330124', '2345254', '2403889', '2352072', '2357254', '2356671', '2401886', '2415083', '2374351', '2411866', '2319437', '2413099', '2413842', '2353865', '2339208', '2368903', '2351724', '2373382', '2326856', '2332037', '2380143', '2330294', '2411367', '2393488', '2322133', '2360127', '2353349', '2345044', '2373155', '2317117', '2339245', '2389666', '2374473', '2410762', '2325817', '3524', '2386442', '2351574', '2394242', '2345283', '2369213', '2365263', '2380024', '2376359', '2409993', '2400161', '2359369', '2392960', '2340975', '2388874', '2406588', '2337106', '2366203', '2417500', '2388733', '2347167', '2385460', '2370496', '2391984', '2345537', '2341245', '2402837', '2414256', '2341028', '2367884', '2336144', '2391257', '2390981', '2413997', '2411900', '2356269', '2342386', '2390189', '2399058', '2412087', '2415472', '2334735', '2373340', '2329113', '2320306', '2344088', '2350800', '2392376', '2389282', '2333242', '2380368', '2347956', '2385172', '2349036', '2360225', '2368103', '2336325', '2360287', '2336668', '2363108', '2381047', '2333962', '2410977', '2385461', '2347157', '2348401', '2402124', '2345255', '2372450', '2333467', '2341431', '2384439', '2354342', '2370343', '2324753', '2388172', '2318461', '2381263', '2377521', '2408653', '2409922', '2400672', '2317165', '2382775', '2340777', '2341785', '2361129', '2412964', '2398017', '2336760', '2377677', '2364629', '2352124', '2357723', '2357856', '2402586', '2379625', '2341187', '2394115', '2364463', '2412028', '2369707', '2394152', '2371926', '2316697', '2364742', '2355306', '2384410', '2316889', '2361855', '2336255', '2372392', '2357848', '2375873', '2341175', '2374654', '2338235', '2338501', '2387508', '2359860', '2346025', '2335228', '2404930', '2361721', '2361025', '2355690', '2336091', '2348939', '2399545', '2366433', '2379759', '2367250', '2402756', '2333815', '2413923', '2411635', '2373922', '2345388', '2381013', '2323566', '2327125', '2360148', '2341890', '2335993', '2318300', '2380510', '2411945', '2371361', '2410757', '2373934', '2360110', '2382330', '2393174', '2374097', '2322813', '2350025', '2396452', '2320897', '2346003', '2317707', '2397549', '2392557', '2389618', '2376132', '2379456', '2348675', '2366243', '2374003', '2404970', '2407026', '2412178', '2325076', '2356053', '2366552', '2387213', '2318529', '2368771', '2369653', '2387088', '2329225', '2401071', '2350763', '2398879', '2375098', '2362648', '2339034', '2381526', '2378990', '2368581', '2342463', '2410548', '2384021', '2412817', '2344622', '2356073', '2336315', '2388101', '2403142', '2396069', '2331045', '2411332', '2388879', '2357014', '2316318', '2348847', '2407133', '2359501', '2330143', '2372453', '2361523', '2320205', '2375771', '1592381', '2344878', '2352917', '2339305', '2395465', '2332111', '2346977', '2374041', '2388431', '2326536', '2402848', '2412679', '2354059', '2394933', '2369502', '2413540', '2326621', '2367096', '2341034', '2363443', '2374619', '2321253', '2342406', '2414189', '2346660', '2353466', '2351967', '2339582', '2397008', '2392194', '2406329', '2389588', '2396340', '2347072', '2378698', '2401822', '2382813', '2320614', '2322420', '2354873', '2364493', '2342919', '2416678', '2348334', '2346679', '2343220', '2351882', '2323317', '2369897', '2392583', '2408943', '2385255', '2347777', '2355399', '2378937', '2403177', '2342324', '2339891', '543', '2377454', '2355566', '2395698', '2410958', '2394635', '2371761', '2381108', '2365692', '2409873', '2407325', '2367729', '2407311', '2382252', '2403309', '2381396', '2350770', '2366958', '2346010', '2396757', '2408546', '2376235', '2354852', '2341672', '2390548', '2377272', '2344453', '2319185', '2370179', '2371763', '2371546', '2409589', '2386989', '2398489', '2390759', '2339004', '2355449', '2397683', '2319462', '2412080', '2411605', '2381761', '2395545', '2392163', '2373065', '2316025', '2380268', '2408617', '2412438', '2341668', '2336765', '2381550', '2358240', '2400462', '2316675', '2383068', '2388838', '2395479', '2390728', '2365043', '2397123', '2334742', '2412430', '2372018', '2361424', '2408144', '2404732', '2397229', '2409530', '2327406', '2400936', '2356858', '2380553', '2352140', '2319561', '2388446', '2321507', '2387187', '2358081', '2345744', '2347586', '2385725', '2407029', '2348398', '2386365', '2319419', '2398747', '2403123', '2371198', '2358999', '2374686', '2341305', '2394677', '2405763', '2381158', '2414099', '2390775', '2323158', '2407637', '2385956', '2403827', '2372426', '2397387', '2338355', '2366603', '2398565', '2355012', '2384747', '2382879', '2347702', '2385541', '2408452', '2319011', '2356436', '2410233', '2355933', '2330218', '2351858', '2338820', '2404002', '2400330', '2398037', '2345601', '2354318', '2353425', '2416937', '2336606', '2346818', '2381711', '2392202', '2354877', '2351596', '2348629', '2374316', '2381335', '2372957', '2395164', '2362840', '2387136', '2358132', '2341434', '2342864', '2400709', '2380640', '2317896', '2392065', '2394298', '2350383', '2382659', '2398866', '2331553', '2403182', '2322086', '2415884', '2332379', '2400542', '2409861', '2415978', '2356542', '2349065', '2350816', '2345625', '2402610', '2366219', '2414365', '2407808', '2387514', '2371255', '2354814', '2318159', '2377334', '2351200', '2394339', '2316574', '2374409', '2413783', '2328094', '2376076', '2411532', '2329559', '2404178', '2401816', '2404791', '2387509', '2354019', '2335492', '2316860', '2400289', '2401940', '2392014', '2367427', '2410352', '2355922', '2351483', '2395566', '2378204', '2369158', '2378294', '2343623', '2383042', '2342646', '2344284', '2349469', '2360556', '2340693', '2396197', '2354266', '2412725', '2326050', '2411637', '2319411', '1512', '2344722', '2322620', '2384101', '2406861', '2380329', '2389532', '2319274', '2411629', '2394536', '2346205', '2393422', '2392176', '2333402', '2349266', '2412056', '2369980', '2332365', '2391502', '2411500', '2407631', '1592372', '2330315', '2360855', '2371907', '2365979', '2405811', '2402053', '2396396', '2403131', '2334724', '2354544', '2400676', '2363823', '2402786', '2403431', '2398246', '2352466', '2350676', '2358414', '2331430', '2359934', '2358660', '2409730', '2384053', '2364313', '2398386', '2323413', '2397224', '2405436', '2404393', '2414493', '2405007', '2345363', '2351156', '2405703', '2391849', '2354362', '2373886', '2400392', '2371441', '2322528', '2414878', '2322397', '2357096', '2371625', '2377951', '2384475', '2333993', '2347274', '2401035', '2370296', '2380874', '2360405', '2346066', '2392507', '2358834', '2373821', '2388891', '2342230', '2410117', '2383962', '2417944', '2405504', '2348951', '2397307', '2409964', '2403749', '2417815', '2357770', '2410491', '2348208', '2390517', '2408377', '2410270', '2341363', '2383665', '2390619', '2380617', '2318253', '2361015', '2407563', '2357859', '2321198', '2401910', '2345382', '2394967', '2349108', '853', '2317625', '2408708', '2349353', '2376251', '2371551', '2348278', '2415457', '2316887', '2403418', '2343155', '2393166', '2343948', '2396669', '2360573', '2367727', '2344540', '2354147', '2353938', '2364799', '2414694', '2379974', '2359027', '2381416', '2389036', '2392765', '2394250', '2396407', '2399390', '2327810', '2375529', '2372145', '2389638', '2387786', '2382666', '2394823', '2391902', '2380908', '2370342', '2365664', '2362607', '2383586', '2405682', '2411960', '2389597', '2379832', '2319434', '2346718', '2405292', '2345464', '498337', '2412032', '2397030', '2357052', '2360201', '2359041', '2409746', '2328180', '2415173', '2369188', '2410088', '2352259', '2377573', '2364664', '2345826', '2360637', '2369615', '2382188', '2361538', '2390131', '2564', '2415473', '2345963', '2364711', '2374001', '2409142', '2355186', '2359734', '2377897', '2410775', '2355527', '2410305', '2317409', '2365687', '2331127', '2336558', '2411311', '2343876', '2398408', '2371017', '2367175', '2366378', '2330167', '2375133', '2399464', '2377832', '2347763', '2374300', '2369292', '2399932', '2344295', '2382977', '2318313', '2411813', '2343356', '2384658', '2395463', '2398544', '2352643', '2376750', '2403306', '2377053', '2329100', '4526', '2351834', '2390500', '2370551', '2355876', '2333581', '2401969', '2405056', '2364869', '2350985', '2408983', '2415510', '2347477', '2390282', '2344595', '2394297', '2391397', '2400181', '2387933', '2337581', '2342464', '2375898', '2333227', '2384614', '2363103', '61515', '2410276', '2387370', '2372279', '2343554', '2408872', '2403553', '2342857', '2353270', '2385349', '2318479', '2373822', '2385693', '2335572', '2401818', '2341844', '2415192', '2372030', '2346365', '2356290', '2410913', '2343589', '2413030', '2387651', '2341390', '2370165', '2367980', '2410681', '2387944', '2364952', '2364830', '2410500', '2388187', '2379551', '2374375', '2410345', '2352132', '2316535', '2387583', '2400705', '2400198', '2356774', '2344294', '2399862', '2355172', '2369580', '2368472', '2393843', '2386965', '2346626', '2317055', '2389880', '2344880', '2373460', '2405096', '2336832', '2396514', '2353172', '2405568', '2388006', '2361585', '2377072', '2352426', '2384368', '2405981', '2366660', '2344000', '2396968', '2342960', '2411125', '2355581', '2411121', '2393513', '2365691', '2400579', '2371228', '2317860', '2319593', '2354526', '2378517', '2316254', '2410070', '1159516', '2358293', '2318856', '2372039', '2406319', '2350783', '2359238', '2398991', '2345484', '2363592', '2368524', '2315529', '2360315', '2324220', '2408638', '2318354', '2413457', '2399569', '2345476', '2365226', '2402050', '2343057', '2382441', '2373021', '2382173', '2382632', '2345621', '2362731', '2379040', '2323848', '2405490', '2401986', '2393778', '2382524', '2340615', '2406927', '2378480', '2389642', '2376475', '2354169', '2361554', '2326672', '2394057', '2388526', '2406376', '2375076', '2374809', '2331362', '2392123', '2397845', '2332173', '2402155', '2349448', '2381847', '2390911', '2379368', '2393134', '2394007', '2362198', '2340394', '2390099', '2374858', '2413411', '2384283', '2361208', '2346654', '2342062', '2366424', '2318214', '2387531', '2317457', '2400046', '2379797', '2371733', '2396351', '2367180', '2401819', '2363548', '2351789', '2336180', '2342649', '2403572', '2414803', '2343586', '2367805', '2399954', '2416836', '2365803', '2357249', '2350678', '2412691', '2385897', '2358578', '2322107', '2375389', '2408481', '2383497', '2411830', '2360483', '2381777', '2354093', '2387137', '2400665', '2374256', '2373225', '2352059', '2338341', '2338001', '2410181', '2330919', '2382039', '2392600', '2376247', '2413702', '2344077', '2382974', '2325024', '2345290', '2392468', '2382722', '2407293', '2392613', '2363437', '2409165', '2409238', '2347170', '2400907', '2395914', '2393093', '2345716', '2409995', '2362360', '2410413', '2358566', '2411064', '2400488', '2402864', '2318784', '2410083', '2346215', '2397756', '2410111', '2382522', '2363057', '2406213', '2390476', '2399831', '2409738', '2349149', '2389587', '2373951', '2504', '2393995', '2396179', '2371338', '2385571', '2336159', '2371432', '2383269', '2368666', '2344637', '2373063', '2391194', '2386673', '2320078', '2391461', '2372285', '2379258', '2341514', '2318841', '2386678', '2400689', '2341725', '2376970', '2406500', '2357507', '2396448', '2395630', '2390890', '2344135', '2385174', '2384854', '2413861', '2386495', '2374212', '2393279', '2382690', '2403866', '2399949', '285802', '2400680', '2399996', '2356573', '2416386', '2344117', '2408673', '2416167', '2391375', '2385815', '2384790', '2317122', '2387553', '2349652', '2363110', '2382543', '2318426', '2390084', '2318528', '2364598', '2412966', '2369601', '2396112', '150344', '2333510', '2324573', '2415027', '2363104', '2372733', '587', '2357502', '2389782', '2387258', '2334930', '2352992', '2345982', '2346439', '2392581', '2345262', '2388166', '2375625', '2351511', '2408359', '2391869', '2392818', '2369855', '2327758', '2409153', '2399525', '2415676', '2400518', '2387989', '2379307', '2363501', '2362555', '2323209', '2359589', '2378223', '2391252', '2316382', '2367052', '2359190', '358', '2328602', '2318174', '2408468', '2355642', '1067', '2352832', '2320127', '2402825', '2394911', '2405788', '2383081', '2353014', '2356926', '2341770', '2354589', '2319025', '2386258', '2361204', '2343537', '2317743', '2409948', '2355213', '2407074', '2327559', '2370652', '2392451', '2403297', '2328024', '1947', '2382933', '2380071', '2386619', '2343802', '2374709', '2395037', '2318491', '2393164', '2366570', '2383358', '2363661', '2401897', '2323005', '2318753', '2402202', '2399182', '2380444', '2415776', '2410116', '2405963', '2408407', '2414540', '2408480', '2392500', '2368332', '2340505', '2393496', '2365504', '2340547', '2361489', '2342678', '2401336', '2348566', '2414418', '2358790', '2404467', '724', '2399843', '2405738', '2393060', '2385216', '2370877', '2350752', '2343739', '2380279', '2379332', '2366798', '2406548', '2364648', '2320207', '2398212', '2375241', '2408244', '2393002', '2364458', '2364569', '2325414', '2374807', '2353337', '2384906', '2390945', '2336229', '2349203', '2368261', '2345893', '2359948', '2398222', '2396670', '2324375', '2399556', '2406733', '2385986', '2361991', '2389426', '2343612', '2360527', '2342329', '2341047', '2367875', '2334644', '2415363', '2392812', '2389754', '2352853', '2339689', '2395881', '2367978', '2365490', '2343596', '2333164', '2367144', '2350140', '2382213', '2315963', '2406648', '2366650', '2344341', '2323588', '2370697', '2410146', '2395696', '2335504', '2362808', '2391256', '2403683', '2410297', '2319554', '2333255', '2339046', '2413904', '2411941', '2317592', '2346843', '262', '2317319', '2412901', '2391111', '2394454', '2392869', '2400554', '59', '2373368', '2344661', '2406056', '2389172', '2354660', '2334516', '2407120', '2368870', '2403096', '2360943', '2338460', '2414226', '2412833', '2360125', '2363840', '2321324', '2400079', '2396864', '2404541', '2343376', '2344370', '2330999', '2414157', '2366795', '2366302', '2354637', '2398486', '2338878', '2402374', '2348627', '2389270', '2372029', '2400643', '2355709', '2362311', '2344881', '2379480', '2329101', '2373700', '2404918', '2348172', '2356669', '2354564', '2397809', '2346342', '2361074', '2377368', '2317513', '2378059', '2340180', '2317591', '2366127', '2390288', '2396390', '2335223', '2341640', '2402546', '2351277', '2374760', '2391209', '2316729', '2317053', '2350520', '2327364', '2386260', '2338004', '2394482', '2347783', '2414805', '2378686', '2397605', '2387464', '2335457', '2350362', '2355111', '2343118', '2409065', '2357554', '2355118', '2383588', '2404485', '2317038', '2337365', '2395949', '2359272', '2382849', '2373510', '2363267', '2387911', '2369971', '2381902', '2336048', '2401096', '2412694', '2393524', '2371744', '2339578', '2369882', '2364907', '2376391', '2399359', '2330962', '2376356', '2364516', '2411640', '2380488', '2357038', '2320356', '2330639', '2360200', '2397120', '2390392', '2346419', '2396290', '2358575', '2343618', '2376371', '2387761', '2390032', '2403502', '2334281', '2382381', '2407666', '2317228', '2391722', '2376016', '2322573', '2336110', '2395507', '2331963', '2395930', '2356795', '2367957', '2342128', '2358287', '2395628', '2369455', '2380117', '2385072', '2338012', '2347982', '2362130', '2378490', '2364094', '2372176', '2389127', '2366148', '2338443', '2343766', '2385053', '2346498', '2416307', '2356329', '2407240', '2315640', '2366855', '2386983', '2403565', '2365595', '2386870', '2367983', '2387207', '2413134', '2359636', '2341700', '2347316', '2377986', '2390394', '2382975', '2411955', '2352243', '2396465', '2391047', '2335091', '2355583', '2405444', '2339483', '2357873', '2341383', '2347742', '2401129', '2342125', '2414784', '1159963', '2349488', '2405390', '2384500', '2366640', '2382714', '2410358', '2399749', '2376467', '2340926', '2361164', '2396478', '2348265', '2391182', '2362917', '2354590', '2408454', '2337033', '2371418', '2411249', '2387946', '2405966', '2397831', '2377290', '2394457', '2414173', '2350934', '2379419', '2408594', '2386847', '2415196', '2388316', '2413037', '2384670', '2346974', '2342078', '2387721', '2375570', '2338625', '2348008', '2352560', '2369013', '2369937', '2317478', '2332609', '2362277', '2408209', '2388111', '2410852', '2385025', '2377512', '2347040', '2366432', '2335275', '2316204', '2396682', '2375612', '2414069', '2355641', '2379154', '813', '2407413', '2323665', '2398578', '2396029', '2391545', '2346922', '2389002', '2340767', '2342976', '2374164', '2321676', '2403291', '2400108', '2375753', '2341674', '2377456', '2377161', '2376001', '2371756', '2365457', '2341703', '2405776', '2361275', '2352062', '2386491', '2408041', '2351669', '2413323', '2406451', '2393764', '2377815', '2356592', '2385066', '2334363', '2364508', '2364582', '2378366', '2407349', '2362280', '2371998', '2397147', '2343805', '2358223', '2378060', '2347102', '2402174', '2316661']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0724 09:10:24.178529 140197742454528 deprecation.py:323] From <ipython-input-1-0434d5425a35>:69: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "W0724 09:10:24.230486 140197742454528 deprecation.py:323] From <ipython-input-1-0434d5425a35>:87: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wemb output>>>>>>>>>>>> <tf.Variable 'Wemb:0' shape=(9904, 1024) dtype=float32_ref> Tensor(\"Shape:0\", shape=(2,), dtype=int32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0724 09:10:24.759396 140197742454528 deprecation.py:506] From /home/student/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0724 09:10:24.765649 140197742454528 deprecation.py:506] From /home/student/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py:738: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start build model:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0724 09:10:25.021539 140197742454528 deprecation.py:323] From <ipython-input-1-0434d5425a35>:167: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "W0724 09:10:25.306723 140197742454528 deprecation.py:323] From <ipython-input-1-0434d5425a35>:198: sparse_to_dense (from tensorflow.python.ops.sparse_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n",
      "Success\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0724 09:11:05.881547 140197742454528 deprecation.py:323] From /home/student/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done w session\n",
      "Img2idx {'2336489': 0, '2386636': 1, '2317584': 2, '2349895': 3, '2348349': 4, '2393268': 5, '2338512': 6, '2319338': 7, '2318897': 8, '2382861': 9, '2359450': 10, '2393810': 11, '2379103': 12, '2400913': 13, '2381116': 14, '2379607': 15, '2372836': 16, '1592156': 17, '2388737': 18, '2316733': 19, '2328394': 20, '2391668': 21, '2341747': 22, '2319741': 23, '2355380': 24, '2353667': 25, '2316710': 26, '2393529': 27, '1160254': 28, '1159529': 29, '2392926': 30, '2399813': 31, '2335023': 32, '2400892': 33, '2367765': 34, '2375715': 35, '2395072': 36, '2349745': 37, '2384897': 38, '2406617': 39, '2373936': 40, '2318431': 41, '2410717': 42, '2353681': 43, '2414772': 44, '2346311': 45, '2369025': 46, '2367716': 47, '2342700': 48, '2355768': 49, '2361773': 50, '2352565': 51, '2356407': 52, '2393837': 53, '2351900': 54, '2378739': 55, '2394493': 56, '2334825': 57, '2346609': 58, '2355839': 59, '2406971': 60, '2327224': 61, '2407039': 62, '2377971': 63, '2367554': 64, '2368478': 65, '2327958': 66, '2366303': 67, '2360963': 68, '2414977': 69, '2416656': 70, '2378809': 71, '2407520': 72, '2366857': 73, '2374646': 74, '2327849': 75, '2376953': 76, '2373691': 77, '1143': 78, '2397944': 79, '2360432': 80, '2354513': 81, '2342794': 82, '2398110': 83, '2339907': 84, '2407806': 85, '2411794': 86, '2400094': 87, '2404585': 88, '2366266': 89, '2319960': 90, '2405522': 91, '2393641': 92, '2413048': 93, '2348859': 94, '2378864': 95, '2401206': 96, '2359079': 97, '2344467': 98, '2388269': 99, '2339331': 100, '2351557': 101, '2382610': 102, '2377309': 103, '2406885': 104, '2381442': 105, '2320357': 106, '2384407': 107, '2356016': 108, '2393655': 109, '2414448': 110, '2395330': 111, '2370017': 112, '2367081': 113, '2363770': 114, '2387731': 115, '2327045': 116, '2342281': 117, '2356452': 118, '2399436': 119, '2367278': 120, '2393255': 121, '2389726': 122, '2357005': 123, '2393315': 124, '2353974': 125, '2332047': 126, '2378386': 127, '2365395': 128, '2413514': 129, '2405893': 130, '2334135': 131, '2346881': 132, '2374191': 133, '2376879': 134, '2412229': 135, '2397071': 136, '2345532': 137, '2387797': 138, '2350585': 139, '2386125': 140, '2343001': 141, '2414097': 142, '2364666': 143, '2380218': 144, '2403512': 145, '2342127': 146, '677': 147, '2396741': 148, '2330715': 149, '1592902': 150, '2386334': 151, '2317900': 152, '2397657': 153, '2369764': 154, '2317356': 155, '2398451': 156, '2353100': 157, '2395289': 158, '2383971': 159, '2330468': 160, '2322189': 161, '2354423': 162, '2395996': 163, '2387049': 164, '2387000': 165, '2396760': 166, '2352342': 167, '2376855': 168, '2416104': 169, '2328135': 170, '2402485': 171, '2408339': 172, '2330793': 173, '2414131': 174, '2372569': 175, '2368058': 176, '2398193': 177, '2324168': 178, '2369286': 179, '2402340': 180, '2362921': 181, '2345482': 182, '2359944': 183, '2414990': 184, '2315498': 185, '2400369': 186, '2404090': 187, '2368969': 188, '2401288': 189, '2322795': 190, '2412037': 191, '2383787': 192, '2341126': 193, '3974': 194, '2359414': 195, '2359892': 196, '2355887': 197, '2379898': 198, '2411891': 199, '2362560': 200, '2385464': 201, '2354529': 202, '2330371': 203, '2340772': 204, '2397509': 205, '2363834': 206, '2367100': 207, '2409862': 208, '2409644': 209, '2373013': 210, '2362819': 211, '3665': 212, '2371493': 213, '2414882': 214, '2403911': 215, '2333007': 216, '2327911': 217, '2379412': 218, '2356812': 219, '2385552': 220, '2354133': 221, '2388352': 222, '2397716': 223, '2374295': 224, '2357064': 225, '2350730': 226, '2325410': 227, '2333717': 228, '2383123': 229, '2384406': 230, '2414408': 231, '2383295': 232, '285905': 233, '2336076': 234, '2397405': 235, '2367095': 236, '538': 237, '2372940': 238, '2325956': 239, '2336279': 240, '2347211': 241, '2415820': 242, '2401918': 243, '2390711': 244, '2367533': 245, '2347068': 246, '2318740': 247, '2348088': 248, '2348163': 249, '2368358': 250, '2316602': 251, '2416559': 252, '2401917': 253, '2336426': 254, '2345624': 255, '2334296': 256, '2407180': 257, '2374141': 258, '2412787': 259, '2385876': 260, '2381546': 261, '2320667': 262, '2385145': 263, '2404878': 264, '2388564': 265, '2390577': 266, '2382276': 267, '2366431': 268, '2367594': 269, '2381613': 270, '2355425': 271, '2353451': 272, '2372907': 273, '2376996': 274, '2346658': 275, '2343595': 276, '2383168': 277, '2385127': 278, '2357255': 279, '2352218': 280, '2357568': 281, '2335133': 282, '2349764': 283, '2346469': 284, '2399802': 285, '2407604': 286, '2351567': 287, '2386257': 288, '2389353': 289, '2341717': 290, '2400412': 291, '2326483': 292, '2362276': 293, '2411576': 294, '2352971': 295, '2403968': 296, '2384387': 297, '2359089': 298, '2388557': 299, '2366034': 300, '2362203': 301, '2400363': 302, '2335727': 303, '2397165': 304, '2400258': 305, '2381999': 306, '2358150': 307, '2362539': 308, '2337138': 309, '2367019': 310, '2330764': 311, '2370517': 312, '2366871': 313, '2380114': 314, '2413978': 315, '2319237': 316, '2355445': 317, '2342671': 318, '3668': 319, '2379975': 320, '2400371': 321, '2412304': 322, '2360521': 323, '2370341': 324, '2325644': 325, '2333038': 326, '2402051': 327, '2323076': 328, '2328285': 329, '2402262': 330, '2399652': 331, '2388627': 332, '2402384': 333, '2400434': 334, '879': 335, '2403224': 336, '2375161': 337, '2399588': 338, '2369272': 339, '2409059': 340, '2399333': 341, '2405822': 342, '2417585': 343, '2387245': 344, '2320951': 345, '2350000': 346, '2368599': 347, '2366510': 348, '2390201': 349, '2387936': 350, '2374610': 351, '2368396': 352, '2393934': 353, '2341732': 354, '2392460': 355, '2352715': 356, '2369389': 357, '2380388': 358, '2391875': 359, '2384206': 360, '2391968': 361, '2393711': 362, '2366615': 363, '2323460': 364, '2414959': 365, '2389636': 366, '2396317': 367, '2320131': 368, '2318134': 369, '2323690': 370, '2343705': 371, '2363606': 372, '2384171': 373, '2360570': 374, '2415871': 375, '2402479': 376, '2407622': 377, '2330444': 378, '2390977': 379, '2333032': 380, '2354631': 381, '2399119': 382, '2366841': 383, '2341737': 384, '2366521': 385, '2319233': 386, '2320748': 387, '2390523': 388, '2347202': 389, '2388960': 390, '2415207': 391, '2350959': 392, '2363427': 393, '2415823': 394, '2410663': 395, '2403736': 396, '2372245': 397, '2324588': 398, '2344657': 399, '2370780': 400, '2405392': 401, '2363469': 402, '2335274': 403, '2402120': 404, '2369075': 405, '2369741': 406, '2336529': 407, '2408558': 408, '2367475': 409, '2382557': 410, '2387430': 411, '2337141': 412, '2409608': 413, '2361964': 414, '2407102': 415, '2332131': 416, '2386687': 417, '2393912': 418, '2335959': 419, '2335333': 420, '2396301': 421, '2358526': 422, '2406374': 423, '2386840': 424, '2374389': 425, '2341989': 426, '2354115': 427, '2356978': 428, '2362795': 429, '2384911': 430, '2389931': 431, '2367352': 432, '2412770': 433, '2382133': 434, '2409345': 435, '2363971': 436, '2374611': 437, '2401191': 438, '2398695': 439, '2375237': 440, '2330650': 441, '2344272': 442, '2396662': 443, '2365464': 444, '2389121': 445, '2407504': 446, '2343682': 447, '2367587': 448, '2327729': 449, '2372154': 450, '2342183': 451, '2352077': 452, '2381840': 453, '2341541': 454, '2391262': 455, '2374929': 456, '2353696': 457, '2345294': 458, '2378992': 459, '2390609': 460, '2377837': 461, '2346501': 462, '2408374': 463, '2358472': 464, '2388519': 465, '2373343': 466, '2356427': 467, '2403767': 468, '2366500': 469, '2392089': 470, '2352126': 471, '2397502': 472, '2325536': 473, '2410425': 474, '2391895': 475, '2348762': 476, '2365437': 477, '2373407': 478, '2375118': 479, '2317132': 480, '2405554': 481, '2360264': 482, '2350622': 483, '2370562': 484, '2364541': 485, '2357009': 486, '4025': 487, '2365122': 488, '2386443': 489, '2390263': 490, '2392787': 491, '2351655': 492, '2368076': 493, '2321387': 494, '2365887': 495, '2382695': 496, '2343486': 497, '2411245': 498, '2401996': 499, '2413035': 500, '2339690': 501, '2388772': 502, '2370025': 503, '2389796': 504, '2374490': 505, '2327449': 506, '2413071': 507, '2356526': 508, '2328010': 509, '2406478': 510, '2390610': 511, '2398309': 512, '2358729': 513, '2364522': 514, '2354220': 515, '2402305': 516, '2381222': 517, '2348689': 518, '2396019': 519, '2347126': 520, '2337314': 521, '2390811': 522, '2381310': 523, '2368501': 524, '2347852': 525, '2349084': 526, '2349725': 527, '2415570': 528, '2388504': 529, '2387542': 530, '2375431': 531, '2387602': 532, '2387844': 533, '2379303': 534, '2323547': 535, '2356915': 536, '1981': 537, '2411735': 538, '185': 539, '2350786': 540, '2331437': 541, '2322190': 542, '2409525': 543, '2389147': 544, '2399256': 545, '2354909': 546, '2330044': 547, '2392336': 548, '2349339': 549, '2330589': 550, '2393434': 551, '2397941': 552, '2383900': 553, '2392630': 554, '2406271': 555, '2404587': 556, '2334130': 557, '2366907': 558, '2368952': 559, '2385158': 560, '2317379': 561, '2352174': 562, '2376974': 563, '2408091': 564, '2318766': 565, '2334854': 566, '2403746': 567, '2376728': 568, '2352834': 569, '2401911': 570, '2402816': 571, '2367808': 572, '2402523': 573, '2395373': 574, '2335613': 575, '2343276': 576, '2369585': 577, '2377187': 578, '2414350': 579, '2363314': 580, '2316641': 581, '2382046': 582, '2414334': 583, '2392061': 584, '2397271': 585, '2412334': 586, '2351275': 587, '2356488': 588, '2355676': 589, '2414155': 590, '2412163': 591, '2332658': 592, '2348280': 593, '2371705': 594, '2363496': 595, '2416072': 596, '2371802': 597, '2383724': 598, '2326852': 599, '2414494': 600, '2415154': 601, '2386515': 602, '2317337': 603, '2362892': 604, '2332039': 605, '2362947': 606, '2413575': 607, '2404620': 608, '2322974': 609, '2402941': 610, '2389832': 611, '2415640': 612, '2362015': 613, '2388186': 614, '2396639': 615, '2392920': 616, '2366216': 617, '2320584': 618, '2385308': 619, '2388858': 620, '2402273': 621, '2390061': 622, '2319024': 623, '2330104': 624, '2336219': 625, '2353047': 626, '2346948': 627, '2406791': 628, '2392746': 629, '2383182': 630, '2354989': 631, '2396279': 632, '2318316': 633, '2411172': 634, '2350037': 635, '2402102': 636, '2404395': 637, '2412624': 638, '2408962': 639, '2316575': 640, '2324955': 641, '2364976': 642, '2328805': 643, '2376384': 644, '2380250': 645, '2411856': 646, '2316555': 647, '2361415': 648, '2378276': 649, '2331431': 650, '2370776': 651, '2368073': 652, '2385974': 653, '2391385': 654, '2344095': 655, '2319466': 656, '2396971': 657, '2387492': 658, '2387882': 659, '2341671': 660, '2359537': 661, '2387201': 662, '1979': 663, '2384089': 664, '2361726': 665, '2399529': 666, '2394449': 667, '2374130': 668, '2362770': 669, '2390516': 670, '2415140': 671, '2374917': 672, '2356811': 673, '2356083': 674, '2363261': 675, '2347475': 676, '2379292': 677, '2375286': 678, '2402846': 679, '2355462': 680, '2379970': 681, '2328347': 682, '2323101': 683, '2372676': 684, '2378426': 685, '2403730': 686, '2359654': 687, '2346934': 688, '2359321': 689, '2351300': 690, '713323': 691, '2408231': 692, '4329': 693, '2358273': 694, '2318508': 695, '2357580': 696, '2401033': 697, '2377611': 698, '2356914': 699, '2326661': 700, '2339402': 701, '2375681': 702, '2358421': 703, '2406595': 704, '2372110': 705, '2366679': 706, '2382545': 707, '2363649': 708, '2340485': 709, '2408489': 710, '2339735': 711, '2377054': 712, '2387943': 713, '1160101': 714, '2342090': 715, '2376379': 716, '2346067': 717, '2364405': 718, '2371567': 719, '2341611': 720, '2343214': 721, '2322095': 722, '2403903': 723, '2414523': 724, '2386462': 725, '2355888': 726, '2385831': 727, '2408088': 728, '2404188': 729, '2354380': 730, '2390199': 731, '2390572': 732, '2391435': 733, '2383119': 734, '2387174': 735, '2390009': 736, '2389937': 737, '2381824': 738, '2391153': 739, '2361388': 740, '2349287': 741, '2357948': 742, '2398860': 743, '2382464': 744, '2351235': 745, '2395464': 746, '2362719': 747, '2395640': 748, '2360192': 749, '2340085': 750, '2367762': 751, '2356491': 752, '2344127': 753, '2379868': 754, '2408400': 755, '2360971': 756, '2412323': 757, '2362383': 758, '2378412': 759, '2358670': 760, '2372220': 761, '2408036': 762, '2340853': 763, '2391663': 764, '2375141': 765, '2326911': 766, '2370645': 767, '2378572': 768, '1186': 769, '2388139': 770, '2383040': 771, '2390109': 772, '2413346': 773, '2318074': 774, '2360829': 775, '2401764': 776, '2384225': 777, '2315631': 778, '2357129': 779, '2409300': 780, '2373049': 781, '2409641': 782, '2322341': 783, '2337232': 784, '2346288': 785, '2329779': 786, '2343913': 787, '2382967': 788, '2349022': 789, '2371037': 790, '2376314': 791, '2390589': 792, '2401112': 793, '2353668': 794, '2400991': 795, '2355163': 796, '2344607': 797, '2359383': 798, '2366081': 799, '2415913': 800, '2318100': 801, '2368634': 802, '2360988': 803, '2339675': 804, '2393319': 805, '2357069': 806, '2381952': 807, '2347378': 808, '2366720': 809, '2399340': 810, '2389264': 811, '2373504': 812, '2405126': 813, '2404202': 814, '2317987': 815, '2351081': 816, '2386694': 817, '2394808': 818, '2350826': 819, '2408710': 820, '2365947': 821, '2348223': 822, '2387276': 823, '2353872': 824, '2355350': 825, '2399577': 826, '2348182': 827, '2371950': 828, '2369310': 829, '2341113': 830, '2358311': 831, '2357321': 832, '2342802': 833, '2387687': 834, '2350248': 835, '2400168': 836, '2360242': 837, '2409509': 838, '2409624': 839, '2339219': 840, '2412609': 841, '2399453': 842, '2360241': 843, '2407054': 844, '2412782': 845, '2389093': 846, '2345813': 847, '2402961': 848, '2367747': 849, '2374927': 850, '2380095': 851, '2352288': 852, '2332290': 853, '2356221': 854, '2358787': 855, '2381052': 856, '2376293': 857, '2382463': 858, '2389704': 859, '2409171': 860, '2398902': 861, '2335666': 862, '2409161': 863, '2398978': 864, '2398703': 865, '2364778': 866, '2350734': 867, '2317914': 868, '2377131': 869, '2382744': 870, '2324748': 871, '2396987': 872, '2415464': 873, '2325271': 874, '2396374': 875, '2400862': 876, '2361143': 877, '195': 878, '2379893': 879, '2339028': 880, '2349530': 881, '2400737': 882, '2318603': 883, '2352327': 884, '2416725': 885, '2377654': 886, '2362679': 887, '2394687': 888, '2386134': 889, '2336105': 890, '2364452': 891, '2390876': 892, '2373588': 893, '2374100': 894, '2385757': 895, '2414806': 896, '2326048': 897, '2365699': 898, '2371216': 899, '2357994': 900, '815': 901, '2393988': 902, '2360115': 903, '2413458': 904, '2406696': 905, '2356563': 906, '2392261': 907, '2325906': 908, '2391950': 909, '2391165': 910, '2356074': 911, '2417769': 912, '2350434': 913, '2389615': 914, '2371836': 915, '2394596': 916, '2414255': 917, '2404259': 918, '2381424': 919, '2405965': 920, '2372008': 921, '2327618': 922, '2346802': 923, '2379698': 924, '2346556': 925, '2371312': 926, '2350160': 927, '2386850': 928, '2373092': 929, '2375622': 930, '2411459': 931, '2389464': 932, '2354957': 933, '2380380': 934, '2358567': 935, '2393354': 936, '2415082': 937, '2413793': 938, '2356957': 939, '2363034': 940, '2399393': 941, '2377371': 942, '2407660': 943, '2326373': 944, '2320101': 945, '2416996': 946, '2367637': 947, '2383435': 948, '2356093': 949, '2330124': 950, '2345254': 951, '2403889': 952, '2352072': 953, '2357254': 954, '2356671': 955, '2401886': 956, '2415083': 957, '2374351': 958, '2411866': 959, '2319437': 960, '2413099': 961, '2413842': 962, '2353865': 963, '2339208': 964, '2368903': 965, '2351724': 966, '2373382': 967, '2326856': 968, '2332037': 969, '2380143': 970, '2330294': 971, '2411367': 972, '2393488': 973, '2322133': 974, '2360127': 975, '2353349': 976, '2345044': 977, '2373155': 978, '2317117': 979, '2339245': 980, '2389666': 981, '2374473': 982, '2410762': 983, '2325817': 984, '3524': 985, '2386442': 986, '2351574': 987, '2394242': 988, '2345283': 989, '2369213': 990, '2365263': 991, '2380024': 992, '2376359': 993, '2409993': 994, '2400161': 995, '2359369': 996, '2392960': 997, '2340975': 998, '2388874': 999, '2406588': 1000, '2337106': 1001, '2366203': 1002, '2417500': 1003, '2388733': 1004, '2347167': 1005, '2385460': 1006, '2370496': 1007, '2391984': 1008, '2345537': 1009, '2341245': 1010, '2402837': 1011, '2414256': 1012, '2341028': 1013, '2367884': 1014, '2336144': 1015, '2391257': 1016, '2390981': 1017, '2413997': 1018, '2411900': 1019, '2356269': 1020, '2342386': 1021, '2390189': 1022, '2399058': 1023, '2412087': 1024, '2415472': 1025, '2334735': 1026, '2373340': 1027, '2329113': 1028, '2320306': 1029, '2344088': 1030, '2350800': 1031, '2392376': 1032, '2389282': 1033, '2333242': 1034, '2380368': 1035, '2347956': 1036, '2385172': 1037, '2349036': 1038, '2360225': 1039, '2368103': 1040, '2336325': 1041, '2360287': 1042, '2336668': 1043, '2363108': 1044, '2381047': 1045, '2333962': 1046, '2410977': 1047, '2385461': 1048, '2347157': 1049, '2348401': 1050, '2402124': 1051, '2345255': 1052, '2372450': 1053, '2333467': 1054, '2341431': 1055, '2384439': 1056, '2354342': 1057, '2370343': 1058, '2324753': 1059, '2388172': 1060, '2318461': 1061, '2381263': 1062, '2377521': 1063, '2408653': 1064, '2409922': 1065, '2400672': 1066, '2317165': 1067, '2382775': 1068, '2340777': 1069, '2341785': 1070, '2361129': 1071, '2412964': 1072, '2398017': 1073, '2336760': 1074, '2377677': 1075, '2364629': 1076, '2352124': 1077, '2357723': 1078, '2357856': 1079, '2402586': 1080, '2379625': 1081, '2341187': 1082, '2394115': 1083, '2364463': 1084, '2412028': 1085, '2369707': 1086, '2394152': 1087, '2371926': 1088, '2316697': 1089, '2364742': 1090, '2355306': 1091, '2384410': 1092, '2316889': 1093, '2361855': 1094, '2336255': 1095, '2372392': 1096, '2357848': 1097, '2375873': 1098, '2341175': 1099, '2374654': 1100, '2338235': 1101, '2338501': 1102, '2387508': 1103, '2359860': 1104, '2346025': 1105, '2335228': 1106, '2404930': 1107, '2361721': 1108, '2361025': 1109, '2355690': 1110, '2336091': 1111, '2348939': 1112, '2399545': 1113, '2366433': 1114, '2379759': 1115, '2367250': 1116, '2402756': 1117, '2333815': 1118, '2413923': 1119, '2411635': 1120, '2373922': 1121, '2345388': 1122, '2381013': 1123, '2323566': 1124, '2327125': 1125, '2360148': 1126, '2341890': 1127, '2335993': 1128, '2318300': 1129, '2380510': 1130, '2411945': 1131, '2371361': 1132, '2410757': 1133, '2373934': 1134, '2360110': 1135, '2382330': 1136, '2393174': 1137, '2374097': 1138, '2322813': 1139, '2350025': 1140, '2396452': 1141, '2320897': 1142, '2346003': 1143, '2317707': 1144, '2397549': 1145, '2392557': 1146, '2389618': 1147, '2376132': 1148, '2379456': 1149, '2348675': 1150, '2366243': 1151, '2374003': 1152, '2404970': 1153, '2407026': 1154, '2412178': 1155, '2325076': 1156, '2356053': 1157, '2366552': 1158, '2387213': 1159, '2318529': 1160, '2368771': 1161, '2369653': 1162, '2387088': 1163, '2329225': 1164, '2401071': 1165, '2350763': 1166, '2398879': 1167, '2375098': 1168, '2362648': 1169, '2339034': 1170, '2381526': 1171, '2378990': 1172, '2368581': 1173, '2342463': 1174, '2410548': 1175, '2384021': 1176, '2412817': 1177, '2344622': 1178, '2356073': 1179, '2336315': 1180, '2388101': 1181, '2403142': 1182, '2396069': 1183, '2331045': 1184, '2411332': 1185, '2388879': 1186, '2357014': 1187, '2316318': 1188, '2348847': 1189, '2407133': 1190, '2359501': 1191, '2330143': 1192, '2372453': 1193, '2361523': 1194, '2320205': 1195, '2375771': 1196, '1592381': 1197, '2344878': 1198, '2352917': 1199, '2339305': 1200, '2395465': 1201, '2332111': 1202, '2346977': 1203, '2374041': 1204, '2388431': 1205, '2326536': 1206, '2402848': 1207, '2412679': 1208, '2354059': 1209, '2394933': 1210, '2369502': 1211, '2413540': 1212, '2326621': 1213, '2367096': 1214, '2341034': 1215, '2363443': 1216, '2374619': 1217, '2321253': 1218, '2342406': 1219, '2414189': 1220, '2346660': 1221, '2353466': 1222, '2351967': 1223, '2339582': 1224, '2397008': 1225, '2392194': 1226, '2406329': 1227, '2389588': 1228, '2396340': 1229, '2347072': 1230, '2378698': 1231, '2401822': 1232, '2382813': 1233, '2320614': 1234, '2322420': 1235, '2354873': 1236, '2364493': 1237, '2342919': 1238, '2416678': 1239, '2348334': 1240, '2346679': 1241, '2343220': 1242, '2351882': 1243, '2323317': 1244, '2369897': 1245, '2392583': 1246, '2408943': 1247, '2385255': 1248, '2347777': 1249, '2355399': 1250, '2378937': 1251, '2403177': 1252, '2342324': 1253, '2339891': 1254, '543': 1255, '2377454': 1256, '2355566': 1257, '2395698': 1258, '2410958': 1259, '2394635': 1260, '2371761': 1261, '2381108': 1262, '2365692': 1263, '2409873': 1264, '2407325': 1265, '2367729': 1266, '2407311': 1267, '2382252': 1268, '2403309': 1269, '2381396': 1270, '2350770': 1271, '2366958': 1272, '2346010': 1273, '2396757': 1274, '2408546': 1275, '2376235': 1276, '2354852': 1277, '2341672': 1278, '2390548': 1279, '2377272': 1280, '2344453': 1281, '2319185': 1282, '2370179': 1283, '2371763': 1284, '2371546': 1285, '2409589': 1286, '2386989': 1287, '2398489': 1288, '2390759': 1289, '2339004': 1290, '2355449': 1291, '2397683': 1292, '2319462': 1293, '2412080': 1294, '2411605': 1295, '2381761': 1296, '2395545': 1297, '2392163': 1298, '2373065': 1299, '2316025': 1300, '2380268': 1301, '2408617': 1302, '2412438': 1303, '2341668': 1304, '2336765': 1305, '2381550': 1306, '2358240': 1307, '2400462': 1308, '2316675': 1309, '2383068': 1310, '2388838': 1311, '2395479': 1312, '2390728': 1313, '2365043': 1314, '2397123': 1315, '2334742': 1316, '2412430': 1317, '2372018': 1318, '2361424': 1319, '2408144': 1320, '2404732': 1321, '2397229': 1322, '2409530': 1323, '2327406': 1324, '2400936': 1325, '2356858': 1326, '2380553': 1327, '2352140': 1328, '2319561': 1329, '2388446': 1330, '2321507': 1331, '2387187': 1332, '2358081': 1333, '2345744': 1334, '2347586': 1335, '2385725': 1336, '2407029': 1337, '2348398': 1338, '2386365': 1339, '2319419': 1340, '2398747': 1341, '2403123': 1342, '2371198': 1343, '2358999': 1344, '2374686': 1345, '2341305': 1346, '2394677': 1347, '2405763': 1348, '2381158': 1349, '2414099': 1350, '2390775': 1351, '2323158': 1352, '2407637': 1353, '2385956': 1354, '2403827': 1355, '2372426': 1356, '2397387': 1357, '2338355': 1358, '2366603': 1359, '2398565': 1360, '2355012': 1361, '2384747': 1362, '2382879': 1363, '2347702': 1364, '2385541': 1365, '2408452': 1366, '2319011': 1367, '2356436': 1368, '2410233': 1369, '2355933': 1370, '2330218': 1371, '2351858': 1372, '2338820': 1373, '2404002': 1374, '2400330': 1375, '2398037': 1376, '2345601': 1377, '2354318': 1378, '2353425': 1379, '2416937': 1380, '2336606': 1381, '2346818': 1382, '2381711': 1383, '2392202': 1384, '2354877': 1385, '2351596': 1386, '2348629': 1387, '2374316': 1388, '2381335': 1389, '2372957': 1390, '2395164': 1391, '2362840': 1392, '2387136': 1393, '2358132': 1394, '2341434': 1395, '2342864': 1396, '2400709': 1397, '2380640': 1398, '2317896': 1399, '2392065': 1400, '2394298': 1401, '2350383': 1402, '2382659': 1403, '2398866': 1404, '2331553': 1405, '2403182': 1406, '2322086': 1407, '2415884': 1408, '2332379': 1409, '2400542': 1410, '2409861': 1411, '2415978': 1412, '2356542': 1413, '2349065': 1414, '2350816': 1415, '2345625': 1416, '2402610': 1417, '2366219': 1418, '2414365': 1419, '2407808': 1420, '2387514': 1421, '2371255': 1422, '2354814': 1423, '2318159': 1424, '2377334': 1425, '2351200': 1426, '2394339': 1427, '2316574': 1428, '2374409': 1429, '2413783': 1430, '2328094': 1431, '2376076': 1432, '2411532': 1433, '2329559': 1434, '2404178': 1435, '2401816': 1436, '2404791': 1437, '2387509': 1438, '2354019': 1439, '2335492': 1440, '2316860': 1441, '2400289': 1442, '2401940': 1443, '2392014': 1444, '2367427': 1445, '2410352': 1446, '2355922': 1447, '2351483': 1448, '2395566': 1449, '2378204': 1450, '2369158': 1451, '2378294': 1452, '2343623': 1453, '2383042': 1454, '2342646': 1455, '2344284': 1456, '2349469': 1457, '2360556': 1458, '2340693': 1459, '2396197': 1460, '2354266': 1461, '2412725': 1462, '2326050': 1463, '2411637': 1464, '2319411': 1465, '1512': 1466, '2344722': 1467, '2322620': 1468, '2384101': 1469, '2406861': 1470, '2380329': 1471, '2389532': 1472, '2319274': 1473, '2411629': 1474, '2394536': 1475, '2346205': 1476, '2393422': 1477, '2392176': 1478, '2333402': 1479, '2349266': 1480, '2412056': 1481, '2369980': 1482, '2332365': 1483, '2391502': 1484, '2411500': 1485, '2407631': 1486, '1592372': 1487, '2330315': 1488, '2360855': 1489, '2371907': 1490, '2365979': 1491, '2405811': 1492, '2402053': 1493, '2396396': 1494, '2403131': 1495, '2334724': 1496, '2354544': 1497, '2400676': 1498, '2363823': 1499, '2402786': 1500, '2403431': 1501, '2398246': 1502, '2352466': 1503, '2350676': 1504, '2358414': 1505, '2331430': 1506, '2359934': 1507, '2358660': 1508, '2409730': 1509, '2384053': 1510, '2364313': 1511, '2398386': 1512, '2323413': 1513, '2397224': 1514, '2405436': 1515, '2404393': 1516, '2414493': 1517, '2405007': 1518, '2345363': 1519, '2351156': 1520, '2405703': 1521, '2391849': 1522, '2354362': 1523, '2373886': 1524, '2400392': 1525, '2371441': 1526, '2322528': 1527, '2414878': 1528, '2322397': 1529, '2357096': 1530, '2371625': 1531, '2377951': 1532, '2384475': 1533, '2333993': 1534, '2347274': 1535, '2401035': 1536, '2370296': 1537, '2380874': 1538, '2360405': 1539, '2346066': 1540, '2392507': 1541, '2358834': 1542, '2373821': 1543, '2388891': 1544, '2342230': 1545, '2410117': 1546, '2383962': 1547, '2417944': 1548, '2405504': 1549, '2348951': 1550, '2397307': 1551, '2409964': 1552, '2403749': 1553, '2417815': 1554, '2357770': 1555, '2410491': 1556, '2348208': 1557, '2390517': 1558, '2408377': 1559, '2410270': 1560, '2341363': 1561, '2383665': 1562, '2390619': 1563, '2380617': 1564, '2318253': 1565, '2361015': 1566, '2407563': 1567, '2357859': 1568, '2321198': 1569, '2401910': 1570, '2345382': 1571, '2394967': 1572, '2349108': 1573, '853': 1574, '2317625': 1575, '2408708': 1576, '2349353': 1577, '2376251': 1578, '2371551': 1579, '2348278': 1580, '2415457': 1581, '2316887': 1582, '2403418': 1583, '2343155': 1584, '2393166': 1585, '2343948': 1586, '2396669': 1587, '2360573': 1588, '2367727': 1589, '2344540': 1590, '2354147': 1591, '2353938': 1592, '2364799': 1593, '2414694': 1594, '2379974': 1595, '2359027': 1596, '2381416': 1597, '2389036': 1598, '2392765': 1599, '2394250': 1600, '2396407': 1601, '2399390': 1602, '2327810': 1603, '2375529': 1604, '2372145': 1605, '2389638': 1606, '2387786': 1607, '2382666': 1608, '2394823': 1609, '2391902': 1610, '2380908': 1611, '2370342': 1612, '2365664': 1613, '2362607': 1614, '2383586': 1615, '2405682': 1616, '2411960': 1617, '2389597': 1618, '2379832': 1619, '2319434': 1620, '2346718': 1621, '2405292': 1622, '2345464': 1623, '498337': 1624, '2412032': 1625, '2397030': 1626, '2357052': 1627, '2360201': 1628, '2359041': 1629, '2409746': 1630, '2328180': 1631, '2415173': 1632, '2369188': 1633, '2410088': 1634, '2352259': 1635, '2377573': 1636, '2364664': 1637, '2345826': 1638, '2360637': 1639, '2369615': 1640, '2382188': 1641, '2361538': 1642, '2390131': 1643, '2564': 1644, '2415473': 1645, '2345963': 1646, '2364711': 1647, '2374001': 1648, '2409142': 1649, '2355186': 1650, '2359734': 1651, '2377897': 1652, '2410775': 1653, '2355527': 1654, '2410305': 1655, '2317409': 1656, '2365687': 1657, '2331127': 1658, '2336558': 1659, '2411311': 1660, '2343876': 1661, '2398408': 1662, '2371017': 1663, '2367175': 1664, '2366378': 1665, '2330167': 1666, '2375133': 1667, '2399464': 1668, '2377832': 1669, '2347763': 1670, '2374300': 1671, '2369292': 1672, '2399932': 1673, '2344295': 1674, '2382977': 1675, '2318313': 1676, '2411813': 1677, '2343356': 1678, '2384658': 1679, '2395463': 1680, '2398544': 1681, '2352643': 1682, '2376750': 1683, '2403306': 1684, '2377053': 1685, '2329100': 1686, '4526': 1687, '2351834': 1688, '2390500': 1689, '2370551': 1690, '2355876': 1691, '2333581': 1692, '2401969': 1693, '2405056': 1694, '2364869': 1695, '2350985': 1696, '2408983': 1697, '2415510': 1698, '2347477': 1699, '2390282': 1700, '2344595': 1701, '2394297': 1702, '2391397': 1703, '2400181': 1704, '2387933': 1705, '2337581': 1706, '2342464': 1707, '2375898': 1708, '2333227': 1709, '2384614': 1710, '2363103': 1711, '61515': 1712, '2410276': 1713, '2387370': 1714, '2372279': 1715, '2343554': 1716, '2408872': 1717, '2403553': 1718, '2342857': 1719, '2353270': 1720, '2385349': 1721, '2318479': 1722, '2373822': 1723, '2385693': 1724, '2335572': 1725, '2401818': 1726, '2341844': 1727, '2415192': 1728, '2372030': 1729, '2346365': 1730, '2356290': 1731, '2410913': 1732, '2343589': 1733, '2413030': 1734, '2387651': 1735, '2341390': 1736, '2370165': 1737, '2367980': 1738, '2410681': 1739, '2387944': 1740, '2364952': 1741, '2364830': 1742, '2410500': 1743, '2388187': 1744, '2379551': 1745, '2374375': 1746, '2410345': 1747, '2352132': 1748, '2316535': 1749, '2387583': 1750, '2400705': 1751, '2400198': 1752, '2356774': 1753, '2344294': 1754, '2399862': 1755, '2355172': 1756, '2369580': 1757, '2368472': 1758, '2393843': 1759, '2386965': 1760, '2346626': 1761, '2317055': 1762, '2389880': 1763, '2344880': 1764, '2373460': 1765, '2405096': 1766, '2336832': 1767, '2396514': 1768, '2353172': 1769, '2405568': 1770, '2388006': 1771, '2361585': 1772, '2377072': 1773, '2352426': 1774, '2384368': 1775, '2405981': 1776, '2366660': 1777, '2344000': 1778, '2396968': 1779, '2342960': 1780, '2411125': 1781, '2355581': 1782, '2411121': 1783, '2393513': 1784, '2365691': 1785, '2400579': 1786, '2371228': 1787, '2317860': 1788, '2319593': 1789, '2354526': 1790, '2378517': 1791, '2316254': 1792, '2410070': 1793, '1159516': 1794, '2358293': 1795, '2318856': 1796, '2372039': 1797, '2406319': 1798, '2350783': 1799, '2359238': 1800, '2398991': 1801, '2345484': 1802, '2363592': 1803, '2368524': 1804, '2315529': 1805, '2360315': 1806, '2324220': 1807, '2408638': 1808, '2318354': 1809, '2413457': 1810, '2399569': 1811, '2345476': 1812, '2365226': 1813, '2402050': 1814, '2343057': 1815, '2382441': 1816, '2373021': 1817, '2382173': 1818, '2382632': 1819, '2345621': 1820, '2362731': 1821, '2379040': 1822, '2323848': 1823, '2405490': 1824, '2401986': 1825, '2393778': 1826, '2382524': 1827, '2340615': 1828, '2406927': 1829, '2378480': 1830, '2389642': 1831, '2376475': 1832, '2354169': 1833, '2361554': 1834, '2326672': 1835, '2394057': 1836, '2388526': 1837, '2406376': 1838, '2375076': 1839, '2374809': 1840, '2331362': 1841, '2392123': 1842, '2397845': 1843, '2332173': 1844, '2402155': 1845, '2349448': 1846, '2381847': 1847, '2390911': 1848, '2379368': 1849, '2393134': 1850, '2394007': 1851, '2362198': 1852, '2340394': 1853, '2390099': 1854, '2374858': 1855, '2413411': 1856, '2384283': 1857, '2361208': 1858, '2346654': 1859, '2342062': 1860, '2366424': 1861, '2318214': 1862, '2387531': 1863, '2317457': 1864, '2400046': 1865, '2379797': 1866, '2371733': 1867, '2396351': 1868, '2367180': 1869, '2401819': 1870, '2363548': 1871, '2351789': 1872, '2336180': 1873, '2342649': 1874, '2403572': 1875, '2414803': 1876, '2343586': 1877, '2367805': 1878, '2399954': 1879, '2416836': 1880, '2365803': 1881, '2357249': 1882, '2350678': 1883, '2412691': 1884, '2385897': 1885, '2358578': 1886, '2322107': 1887, '2375389': 1888, '2408481': 1889, '2383497': 1890, '2411830': 1891, '2360483': 1892, '2381777': 1893, '2354093': 1894, '2387137': 1895, '2400665': 1896, '2374256': 1897, '2373225': 1898, '2352059': 1899, '2338341': 1900, '2338001': 1901, '2410181': 1902, '2330919': 1903, '2382039': 1904, '2392600': 1905, '2376247': 1906, '2413702': 1907, '2344077': 1908, '2382974': 1909, '2325024': 1910, '2345290': 1911, '2392468': 1912, '2382722': 1913, '2407293': 1914, '2392613': 1915, '2363437': 1916, '2409165': 1917, '2409238': 1918, '2347170': 1919, '2400907': 1920, '2395914': 1921, '2393093': 1922, '2345716': 1923, '2409995': 1924, '2362360': 1925, '2410413': 1926, '2358566': 1927, '2411064': 1928, '2400488': 1929, '2402864': 1930, '2318784': 1931, '2410083': 1932, '2346215': 1933, '2397756': 1934, '2410111': 1935, '2382522': 1936, '2363057': 1937, '2406213': 1938, '2390476': 1939, '2399831': 1940, '2409738': 1941, '2349149': 1942, '2389587': 1943, '2373951': 1944, '2504': 1945, '2393995': 1946, '2396179': 1947, '2371338': 1948, '2385571': 1949, '2336159': 1950, '2371432': 1951, '2383269': 1952, '2368666': 1953, '2344637': 1954, '2373063': 1955, '2391194': 1956, '2386673': 1957, '2320078': 1958, '2391461': 1959, '2372285': 1960, '2379258': 1961, '2341514': 1962, '2318841': 1963, '2386678': 1964, '2400689': 1965, '2341725': 1966, '2376970': 1967, '2406500': 1968, '2357507': 1969, '2396448': 1970, '2395630': 1971, '2390890': 1972, '2344135': 1973, '2385174': 1974, '2384854': 1975, '2413861': 1976, '2386495': 1977, '2374212': 1978, '2393279': 1979, '2382690': 1980, '2403866': 1981, '2399949': 1982, '285802': 1983, '2400680': 1984, '2399996': 1985, '2356573': 1986, '2416386': 1987, '2344117': 1988, '2408673': 1989, '2416167': 1990, '2391375': 1991, '2385815': 1992, '2384790': 1993, '2317122': 1994, '2387553': 1995, '2349652': 1996, '2363110': 1997, '2382543': 1998, '2318426': 1999, '2390084': 2000, '2318528': 2001, '2364598': 2002, '2412966': 2003, '2369601': 2004, '2396112': 2005, '150344': 2006, '2333510': 2007, '2324573': 2008, '2415027': 2009, '2363104': 2010, '2372733': 2011, '587': 2012, '2357502': 2013, '2389782': 2014, '2387258': 2015, '2334930': 2016, '2352992': 2017, '2345982': 2018, '2346439': 2019, '2392581': 2020, '2345262': 2021, '2388166': 2022, '2375625': 2023, '2351511': 2024, '2408359': 2025, '2391869': 2026, '2392818': 2027, '2369855': 2028, '2327758': 2029, '2409153': 2030, '2399525': 2031, '2415676': 2032, '2400518': 2033, '2387989': 2034, '2379307': 2035, '2363501': 2036, '2362555': 2037, '2323209': 2038, '2359589': 2039, '2378223': 2040, '2391252': 2041, '2316382': 2042, '2367052': 2043, '2359190': 2044, '358': 2045, '2328602': 2046, '2318174': 2047, '2408468': 2048, '2355642': 2049, '1067': 2050, '2352832': 2051, '2320127': 2052, '2402825': 2053, '2394911': 2054, '2405788': 2055, '2383081': 2056, '2353014': 2057, '2356926': 2058, '2341770': 2059, '2354589': 2060, '2319025': 2061, '2386258': 2062, '2361204': 2063, '2343537': 2064, '2317743': 2065, '2409948': 2066, '2355213': 2067, '2407074': 2068, '2327559': 2069, '2370652': 2070, '2392451': 2071, '2403297': 2072, '2328024': 2073, '1947': 2074, '2382933': 2075, '2380071': 2076, '2386619': 2077, '2343802': 2078, '2374709': 2079, '2395037': 2080, '2318491': 2081, '2393164': 2082, '2366570': 2083, '2383358': 2084, '2363661': 2085, '2401897': 2086, '2323005': 2087, '2318753': 2088, '2402202': 2089, '2399182': 2090, '2380444': 2091, '2415776': 2092, '2410116': 2093, '2405963': 2094, '2408407': 2095, '2414540': 2096, '2408480': 2097, '2392500': 2098, '2368332': 2099, '2340505': 2100, '2393496': 2101, '2365504': 2102, '2340547': 2103, '2361489': 2104, '2342678': 2105, '2401336': 2106, '2348566': 2107, '2414418': 2108, '2358790': 2109, '2404467': 2110, '724': 2111, '2399843': 2112, '2405738': 2113, '2393060': 2114, '2385216': 2115, '2370877': 2116, '2350752': 2117, '2343739': 2118, '2380279': 2119, '2379332': 2120, '2366798': 2121, '2406548': 2122, '2364648': 2123, '2320207': 2124, '2398212': 2125, '2375241': 2126, '2408244': 2127, '2393002': 2128, '2364458': 2129, '2364569': 2130, '2325414': 2131, '2374807': 2132, '2353337': 2133, '2384906': 2134, '2390945': 2135, '2336229': 2136, '2349203': 2137, '2368261': 2138, '2345893': 2139, '2359948': 2140, '2398222': 2141, '2396670': 2142, '2324375': 2143, '2399556': 2144, '2406733': 2145, '2385986': 2146, '2361991': 2147, '2389426': 2148, '2343612': 2149, '2360527': 2150, '2342329': 2151, '2341047': 2152, '2367875': 2153, '2334644': 2154, '2415363': 2155, '2392812': 2156, '2389754': 2157, '2352853': 2158, '2339689': 2159, '2395881': 2160, '2367978': 2161, '2365490': 2162, '2343596': 2163, '2333164': 2164, '2367144': 2165, '2350140': 2166, '2382213': 2167, '2315963': 2168, '2406648': 2169, '2366650': 2170, '2344341': 2171, '2323588': 2172, '2370697': 2173, '2410146': 2174, '2395696': 2175, '2335504': 2176, '2362808': 2177, '2391256': 2178, '2403683': 2179, '2410297': 2180, '2319554': 2181, '2333255': 2182, '2339046': 2183, '2413904': 2184, '2411941': 2185, '2317592': 2186, '2346843': 2187, '262': 2188, '2317319': 2189, '2412901': 2190, '2391111': 2191, '2394454': 2192, '2392869': 2193, '2400554': 2194, '59': 2195, '2373368': 2196, '2344661': 2197, '2406056': 2198, '2389172': 2199, '2354660': 2200, '2334516': 2201, '2407120': 2202, '2368870': 2203, '2403096': 2204, '2360943': 2205, '2338460': 2206, '2414226': 2207, '2412833': 2208, '2360125': 2209, '2363840': 2210, '2321324': 2211, '2400079': 2212, '2396864': 2213, '2404541': 2214, '2343376': 2215, '2344370': 2216, '2330999': 2217, '2414157': 2218, '2366795': 2219, '2366302': 2220, '2354637': 2221, '2398486': 2222, '2338878': 2223, '2402374': 2224, '2348627': 2225, '2389270': 2226, '2372029': 2227, '2400643': 2228, '2355709': 2229, '2362311': 2230, '2344881': 2231, '2379480': 2232, '2329101': 2233, '2373700': 2234, '2404918': 2235, '2348172': 2236, '2356669': 2237, '2354564': 2238, '2397809': 2239, '2346342': 2240, '2361074': 2241, '2377368': 2242, '2317513': 2243, '2378059': 2244, '2340180': 2245, '2317591': 2246, '2366127': 2247, '2390288': 2248, '2396390': 2249, '2335223': 2250, '2341640': 2251, '2402546': 2252, '2351277': 2253, '2374760': 2254, '2391209': 2255, '2316729': 2256, '2317053': 2257, '2350520': 2258, '2327364': 2259, '2386260': 2260, '2338004': 2261, '2394482': 2262, '2347783': 2263, '2414805': 2264, '2378686': 2265, '2397605': 2266, '2387464': 2267, '2335457': 2268, '2350362': 2269, '2355111': 2270, '2343118': 2271, '2409065': 2272, '2357554': 2273, '2355118': 2274, '2383588': 2275, '2404485': 2276, '2317038': 2277, '2337365': 2278, '2395949': 2279, '2359272': 2280, '2382849': 2281, '2373510': 2282, '2363267': 2283, '2387911': 2284, '2369971': 2285, '2381902': 2286, '2336048': 2287, '2401096': 2288, '2412694': 2289, '2393524': 2290, '2371744': 2291, '2339578': 2292, '2369882': 2293, '2364907': 2294, '2376391': 2295, '2399359': 2296, '2330962': 2297, '2376356': 2298, '2364516': 2299, '2411640': 2300, '2380488': 2301, '2357038': 2302, '2320356': 2303, '2330639': 2304, '2360200': 2305, '2397120': 2306, '2390392': 2307, '2346419': 2308, '2396290': 2309, '2358575': 2310, '2343618': 2311, '2376371': 2312, '2387761': 2313, '2390032': 2314, '2403502': 2315, '2334281': 2316, '2382381': 2317, '2407666': 2318, '2317228': 2319, '2391722': 2320, '2376016': 2321, '2322573': 2322, '2336110': 2323, '2395507': 2324, '2331963': 2325, '2395930': 2326, '2356795': 2327, '2367957': 2328, '2342128': 2329, '2358287': 2330, '2395628': 2331, '2369455': 2332, '2380117': 2333, '2385072': 2334, '2338012': 2335, '2347982': 2336, '2362130': 2337, '2378490': 2338, '2364094': 2339, '2372176': 2340, '2389127': 2341, '2366148': 2342, '2338443': 2343, '2343766': 2344, '2385053': 2345, '2346498': 2346, '2416307': 2347, '2356329': 2348, '2407240': 2349, '2315640': 2350, '2366855': 2351, '2386983': 2352, '2403565': 2353, '2365595': 2354, '2386870': 2355, '2367983': 2356, '2387207': 2357, '2413134': 2358, '2359636': 2359, '2341700': 2360, '2347316': 2361, '2377986': 2362, '2390394': 2363, '2382975': 2364, '2411955': 2365, '2352243': 2366, '2396465': 2367, '2391047': 2368, '2335091': 2369, '2355583': 2370, '2405444': 2371, '2339483': 2372, '2357873': 2373, '2341383': 2374, '2347742': 2375, '2401129': 2376, '2342125': 2377, '2414784': 2378, '1159963': 2379, '2349488': 2380, '2405390': 2381, '2384500': 2382, '2366640': 2383, '2382714': 2384, '2410358': 2385, '2399749': 2386, '2376467': 2387, '2340926': 2388, '2361164': 2389, '2396478': 2390, '2348265': 2391, '2391182': 2392, '2362917': 2393, '2354590': 2394, '2408454': 2395, '2337033': 2396, '2371418': 2397, '2411249': 2398, '2387946': 2399, '2405966': 2400, '2397831': 2401, '2377290': 2402, '2394457': 2403, '2414173': 2404, '2350934': 2405, '2379419': 2406, '2408594': 2407, '2386847': 2408, '2415196': 2409, '2388316': 2410, '2413037': 2411, '2384670': 2412, '2346974': 2413, '2342078': 2414, '2387721': 2415, '2375570': 2416, '2338625': 2417, '2348008': 2418, '2352560': 2419, '2369013': 2420, '2369937': 2421, '2317478': 2422, '2332609': 2423, '2362277': 2424, '2408209': 2425, '2388111': 2426, '2410852': 2427, '2385025': 2428, '2377512': 2429, '2347040': 2430, '2366432': 2431, '2335275': 2432, '2316204': 2433, '2396682': 2434, '2375612': 2435, '2414069': 2436, '2355641': 2437, '2379154': 2438, '813': 2439, '2407413': 2440, '2323665': 2441, '2398578': 2442, '2396029': 2443, '2391545': 2444, '2346922': 2445, '2389002': 2446, '2340767': 2447, '2342976': 2448, '2374164': 2449, '2321676': 2450, '2403291': 2451, '2400108': 2452, '2375753': 2453, '2341674': 2454, '2377456': 2455, '2377161': 2456, '2376001': 2457, '2371756': 2458, '2365457': 2459, '2341703': 2460, '2405776': 2461, '2361275': 2462, '2352062': 2463, '2386491': 2464, '2408041': 2465, '2351669': 2466, '2413323': 2467, '2406451': 2468, '2393764': 2469, '2377815': 2470, '2356592': 2471, '2385066': 2472, '2334363': 2473, '2364508': 2474, '2364582': 2475, '2378366': 2476, '2407349': 2477, '2362280': 2478, '2371998': 2479, '2397147': 2480, '2343805': 2481, '2358223': 2482, '2378060': 2483, '2347102': 2484, '2402174': 2485, '2316661': 2486}\n",
      "Before epoch loop->>>>>>>>>>>>>>>>>>>>>>>>>>>>> the len is 113\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pretrained model loaded successfully\n",
      "Start and end (0, 5)\n",
      "idx:  0  Epoch:  113  loss:  380.99615  loss_sent:  3.8504834  loss_word:  361.7437  Time cost:  87.46882843971252\n",
      "Start and end (5, 10)\n",
      "idx:  5  Epoch:  113  loss:  339.97424  loss_sent:  3.4248228  loss_word:  322.85013  Time cost:  7.107832908630371\n",
      "Start and end (10, 15)\n",
      "idx:  10  Epoch:  113  loss:  416.8274  loss_sent:  3.8823729  loss_word:  397.4155  Time cost:  6.906096935272217\n",
      "Start and end (15, 20)\n",
      "idx:  15  Epoch:  113  loss:  416.82837  loss_sent:  4.090528  loss_word:  396.3757  Time cost:  6.872644424438477\n",
      "Start and end (20, 25)\n",
      "idx:  20  Epoch:  113  loss:  376.28152  loss_sent:  3.0152397  loss_word:  361.20535  Time cost:  7.041353225708008\n",
      "Start and end (25, 30)\n",
      "idx:  25  Epoch:  113  loss:  377.1209  loss_sent:  2.7390094  loss_word:  363.42587  Time cost:  8.141240358352661\n",
      "Start and end (30, 35)\n",
      "idx:  30  Epoch:  113  loss:  361.34146  loss_sent:  4.139288  loss_word:  340.64502  Time cost:  7.06324028968811\n",
      "Start and end (35, 40)\n",
      "idx:  35  Epoch:  113  loss:  281.3978  loss_sent:  3.0105844  loss_word:  266.34488  Time cost:  7.159361839294434\n",
      "Start and end (40, 45)\n",
      "idx:  40  Epoch:  113  loss:  316.92932  loss_sent:  4.8020663  loss_word:  292.91895  Time cost:  7.334784746170044\n",
      "Start and end (45, 50)\n",
      "idx:  45  Epoch:  113  loss:  326.54828  loss_sent:  5.179779  loss_word:  300.64932  Time cost:  7.145821571350098\n",
      "Start and end (50, 55)\n",
      "idx:  50  Epoch:  113  loss:  367.4918  loss_sent:  3.2097082  loss_word:  351.44324  Time cost:  6.945405960083008\n",
      "Start and end (55, 60)\n",
      "idx:  55  Epoch:  113  loss:  334.11188  loss_sent:  3.2515886  loss_word:  317.85394  Time cost:  6.730547666549683\n",
      "Start and end (60, 65)\n",
      "idx:  60  Epoch:  113  loss:  283.84097  loss_sent:  3.3849413  loss_word:  266.91626  Time cost:  6.6678783893585205\n",
      "Start and end (65, 70)\n",
      "idx:  65  Epoch:  113  loss:  291.09363  loss_sent:  3.6049273  loss_word:  273.0689  Time cost:  6.710900068283081\n",
      "Start and end (70, 75)\n",
      "idx:  70  Epoch:  113  loss:  461.16385  loss_sent:  3.761286  loss_word:  442.35748  Time cost:  6.633038520812988\n",
      "Start and end (75, 80)\n",
      "idx:  75  Epoch:  113  loss:  417.99472  loss_sent:  3.8238165  loss_word:  398.87564  Time cost:  7.395177364349365\n",
      "Start and end (80, 85)\n",
      "idx:  80  Epoch:  113  loss:  338.79047  loss_sent:  3.610502  loss_word:  320.738  Time cost:  7.3259875774383545\n",
      "Start and end (85, 90)\n",
      "idx:  85  Epoch:  113  loss:  465.3672  loss_sent:  3.5674136  loss_word:  447.53012  Time cost:  7.12952995300293\n",
      "Start and end (90, 95)\n",
      "idx:  90  Epoch:  113  loss:  281.86465  loss_sent:  3.453714  loss_word:  264.59604  Time cost:  7.027120351791382\n",
      "Start and end (95, 100)\n",
      "idx:  95  Epoch:  113  loss:  390.1651  loss_sent:  2.506027  loss_word:  377.63495  Time cost:  7.132840633392334\n",
      "Start and end (100, 105)\n",
      "idx:  100  Epoch:  113  loss:  431.3752  loss_sent:  3.2052834  loss_word:  415.34875  Time cost:  7.468597650527954\n",
      "Start and end (105, 110)\n",
      "idx:  105  Epoch:  113  loss:  387.98407  loss_sent:  3.9729114  loss_word:  368.1195  Time cost:  7.51854944229126\n",
      "Start and end (110, 115)\n",
      "idx:  110  Epoch:  113  loss:  365.38693  loss_sent:  2.560881  loss_word:  352.58252  Time cost:  7.013994455337524\n",
      "Start and end (115, 120)\n",
      "idx:  115  Epoch:  113  loss:  349.42313  loss_sent:  3.1166656  loss_word:  333.8398  Time cost:  7.065019607543945\n",
      "Start and end (120, 125)\n",
      "idx:  120  Epoch:  113  loss:  339.97815  loss_sent:  2.6816125  loss_word:  326.57007  Time cost:  7.607052326202393\n",
      "Start and end (125, 130)\n",
      "idx:  125  Epoch:  113  loss:  327.84048  loss_sent:  3.5598094  loss_word:  310.04144  Time cost:  6.496639251708984\n",
      "Start and end (130, 135)\n",
      "idx:  130  Epoch:  113  loss:  413.876  loss_sent:  3.0390644  loss_word:  398.68073  Time cost:  6.919837713241577\n",
      "Start and end (135, 140)\n",
      "idx:  135  Epoch:  113  loss:  436.22623  loss_sent:  2.4656181  loss_word:  423.89813  Time cost:  6.848836660385132\n",
      "Start and end (140, 145)\n",
      "idx:  140  Epoch:  113  loss:  509.92212  loss_sent:  2.4731908  loss_word:  497.55618  Time cost:  6.65267014503479\n",
      "Start and end (145, 150)\n",
      "idx:  145  Epoch:  113  loss:  277.07544  loss_sent:  3.4158826  loss_word:  259.99603  Time cost:  6.851868152618408\n",
      "Start and end (150, 155)\n",
      "idx:  150  Epoch:  113  loss:  360.0161  loss_sent:  2.3578281  loss_word:  348.22696  Time cost:  6.486953973770142\n",
      "Start and end (155, 160)\n",
      "idx:  155  Epoch:  113  loss:  326.5493  loss_sent:  2.6130004  loss_word:  313.48428  Time cost:  6.475950479507446\n",
      "Start and end (160, 165)\n",
      "idx:  160  Epoch:  113  loss:  350.10812  loss_sent:  4.982674  loss_word:  325.19476  Time cost:  6.774385452270508\n",
      "Start and end (165, 170)\n",
      "idx:  165  Epoch:  113  loss:  357.3426  loss_sent:  3.3530922  loss_word:  340.57718  Time cost:  6.644436836242676\n",
      "Start and end (170, 175)\n",
      "idx:  170  Epoch:  113  loss:  327.35156  loss_sent:  3.2604055  loss_word:  311.04956  Time cost:  6.735434055328369\n",
      "Start and end (175, 180)\n",
      "idx:  175  Epoch:  113  loss:  386.09363  loss_sent:  2.8018763  loss_word:  372.08423  Time cost:  6.6422059535980225\n",
      "Start and end (180, 185)\n",
      "idx:  180  Epoch:  113  loss:  403.2226  loss_sent:  2.4649246  loss_word:  390.89798  Time cost:  7.191210746765137\n",
      "Start and end (185, 190)\n",
      "idx:  185  Epoch:  113  loss:  371.17377  loss_sent:  2.4059324  loss_word:  359.14417  Time cost:  7.626727342605591\n",
      "Start and end (190, 195)\n",
      "idx:  190  Epoch:  113  loss:  246.36403  loss_sent:  2.7781348  loss_word:  232.47336  Time cost:  7.711175441741943\n",
      "Start and end (195, 200)\n",
      "idx:  195  Epoch:  113  loss:  314.7724  loss_sent:  2.7923467  loss_word:  300.81067  Time cost:  7.250460624694824\n",
      "Start and end (200, 205)\n",
      "idx:  200  Epoch:  113  loss:  343.5494  loss_sent:  4.3182855  loss_word:  321.95798  Time cost:  6.759821891784668\n",
      "Start and end (205, 210)\n",
      "idx:  205  Epoch:  113  loss:  307.51514  loss_sent:  3.2843418  loss_word:  291.0934  Time cost:  6.654387712478638\n",
      "Start and end (210, 215)\n",
      "idx:  210  Epoch:  113  loss:  313.21164  loss_sent:  2.984654  loss_word:  298.2884  Time cost:  6.613433599472046\n",
      "Start and end (215, 220)\n",
      "idx:  215  Epoch:  113  loss:  292.1781  loss_sent:  3.4629421  loss_word:  274.8634  Time cost:  6.874276876449585\n",
      "Start and end (220, 225)\n",
      "idx:  220  Epoch:  113  loss:  204.01135  loss_sent:  2.9080224  loss_word:  189.47125  Time cost:  6.7323317527771\n",
      "Start and end (225, 230)\n",
      "idx:  225  Epoch:  113  loss:  379.14178  loss_sent:  3.5251687  loss_word:  361.516  Time cost:  6.873603582382202\n",
      "Start and end (230, 235)\n",
      "idx:  230  Epoch:  113  loss:  394.47195  loss_sent:  2.9413877  loss_word:  379.765  Time cost:  6.737966060638428\n",
      "Start and end (235, 240)\n",
      "idx:  235  Epoch:  113  loss:  288.6796  loss_sent:  2.4553802  loss_word:  276.40274  Time cost:  6.831692457199097\n",
      "Start and end (240, 245)\n",
      "idx:  240  Epoch:  113  loss:  306.33582  loss_sent:  2.4139695  loss_word:  294.266  Time cost:  6.93682599067688\n",
      "Start and end (245, 250)\n",
      "idx:  245  Epoch:  113  loss:  363.9012  loss_sent:  3.651839  loss_word:  345.64203  Time cost:  6.989392995834351\n",
      "Start and end (250, 255)\n",
      "idx:  250  Epoch:  113  loss:  260.17496  loss_sent:  3.581883  loss_word:  242.26558  Time cost:  6.960269451141357\n",
      "Start and end (255, 260)\n",
      "idx:  255  Epoch:  113  loss:  349.62393  loss_sent:  3.2388475  loss_word:  333.42966  Time cost:  6.704581022262573\n",
      "Start and end (260, 265)\n",
      "idx:  260  Epoch:  113  loss:  333.90033  loss_sent:  3.4498982  loss_word:  316.65082  Time cost:  7.270569086074829\n",
      "Start and end (265, 270)\n",
      "idx:  265  Epoch:  113  loss:  229.26172  loss_sent:  4.633541  loss_word:  206.09402  Time cost:  6.906329870223999\n",
      "Start and end (270, 275)\n",
      "idx:  270  Epoch:  113  loss:  294.84964  loss_sent:  3.3531847  loss_word:  278.08368  Time cost:  6.926953315734863\n",
      "Start and end (275, 280)\n",
      "idx:  275  Epoch:  113  loss:  342.35138  loss_sent:  3.004678  loss_word:  327.32794  Time cost:  7.161679744720459\n",
      "Start and end (280, 285)\n",
      "idx:  280  Epoch:  113  loss:  253.78255  loss_sent:  3.265201  loss_word:  237.45654  Time cost:  6.81896448135376\n",
      "Start and end (285, 290)\n",
      "idx:  285  Epoch:  113  loss:  339.50693  loss_sent:  2.8467228  loss_word:  325.2733  Time cost:  7.1118693351745605\n",
      "Start and end (290, 295)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  290  Epoch:  113  loss:  298.66086  loss_sent:  3.0648222  loss_word:  283.33676  Time cost:  6.631594181060791\n",
      "Start and end (295, 300)\n",
      "idx:  295  Epoch:  113  loss:  343.85715  loss_sent:  2.7962568  loss_word:  329.87585  Time cost:  6.839798927307129\n",
      "Start and end (300, 305)\n",
      "idx:  300  Epoch:  113  loss:  330.5538  loss_sent:  3.1283553  loss_word:  314.91202  Time cost:  6.768357515335083\n",
      "Start and end (305, 310)\n",
      "idx:  305  Epoch:  113  loss:  291.88242  loss_sent:  3.714604  loss_word:  273.3094  Time cost:  6.733511447906494\n",
      "Start and end (310, 315)\n",
      "idx:  310  Epoch:  113  loss:  316.52112  loss_sent:  3.52972  loss_word:  298.8725  Time cost:  6.789605379104614\n",
      "Start and end (315, 320)\n",
      "idx:  315  Epoch:  113  loss:  352.06332  loss_sent:  3.2165115  loss_word:  335.9808  Time cost:  6.6825456619262695\n",
      "Start and end (320, 325)\n",
      "idx:  320  Epoch:  113  loss:  286.21902  loss_sent:  2.4085052  loss_word:  274.17648  Time cost:  6.680880308151245\n",
      "Start and end (325, 330)\n",
      "idx:  325  Epoch:  113  loss:  277.4311  loss_sent:  2.5454211  loss_word:  264.70398  Time cost:  6.649538516998291\n",
      "Start and end (330, 335)\n",
      "idx:  330  Epoch:  113  loss:  311.0  loss_sent:  3.4703856  loss_word:  293.64807  Time cost:  6.79263973236084\n",
      "Start and end (335, 340)\n",
      "idx:  335  Epoch:  113  loss:  312.9864  loss_sent:  2.684633  loss_word:  299.56317  Time cost:  6.658918142318726\n",
      "Start and end (340, 345)\n",
      "idx:  340  Epoch:  113  loss:  417.79  loss_sent:  3.4471188  loss_word:  400.55444  Time cost:  6.501842975616455\n",
      "Start and end (345, 350)\n",
      "idx:  345  Epoch:  113  loss:  310.39966  loss_sent:  3.2408228  loss_word:  294.19553  Time cost:  6.653593063354492\n",
      "Start and end (350, 355)\n",
      "idx:  350  Epoch:  113  loss:  377.94388  loss_sent:  3.4139273  loss_word:  360.87424  Time cost:  6.662992715835571\n",
      "Start and end (355, 360)\n",
      "idx:  355  Epoch:  113  loss:  261.51025  loss_sent:  2.2442088  loss_word:  250.2892  Time cost:  6.7733824253082275\n",
      "Start and end (360, 365)\n",
      "idx:  360  Epoch:  113  loss:  258.3134  loss_sent:  2.901619  loss_word:  243.80528  Time cost:  6.615208148956299\n",
      "Start and end (365, 370)\n",
      "idx:  365  Epoch:  113  loss:  302.10294  loss_sent:  4.78052  loss_word:  278.20035  Time cost:  6.532532453536987\n",
      "Start and end (370, 375)\n",
      "idx:  370  Epoch:  113  loss:  265.20483  loss_sent:  3.3727522  loss_word:  248.34102  Time cost:  6.655355930328369\n",
      "Start and end (375, 380)\n",
      "idx:  375  Epoch:  113  loss:  372.6636  loss_sent:  2.727838  loss_word:  359.02438  Time cost:  6.591959238052368\n",
      "Start and end (380, 385)\n",
      "idx:  380  Epoch:  113  loss:  376.00116  loss_sent:  3.7717686  loss_word:  357.14236  Time cost:  6.739198207855225\n",
      "Start and end (385, 390)\n",
      "idx:  385  Epoch:  113  loss:  388.61792  loss_sent:  2.9840236  loss_word:  373.69778  Time cost:  6.53820013999939\n",
      "Start and end (390, 395)\n",
      "idx:  390  Epoch:  113  loss:  406.78775  loss_sent:  2.908192  loss_word:  392.2468  Time cost:  6.649945020675659\n",
      "Start and end (395, 400)\n",
      "idx:  395  Epoch:  113  loss:  308.24585  loss_sent:  2.8885317  loss_word:  293.8032  Time cost:  6.561155796051025\n",
      "Start and end (400, 405)\n",
      "idx:  400  Epoch:  113  loss:  310.84958  loss_sent:  2.8654828  loss_word:  296.52213  Time cost:  6.859793663024902\n",
      "Start and end (405, 410)\n",
      "idx:  405  Epoch:  113  loss:  312.54434  loss_sent:  2.74745  loss_word:  298.80713  Time cost:  7.653229236602783\n",
      "Start and end (410, 415)\n",
      "idx:  410  Epoch:  113  loss:  281.6821  loss_sent:  3.5920374  loss_word:  263.72195  Time cost:  6.907472848892212\n",
      "Start and end (415, 420)\n",
      "idx:  415  Epoch:  113  loss:  288.91507  loss_sent:  2.405551  loss_word:  276.8873  Time cost:  7.15584135055542\n",
      "Start and end (420, 425)\n",
      "idx:  420  Epoch:  113  loss:  298.30362  loss_sent:  2.698319  loss_word:  284.812  Time cost:  7.143011569976807\n",
      "Start and end (425, 430)\n",
      "idx:  425  Epoch:  113  loss:  298.81992  loss_sent:  2.777613  loss_word:  284.9319  Time cost:  7.9464499950408936\n",
      "Start and end (430, 435)\n",
      "idx:  430  Epoch:  113  loss:  304.42566  loss_sent:  2.7289712  loss_word:  290.78082  Time cost:  7.046435832977295\n",
      "Start and end (435, 440)\n",
      "idx:  435  Epoch:  113  loss:  274.44135  loss_sent:  2.9351006  loss_word:  259.76584  Time cost:  6.923158168792725\n",
      "Start and end (440, 445)\n",
      "idx:  440  Epoch:  113  loss:  311.87253  loss_sent:  2.3287177  loss_word:  300.22894  Time cost:  6.742314100265503\n",
      "Start and end (445, 450)\n",
      "idx:  445  Epoch:  113  loss:  316.09335  loss_sent:  3.1264377  loss_word:  300.46115  Time cost:  6.7509238719940186\n",
      "Start and end (450, 455)\n",
      "idx:  450  Epoch:  113  loss:  275.05554  loss_sent:  3.7623112  loss_word:  256.24393  Time cost:  6.885547637939453\n",
      "Start and end (455, 460)\n",
      "idx:  455  Epoch:  113  loss:  337.87048  loss_sent:  2.9602656  loss_word:  323.06915  Time cost:  6.804346323013306\n",
      "Start and end (460, 465)\n",
      "idx:  460  Epoch:  113  loss:  348.00168  loss_sent:  3.697696  loss_word:  329.5132  Time cost:  6.873573064804077\n",
      "Start and end (465, 470)\n",
      "idx:  465  Epoch:  113  loss:  333.81552  loss_sent:  3.0657864  loss_word:  318.48657  Time cost:  6.724589824676514\n",
      "Start and end (470, 475)\n",
      "idx:  470  Epoch:  113  loss:  333.2003  loss_sent:  2.7344787  loss_word:  319.52795  Time cost:  6.792287826538086\n",
      "Start and end (475, 480)\n",
      "idx:  475  Epoch:  113  loss:  331.99493  loss_sent:  2.9499328  loss_word:  317.24527  Time cost:  7.02976131439209\n",
      "Start and end (480, 485)\n",
      "idx:  480  Epoch:  113  loss:  372.33224  loss_sent:  3.5393977  loss_word:  354.63528  Time cost:  6.807053089141846\n",
      "Start and end (485, 490)\n",
      "idx:  485  Epoch:  113  loss:  318.7629  loss_sent:  2.6364632  loss_word:  305.58057  Time cost:  6.9383039474487305\n",
      "Start and end (490, 495)\n",
      "idx:  490  Epoch:  113  loss:  237.42078  loss_sent:  3.0218658  loss_word:  222.31142  Time cost:  6.729754209518433\n",
      "Start and end (495, 500)\n",
      "idx:  495  Epoch:  113  loss:  277.9615  loss_sent:  2.4100056  loss_word:  265.9115  Time cost:  6.953530311584473\n",
      "Start and end (500, 505)\n",
      "idx:  500  Epoch:  113  loss:  380.03314  loss_sent:  2.668845  loss_word:  366.68896  Time cost:  6.715427875518799\n",
      "Start and end (505, 510)\n",
      "idx:  505  Epoch:  113  loss:  185.57532  loss_sent:  3.4837685  loss_word:  168.15646  Time cost:  6.773075580596924\n",
      "Start and end (510, 515)\n",
      "idx:  510  Epoch:  113  loss:  292.19565  loss_sent:  2.6819627  loss_word:  278.78583  Time cost:  6.9959797859191895\n",
      "Start and end (515, 520)\n",
      "idx:  515  Epoch:  113  loss:  362.6284  loss_sent:  3.0676627  loss_word:  347.29004  Time cost:  6.778417110443115\n",
      "Start and end (520, 525)\n",
      "idx:  520  Epoch:  113  loss:  279.789  loss_sent:  2.4687119  loss_word:  267.44543  Time cost:  6.844151258468628\n",
      "Start and end (525, 530)\n",
      "idx:  525  Epoch:  113  loss:  371.35217  loss_sent:  2.731509  loss_word:  357.6946  Time cost:  6.784470796585083\n",
      "Start and end (530, 535)\n",
      "idx:  530  Epoch:  113  loss:  344.88904  loss_sent:  2.450778  loss_word:  332.6351  Time cost:  6.861417055130005\n",
      "Start and end (535, 540)\n",
      "idx:  535  Epoch:  113  loss:  207.82239  loss_sent:  2.8576221  loss_word:  193.53427  Time cost:  6.72818922996521\n",
      "Start and end (540, 545)\n",
      "idx:  540  Epoch:  113  loss:  330.017  loss_sent:  2.9850273  loss_word:  315.09183  Time cost:  6.896557569503784\n",
      "Start and end (545, 550)\n",
      "idx:  545  Epoch:  113  loss:  431.3886  loss_sent:  2.3554003  loss_word:  419.6116  Time cost:  7.066438674926758\n",
      "Start and end (550, 555)\n",
      "idx:  550  Epoch:  113  loss:  341.93362  loss_sent:  3.1271532  loss_word:  326.29788  Time cost:  6.724860429763794\n",
      "Start and end (555, 560)\n",
      "idx:  555  Epoch:  113  loss:  247.59848  loss_sent:  2.8274176  loss_word:  233.46138  Time cost:  6.916019678115845\n",
      "Start and end (560, 565)\n",
      "idx:  560  Epoch:  113  loss:  376.5061  loss_sent:  2.6521788  loss_word:  363.24524  Time cost:  6.760907173156738\n",
      "Start and end (565, 570)\n",
      "idx:  565  Epoch:  113  loss:  277.30545  loss_sent:  3.2205315  loss_word:  261.2028  Time cost:  6.77758526802063\n",
      "Start and end (570, 575)\n",
      "idx:  570  Epoch:  113  loss:  253.26096  loss_sent:  2.8185747  loss_word:  239.16809  Time cost:  6.678457021713257\n",
      "Start and end (575, 580)\n",
      "idx:  575  Epoch:  113  loss:  364.3303  loss_sent:  2.42639  loss_word:  352.19843  Time cost:  6.734595060348511\n",
      "Start and end (580, 585)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  580  Epoch:  113  loss:  213.06422  loss_sent:  2.7688742  loss_word:  199.21986  Time cost:  6.933086395263672\n",
      "Start and end (585, 590)\n",
      "idx:  585  Epoch:  113  loss:  403.0985  loss_sent:  2.7083018  loss_word:  389.557  Time cost:  6.746628284454346\n",
      "Start and end (590, 595)\n",
      "idx:  590  Epoch:  113  loss:  366.28464  loss_sent:  4.066497  loss_word:  345.95215  Time cost:  6.899180173873901\n",
      "Start and end (595, 600)\n",
      "idx:  595  Epoch:  113  loss:  316.60895  loss_sent:  3.0060163  loss_word:  301.57886  Time cost:  6.690722703933716\n",
      "Start and end (600, 605)\n",
      "idx:  600  Epoch:  113  loss:  346.00534  loss_sent:  2.8974984  loss_word:  331.51788  Time cost:  6.911505699157715\n",
      "Start and end (605, 610)\n",
      "idx:  605  Epoch:  113  loss:  263.2942  loss_sent:  2.4926753  loss_word:  250.83083  Time cost:  6.740240097045898\n",
      "Start and end (610, 615)\n",
      "idx:  610  Epoch:  113  loss:  288.40073  loss_sent:  2.8011975  loss_word:  274.3947  Time cost:  6.767743825912476\n",
      "Start and end (615, 620)\n",
      "idx:  615  Epoch:  113  loss:  194.94875  loss_sent:  2.5579002  loss_word:  182.15929  Time cost:  6.7409257888793945\n",
      "Start and end (620, 625)\n",
      "idx:  620  Epoch:  113  loss:  260.85928  loss_sent:  3.0484564  loss_word:  245.617  Time cost:  6.7364280223846436\n",
      "Start and end (625, 630)\n",
      "idx:  625  Epoch:  113  loss:  336.28128  loss_sent:  3.224962  loss_word:  320.15652  Time cost:  6.915250539779663\n",
      "Start and end (630, 635)\n",
      "idx:  630  Epoch:  113  loss:  326.46982  loss_sent:  3.0171397  loss_word:  311.3841  Time cost:  6.693915367126465\n",
      "Start and end (635, 640)\n",
      "idx:  635  Epoch:  113  loss:  340.85867  loss_sent:  3.4060931  loss_word:  323.82828  Time cost:  7.099940299987793\n",
      "Start and end (640, 645)\n",
      "idx:  640  Epoch:  113  loss:  277.85733  loss_sent:  3.2559195  loss_word:  261.57776  Time cost:  6.717522621154785\n",
      "Start and end (645, 650)\n",
      "idx:  645  Epoch:  113  loss:  366.90717  loss_sent:  3.0868313  loss_word:  351.47302  Time cost:  6.693183183670044\n",
      "Start and end (650, 655)\n",
      "idx:  650  Epoch:  113  loss:  351.0242  loss_sent:  2.7052338  loss_word:  337.49805  Time cost:  6.863908529281616\n",
      "Start and end (655, 660)\n",
      "idx:  655  Epoch:  113  loss:  289.139  loss_sent:  2.8995826  loss_word:  274.64108  Time cost:  6.707965612411499\n",
      "Start and end (660, 665)\n",
      "idx:  660  Epoch:  113  loss:  350.9156  loss_sent:  3.1667416  loss_word:  335.08185  Time cost:  6.868849515914917\n",
      "Start and end (665, 670)\n",
      "idx:  665  Epoch:  113  loss:  428.27805  loss_sent:  2.9124842  loss_word:  413.71564  Time cost:  6.740250825881958\n",
      "Start and end (670, 675)\n",
      "idx:  670  Epoch:  113  loss:  328.98016  loss_sent:  2.3962002  loss_word:  316.99915  Time cost:  6.9461510181427\n",
      "Start and end (675, 680)\n",
      "idx:  675  Epoch:  113  loss:  219.52483  loss_sent:  3.4773006  loss_word:  202.1383  Time cost:  6.676288843154907\n",
      "Start and end (680, 685)\n",
      "idx:  680  Epoch:  113  loss:  341.96942  loss_sent:  2.587028  loss_word:  329.03424  Time cost:  6.65616512298584\n",
      "Start and end (685, 690)\n",
      "idx:  685  Epoch:  113  loss:  280.2512  loss_sent:  2.2935245  loss_word:  268.7836  Time cost:  6.845770597457886\n",
      "Start and end (690, 695)\n",
      "idx:  690  Epoch:  113  loss:  240.9021  loss_sent:  3.6310675  loss_word:  222.74677  Time cost:  6.782256364822388\n",
      "Start and end (695, 700)\n",
      "idx:  695  Epoch:  113  loss:  298.6269  loss_sent:  2.920806  loss_word:  284.0229  Time cost:  6.971867084503174\n",
      "Start and end (700, 705)\n",
      "idx:  700  Epoch:  113  loss:  306.8452  loss_sent:  3.2638674  loss_word:  290.52588  Time cost:  6.761789798736572\n",
      "Start and end (705, 710)\n",
      "idx:  705  Epoch:  113  loss:  284.08408  loss_sent:  3.4130783  loss_word:  267.0187  Time cost:  6.7456347942352295\n",
      "Start and end (710, 715)\n",
      "idx:  710  Epoch:  113  loss:  327.49548  loss_sent:  2.4161212  loss_word:  315.4149  Time cost:  6.741267919540405\n",
      "Start and end (715, 720)\n",
      "idx:  715  Epoch:  113  loss:  265.5706  loss_sent:  3.4531043  loss_word:  248.30502  Time cost:  6.755917072296143\n",
      "Start and end (720, 725)\n",
      "idx:  720  Epoch:  113  loss:  257.15018  loss_sent:  2.8176057  loss_word:  243.06212  Time cost:  7.009196043014526\n",
      "Start and end (725, 730)\n",
      "idx:  725  Epoch:  113  loss:  296.01074  loss_sent:  2.404437  loss_word:  283.98856  Time cost:  6.922296047210693\n",
      "Start and end (730, 735)\n",
      "idx:  730  Epoch:  113  loss:  346.04044  loss_sent:  2.8187194  loss_word:  331.94684  Time cost:  6.923710823059082\n",
      "Start and end (735, 740)\n",
      "idx:  735  Epoch:  113  loss:  316.41504  loss_sent:  2.4071443  loss_word:  304.37933  Time cost:  6.789369583129883\n",
      "Start and end (740, 745)\n",
      "idx:  740  Epoch:  113  loss:  388.33722  loss_sent:  2.4782472  loss_word:  375.94598  Time cost:  6.799794435501099\n",
      "Start and end (745, 750)\n",
      "idx:  745  Epoch:  113  loss:  236.31274  loss_sent:  2.5761135  loss_word:  223.43217  Time cost:  6.718826770782471\n",
      "Start and end (750, 755)\n",
      "idx:  750  Epoch:  113  loss:  245.67743  loss_sent:  2.8059778  loss_word:  231.64755  Time cost:  6.788735628128052\n",
      "Start and end (755, 760)\n",
      "idx:  755  Epoch:  113  loss:  304.35815  loss_sent:  3.8058212  loss_word:  285.32904  Time cost:  6.872631072998047\n",
      "Start and end (760, 765)\n",
      "idx:  760  Epoch:  113  loss:  277.8572  loss_sent:  2.6248007  loss_word:  264.73322  Time cost:  6.807480335235596\n",
      "Start and end (765, 770)\n",
      "idx:  765  Epoch:  113  loss:  229.29678  loss_sent:  3.1129196  loss_word:  213.7322  Time cost:  6.949341297149658\n",
      "Start and end (770, 775)\n",
      "idx:  770  Epoch:  113  loss:  269.11206  loss_sent:  2.563364  loss_word:  256.29526  Time cost:  6.801580190658569\n",
      "Start and end (775, 780)\n",
      "idx:  775  Epoch:  113  loss:  296.35403  loss_sent:  3.01917  loss_word:  281.25812  Time cost:  6.71972393989563\n",
      "Start and end (780, 785)\n",
      "idx:  780  Epoch:  113  loss:  320.6234  loss_sent:  2.3959327  loss_word:  308.64374  Time cost:  6.871323823928833\n",
      "Start and end (785, 790)\n",
      "idx:  785  Epoch:  113  loss:  296.52856  loss_sent:  2.7325432  loss_word:  282.86588  Time cost:  6.699929237365723\n",
      "Start and end (790, 795)\n",
      "idx:  790  Epoch:  113  loss:  398.9375  loss_sent:  3.4093757  loss_word:  381.8906  Time cost:  6.925294876098633\n",
      "Start and end (795, 800)\n",
      "idx:  795  Epoch:  113  loss:  302.03406  loss_sent:  3.8527124  loss_word:  282.77048  Time cost:  6.684763431549072\n",
      "Start and end (800, 805)\n",
      "idx:  800  Epoch:  113  loss:  414.7216  loss_sent:  4.9423637  loss_word:  390.00974  Time cost:  6.843671798706055\n",
      "Start and end (805, 810)\n",
      "idx:  805  Epoch:  113  loss:  321.63904  loss_sent:  2.3632467  loss_word:  309.8228  Time cost:  6.753785610198975\n",
      "Start and end (810, 815)\n",
      "idx:  810  Epoch:  113  loss:  359.39978  loss_sent:  2.7695124  loss_word:  345.55222  Time cost:  6.841790199279785\n",
      "Start and end (815, 820)\n",
      "idx:  815  Epoch:  113  loss:  350.9636  loss_sent:  3.2205882  loss_word:  334.8607  Time cost:  6.912919282913208\n",
      "Start and end (820, 825)\n",
      "idx:  820  Epoch:  113  loss:  272.05362  loss_sent:  3.5215793  loss_word:  254.44568  Time cost:  6.7285120487213135\n",
      "Start and end (825, 830)\n",
      "idx:  825  Epoch:  113  loss:  348.9197  loss_sent:  2.3292217  loss_word:  337.27362  Time cost:  6.845846176147461\n",
      "Start and end (830, 835)\n",
      "idx:  830  Epoch:  113  loss:  390.1767  loss_sent:  2.942539  loss_word:  375.46402  Time cost:  6.734791994094849\n",
      "Start and end (835, 840)\n",
      "idx:  835  Epoch:  113  loss:  203.73372  loss_sent:  2.1526241  loss_word:  192.9706  Time cost:  6.9643542766571045\n",
      "Start and end (840, 845)\n",
      "idx:  840  Epoch:  113  loss:  291.259  loss_sent:  3.148415  loss_word:  275.51697  Time cost:  6.664039373397827\n",
      "Start and end (845, 850)\n",
      "idx:  845  Epoch:  113  loss:  249.2381  loss_sent:  2.8669295  loss_word:  234.90347  Time cost:  6.775873184204102\n",
      "Start and end (850, 855)\n",
      "idx:  850  Epoch:  113  loss:  382.11478  loss_sent:  3.2054331  loss_word:  366.08765  Time cost:  6.874203443527222\n",
      "Start and end (855, 860)\n",
      "idx:  855  Epoch:  113  loss:  281.78464  loss_sent:  3.2140458  loss_word:  265.7144  Time cost:  6.793632984161377\n",
      "Start and end (860, 865)\n",
      "idx:  860  Epoch:  113  loss:  323.71524  loss_sent:  4.445692  loss_word:  301.48672  Time cost:  6.929004430770874\n",
      "Start and end (865, 870)\n",
      "idx:  865  Epoch:  113  loss:  358.15512  loss_sent:  2.715033  loss_word:  344.57996  Time cost:  6.704594850540161\n",
      "Start and end (870, 875)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  870  Epoch:  113  loss:  189.66171  loss_sent:  3.2696757  loss_word:  173.31334  Time cost:  6.795580625534058\n",
      "Start and end (875, 880)\n",
      "idx:  875  Epoch:  113  loss:  311.24802  loss_sent:  3.186697  loss_word:  295.3145  Time cost:  6.703605890274048\n",
      "Start and end (880, 885)\n",
      "idx:  880  Epoch:  113  loss:  315.97586  loss_sent:  2.7535682  loss_word:  302.20798  Time cost:  6.669618606567383\n",
      "Start and end (885, 890)\n",
      "idx:  885  Epoch:  113  loss:  262.65137  loss_sent:  2.7935047  loss_word:  248.68384  Time cost:  6.908123254776001\n",
      "Start and end (890, 895)\n",
      "idx:  890  Epoch:  113  loss:  268.44092  loss_sent:  2.7285655  loss_word:  254.79807  Time cost:  6.729360818862915\n",
      "Start and end (895, 900)\n",
      "idx:  895  Epoch:  113  loss:  365.2565  loss_sent:  2.8046691  loss_word:  351.23312  Time cost:  6.982182264328003\n",
      "Start and end (900, 905)\n",
      "idx:  900  Epoch:  113  loss:  297.1969  loss_sent:  2.9048443  loss_word:  282.67264  Time cost:  6.756896734237671\n",
      "Start and end (905, 910)\n",
      "idx:  905  Epoch:  113  loss:  351.0612  loss_sent:  3.0903842  loss_word:  335.60925  Time cost:  6.935586214065552\n",
      "Start and end (910, 915)\n",
      "idx:  910  Epoch:  113  loss:  274.54913  loss_sent:  3.0267012  loss_word:  259.41568  Time cost:  6.744941234588623\n",
      "Start and end (915, 920)\n",
      "idx:  915  Epoch:  113  loss:  396.36105  loss_sent:  3.4565785  loss_word:  379.07816  Time cost:  6.711087942123413\n",
      "Start and end (920, 925)\n",
      "idx:  920  Epoch:  113  loss:  266.08453  loss_sent:  2.7193854  loss_word:  252.48766  Time cost:  6.886954307556152\n",
      "Start and end (925, 930)\n",
      "idx:  925  Epoch:  113  loss:  260.4118  loss_sent:  3.2878401  loss_word:  243.97255  Time cost:  6.7736968994140625\n",
      "Start and end (930, 935)\n",
      "idx:  930  Epoch:  113  loss:  218.73553  loss_sent:  3.419247  loss_word:  201.63933  Time cost:  6.884043455123901\n",
      "Start and end (935, 940)\n",
      "idx:  935  Epoch:  113  loss:  453.80887  loss_sent:  3.077919  loss_word:  438.41928  Time cost:  6.793637037277222\n",
      "Start and end (940, 945)\n",
      "idx:  940  Epoch:  113  loss:  362.46597  loss_sent:  2.567123  loss_word:  349.63034  Time cost:  6.92429256439209\n",
      "Start and end (945, 950)\n",
      "idx:  945  Epoch:  113  loss:  253.81903  loss_sent:  2.488873  loss_word:  241.37466  Time cost:  6.734550952911377\n",
      "Start and end (950, 955)\n",
      "idx:  950  Epoch:  113  loss:  317.6407  loss_sent:  2.2274873  loss_word:  306.50323  Time cost:  6.773097038269043\n",
      "Start and end (955, 960)\n",
      "idx:  955  Epoch:  113  loss:  286.76874  loss_sent:  3.3987606  loss_word:  269.7749  Time cost:  6.900685548782349\n",
      "Start and end (960, 965)\n",
      "idx:  960  Epoch:  113  loss:  253.36162  loss_sent:  3.055127  loss_word:  238.08598  Time cost:  6.667476415634155\n",
      "Start and end (965, 970)\n",
      "idx:  965  Epoch:  113  loss:  271.5373  loss_sent:  3.3848145  loss_word:  254.61328  Time cost:  6.904717445373535\n",
      "Start and end (970, 975)\n",
      "idx:  970  Epoch:  113  loss:  323.69394  loss_sent:  2.0063658  loss_word:  313.6621  Time cost:  6.729633092880249\n",
      "Start and end (975, 980)\n",
      "idx:  975  Epoch:  113  loss:  367.31305  loss_sent:  2.3441463  loss_word:  355.59232  Time cost:  6.700046539306641\n",
      "Start and end (980, 985)\n",
      "idx:  980  Epoch:  113  loss:  271.28503  loss_sent:  3.1568906  loss_word:  255.50061  Time cost:  6.802291631698608\n",
      "Start and end (985, 990)\n",
      "idx:  985  Epoch:  113  loss:  347.4562  loss_sent:  2.177537  loss_word:  336.5685  Time cost:  6.900811195373535\n",
      "Start and end (990, 995)\n",
      "idx:  990  Epoch:  113  loss:  309.83936  loss_sent:  2.6859143  loss_word:  296.4098  Time cost:  6.925567150115967\n",
      "Start and end (995, 1000)\n",
      "idx:  995  Epoch:  113  loss:  411.0708  loss_sent:  3.6685238  loss_word:  392.7282  Time cost:  6.735620021820068\n",
      "Start and end (1000, 1005)\n",
      "idx:  1000  Epoch:  113  loss:  339.13657  loss_sent:  2.1971307  loss_word:  328.15088  Time cost:  6.886556625366211\n",
      "Start and end (1005, 1010)\n",
      "idx:  1005  Epoch:  113  loss:  372.26706  loss_sent:  3.899253  loss_word:  352.77075  Time cost:  6.712144136428833\n",
      "Start and end (1010, 1015)\n",
      "idx:  1010  Epoch:  113  loss:  337.89108  loss_sent:  2.5068  loss_word:  325.3571  Time cost:  6.753278970718384\n",
      "Start and end (1015, 1020)\n",
      "idx:  1015  Epoch:  113  loss:  354.47382  loss_sent:  2.478775  loss_word:  342.07993  Time cost:  7.001210689544678\n",
      "Start and end (1020, 1025)\n",
      "idx:  1020  Epoch:  113  loss:  298.04404  loss_sent:  2.6425004  loss_word:  284.8315  Time cost:  6.787783861160278\n",
      "Start and end (1025, 1030)\n",
      "idx:  1025  Epoch:  113  loss:  284.51944  loss_sent:  3.189921  loss_word:  268.56985  Time cost:  6.950839281082153\n",
      "Start and end (1030, 1035)\n",
      "idx:  1030  Epoch:  113  loss:  310.90518  loss_sent:  2.5243323  loss_word:  298.28345  Time cost:  6.74889612197876\n",
      "Start and end (1035, 1040)\n",
      "idx:  1035  Epoch:  113  loss:  256.2648  loss_sent:  2.9408486  loss_word:  241.5606  Time cost:  6.768239736557007\n",
      "Start and end (1040, 1045)\n",
      "idx:  1040  Epoch:  113  loss:  240.4612  loss_sent:  3.1383996  loss_word:  224.76921  Time cost:  6.810354232788086\n",
      "Start and end (1045, 1050)\n",
      "idx:  1045  Epoch:  113  loss:  339.5991  loss_sent:  2.4449391  loss_word:  327.37436  Time cost:  6.7979559898376465\n",
      "Start and end (1050, 1055)\n",
      "idx:  1050  Epoch:  113  loss:  274.2726  loss_sent:  3.2928848  loss_word:  257.80817  Time cost:  6.858759880065918\n",
      "Start and end (1055, 1060)\n",
      "idx:  1055  Epoch:  113  loss:  309.91855  loss_sent:  2.8039892  loss_word:  295.89865  Time cost:  6.7190024852752686\n",
      "Start and end (1060, 1065)\n",
      "idx:  1060  Epoch:  113  loss:  369.49564  loss_sent:  2.4796298  loss_word:  357.0975  Time cost:  6.823031663894653\n",
      "Start and end (1065, 1070)\n",
      "idx:  1065  Epoch:  113  loss:  299.5418  loss_sent:  2.946746  loss_word:  284.80804  Time cost:  6.711706161499023\n",
      "Start and end (1070, 1075)\n",
      "idx:  1070  Epoch:  113  loss:  360.9555  loss_sent:  2.2488623  loss_word:  349.71118  Time cost:  6.8401618003845215\n",
      "Start and end (1075, 1080)\n",
      "idx:  1075  Epoch:  113  loss:  246.353  loss_sent:  3.1840794  loss_word:  230.43259  Time cost:  6.869481563568115\n",
      "Start and end (1080, 1085)\n",
      "idx:  1080  Epoch:  113  loss:  283.65225  loss_sent:  2.7480972  loss_word:  269.9118  Time cost:  6.745631456375122\n",
      "Start and end (1085, 1090)\n",
      "idx:  1085  Epoch:  113  loss:  244.84232  loss_sent:  3.5701666  loss_word:  226.99147  Time cost:  6.903555393218994\n",
      "Start and end (1090, 1095)\n",
      "idx:  1090  Epoch:  113  loss:  279.2019  loss_sent:  2.7962685  loss_word:  265.22058  Time cost:  6.721405029296875\n",
      "Start and end (1095, 1100)\n",
      "idx:  1095  Epoch:  113  loss:  285.65945  loss_sent:  2.6067955  loss_word:  272.62552  Time cost:  6.872812032699585\n",
      "Start and end (1100, 1105)\n",
      "idx:  1100  Epoch:  113  loss:  264.9827  loss_sent:  2.6935205  loss_word:  251.5151  Time cost:  6.766766786575317\n",
      "Start and end (1105, 1110)\n",
      "idx:  1105  Epoch:  113  loss:  322.3907  loss_sent:  2.7776065  loss_word:  308.50266  Time cost:  6.848455429077148\n",
      "Start and end (1110, 1115)\n",
      "idx:  1110  Epoch:  113  loss:  306.821  loss_sent:  2.7615259  loss_word:  293.0134  Time cost:  6.739267349243164\n",
      "Start and end (1115, 1120)\n",
      "idx:  1115  Epoch:  113  loss:  224.14665  loss_sent:  2.992677  loss_word:  209.18324  Time cost:  6.72714638710022\n",
      "Start and end (1120, 1125)\n",
      "idx:  1120  Epoch:  113  loss:  288.51514  loss_sent:  2.5532186  loss_word:  275.74896  Time cost:  6.894974231719971\n",
      "Start and end (1125, 1130)\n",
      "idx:  1125  Epoch:  113  loss:  307.58533  loss_sent:  2.9987295  loss_word:  292.59167  Time cost:  7.2496018409729\n",
      "Start and end (1130, 1135)\n",
      "idx:  1130  Epoch:  113  loss:  374.4678  loss_sent:  2.2649746  loss_word:  363.14294  Time cost:  6.919393301010132\n",
      "Start and end (1135, 1140)\n",
      "idx:  1135  Epoch:  113  loss:  248.04314  loss_sent:  3.418479  loss_word:  230.95074  Time cost:  6.753906726837158\n",
      "Start and end (1140, 1145)\n",
      "idx:  1140  Epoch:  113  loss:  241.48567  loss_sent:  3.4365404  loss_word:  224.30296  Time cost:  6.829108715057373\n",
      "Start and end (1145, 1150)\n",
      "idx:  1145  Epoch:  113  loss:  280.27887  loss_sent:  2.979909  loss_word:  265.37936  Time cost:  6.734829664230347\n",
      "Start and end (1150, 1155)\n",
      "idx:  1150  Epoch:  113  loss:  238.58716  loss_sent:  3.5894823  loss_word:  220.63974  Time cost:  6.688798189163208\n",
      "Start and end (1155, 1160)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1155  Epoch:  113  loss:  276.33328  loss_sent:  2.846242  loss_word:  262.10202  Time cost:  6.858116388320923\n",
      "Start and end (1160, 1165)\n",
      "idx:  1160  Epoch:  113  loss:  228.20634  loss_sent:  2.1631074  loss_word:  217.39081  Time cost:  6.741997718811035\n",
      "Start and end (1165, 1170)\n",
      "idx:  1165  Epoch:  113  loss:  308.0768  loss_sent:  2.744841  loss_word:  294.35266  Time cost:  6.995177745819092\n",
      "Start and end (1170, 1175)\n",
      "idx:  1170  Epoch:  113  loss:  181.55412  loss_sent:  2.3293254  loss_word:  169.9075  Time cost:  6.749823570251465\n",
      "Start and end (1175, 1180)\n",
      "idx:  1175  Epoch:  113  loss:  291.26205  loss_sent:  3.21319  loss_word:  275.19604  Time cost:  6.841775178909302\n",
      "Start and end (1180, 1185)\n",
      "idx:  1180  Epoch:  113  loss:  231.47902  loss_sent:  3.7367435  loss_word:  212.79532  Time cost:  6.707117080688477\n",
      "Start and end (1185, 1190)\n",
      "idx:  1185  Epoch:  113  loss:  262.55984  loss_sent:  3.4075818  loss_word:  245.52197  Time cost:  6.6757447719573975\n",
      "Start and end (1190, 1195)\n",
      "idx:  1190  Epoch:  113  loss:  380.7951  loss_sent:  2.9042287  loss_word:  366.274  Time cost:  6.818150520324707\n",
      "Start and end (1195, 1200)\n",
      "idx:  1195  Epoch:  113  loss:  295.40393  loss_sent:  3.4630752  loss_word:  278.08856  Time cost:  6.763295888900757\n",
      "Start and end (1200, 1205)\n",
      "idx:  1200  Epoch:  113  loss:  255.77045  loss_sent:  3.433948  loss_word:  238.60071  Time cost:  6.8323540687561035\n",
      "Start and end (1205, 1210)\n",
      "idx:  1205  Epoch:  113  loss:  341.8056  loss_sent:  2.5885413  loss_word:  328.86288  Time cost:  6.726926565170288\n",
      "Start and end (1210, 1215)\n",
      "idx:  1210  Epoch:  113  loss:  318.70712  loss_sent:  2.630037  loss_word:  305.557  Time cost:  6.804861783981323\n",
      "Start and end (1215, 1220)\n",
      "idx:  1215  Epoch:  113  loss:  279.19525  loss_sent:  3.2296383  loss_word:  263.047  Time cost:  6.769101142883301\n",
      "Start and end (1220, 1225)\n",
      "idx:  1220  Epoch:  113  loss:  246.21484  loss_sent:  2.4169257  loss_word:  234.1302  Time cost:  6.787551641464233\n",
      "Start and end (1225, 1230)\n",
      "idx:  1225  Epoch:  113  loss:  277.37363  loss_sent:  2.1585712  loss_word:  266.58075  Time cost:  6.9797070026397705\n",
      "Start and end (1230, 1235)\n",
      "idx:  1230  Epoch:  113  loss:  221.3926  loss_sent:  3.2072165  loss_word:  205.35649  Time cost:  6.7529988288879395\n",
      "Start and end (1235, 1240)\n",
      "idx:  1235  Epoch:  113  loss:  252.07048  loss_sent:  3.4549797  loss_word:  234.7956  Time cost:  6.914397239685059\n",
      "Start and end (1240, 1245)\n",
      "idx:  1240  Epoch:  113  loss:  267.51898  loss_sent:  3.3332686  loss_word:  250.85266  Time cost:  6.7311718463897705\n",
      "Start and end (1245, 1250)\n",
      "idx:  1245  Epoch:  113  loss:  334.40378  loss_sent:  2.1014984  loss_word:  323.89627  Time cost:  6.936429023742676\n",
      "Start and end (1250, 1255)\n",
      "idx:  1250  Epoch:  113  loss:  279.74948  loss_sent:  2.7683127  loss_word:  265.90793  Time cost:  6.961471080780029\n",
      "Start and end (1255, 1260)\n",
      "idx:  1255  Epoch:  113  loss:  220.39359  loss_sent:  3.4088156  loss_word:  203.34953  Time cost:  6.758615493774414\n",
      "Start and end (1260, 1265)\n",
      "idx:  1260  Epoch:  113  loss:  264.03052  loss_sent:  2.8616648  loss_word:  249.72218  Time cost:  6.765510082244873\n",
      "Start and end (1265, 1270)\n",
      "idx:  1265  Epoch:  113  loss:  275.74588  loss_sent:  2.8859205  loss_word:  261.3163  Time cost:  6.732079029083252\n",
      "Start and end (1270, 1275)\n",
      "idx:  1270  Epoch:  113  loss:  237.2588  loss_sent:  3.0974724  loss_word:  221.77147  Time cost:  6.756551027297974\n",
      "Start and end (1275, 1280)\n",
      "idx:  1275  Epoch:  113  loss:  331.31815  loss_sent:  2.8165324  loss_word:  317.23547  Time cost:  6.7339701652526855\n",
      "Start and end (1280, 1285)\n",
      "idx:  1280  Epoch:  113  loss:  364.3637  loss_sent:  2.6662412  loss_word:  351.03244  Time cost:  6.740077495574951\n",
      "Start and end (1285, 1290)\n",
      "idx:  1285  Epoch:  113  loss:  265.04858  loss_sent:  2.882519  loss_word:  250.636  Time cost:  6.874614953994751\n",
      "Start and end (1290, 1295)\n",
      "idx:  1290  Epoch:  113  loss:  225.401  loss_sent:  3.2592604  loss_word:  209.1047  Time cost:  6.684319734573364\n",
      "Start and end (1295, 1300)\n",
      "idx:  1295  Epoch:  113  loss:  351.379  loss_sent:  3.0889363  loss_word:  335.93427  Time cost:  6.818950653076172\n",
      "Start and end (1300, 1305)\n",
      "idx:  1300  Epoch:  113  loss:  197.6655  loss_sent:  2.7632916  loss_word:  183.84903  Time cost:  6.670636892318726\n",
      "Start and end (1305, 1310)\n",
      "idx:  1305  Epoch:  113  loss:  279.54504  loss_sent:  2.8796403  loss_word:  265.14682  Time cost:  6.842609405517578\n",
      "Start and end (1310, 1315)\n",
      "idx:  1310  Epoch:  113  loss:  381.31226  loss_sent:  2.5440896  loss_word:  368.59186  Time cost:  6.83842658996582\n",
      "Start and end (1315, 1320)\n",
      "idx:  1315  Epoch:  113  loss:  273.90775  loss_sent:  3.4232268  loss_word:  256.79156  Time cost:  6.708977937698364\n",
      "Start and end (1320, 1325)\n",
      "idx:  1320  Epoch:  113  loss:  241.12686  loss_sent:  3.900873  loss_word:  221.6225  Time cost:  6.893713712692261\n",
      "Start and end (1325, 1330)\n",
      "idx:  1325  Epoch:  113  loss:  304.17484  loss_sent:  3.37111  loss_word:  287.31924  Time cost:  6.763946533203125\n",
      "Start and end (1330, 1335)\n",
      "idx:  1330  Epoch:  113  loss:  290.7714  loss_sent:  2.4206138  loss_word:  278.66837  Time cost:  6.899637937545776\n",
      "Start and end (1335, 1340)\n",
      "idx:  1335  Epoch:  113  loss:  313.38266  loss_sent:  3.0266743  loss_word:  298.24924  Time cost:  6.664664268493652\n",
      "Start and end (1340, 1345)\n",
      "idx:  1340  Epoch:  113  loss:  355.93753  loss_sent:  2.4837852  loss_word:  343.51862  Time cost:  7.115694999694824\n",
      "Start and end (1345, 1350)\n",
      "idx:  1345  Epoch:  113  loss:  362.91144  loss_sent:  2.4233212  loss_word:  350.79486  Time cost:  6.74022364616394\n",
      "Start and end (1350, 1355)\n",
      "idx:  1350  Epoch:  113  loss:  328.2361  loss_sent:  3.4126227  loss_word:  311.173  Time cost:  6.72753119468689\n",
      "Start and end (1355, 1360)\n",
      "idx:  1355  Epoch:  113  loss:  354.89243  loss_sent:  2.4378233  loss_word:  342.70328  Time cost:  6.803309917449951\n",
      "Start and end (1360, 1365)\n",
      "idx:  1360  Epoch:  113  loss:  342.51407  loss_sent:  3.4162807  loss_word:  325.43268  Time cost:  6.680808067321777\n",
      "Start and end (1365, 1370)\n",
      "idx:  1365  Epoch:  113  loss:  300.92166  loss_sent:  2.5406358  loss_word:  288.21848  Time cost:  6.8670654296875\n",
      "Start and end (1370, 1375)\n",
      "idx:  1370  Epoch:  113  loss:  295.92233  loss_sent:  2.077634  loss_word:  285.53415  Time cost:  6.701492786407471\n",
      "Start and end (1375, 1380)\n",
      "idx:  1375  Epoch:  113  loss:  414.05374  loss_sent:  2.801065  loss_word:  400.04843  Time cost:  6.905268669128418\n",
      "Start and end (1380, 1385)\n",
      "idx:  1380  Epoch:  113  loss:  331.01086  loss_sent:  3.225315  loss_word:  314.8843  Time cost:  6.818998098373413\n",
      "Start and end (1385, 1390)\n",
      "idx:  1385  Epoch:  113  loss:  402.87338  loss_sent:  2.4714189  loss_word:  390.51633  Time cost:  6.777966022491455\n",
      "Start and end (1390, 1395)\n",
      "idx:  1390  Epoch:  113  loss:  285.76852  loss_sent:  3.0179434  loss_word:  270.6788  Time cost:  6.83146858215332\n",
      "Start and end (1395, 1400)\n",
      "idx:  1395  Epoch:  113  loss:  308.1155  loss_sent:  3.2995872  loss_word:  291.61752  Time cost:  6.82521653175354\n",
      "Start and end (1400, 1405)\n",
      "idx:  1400  Epoch:  113  loss:  200.27675  loss_sent:  3.0112839  loss_word:  185.2203  Time cost:  6.908591270446777\n",
      "Start and end (1405, 1410)\n",
      "idx:  1405  Epoch:  113  loss:  265.11374  loss_sent:  2.524725  loss_word:  252.49014  Time cost:  6.75763463973999\n",
      "Start and end (1410, 1415)\n",
      "idx:  1410  Epoch:  113  loss:  297.8242  loss_sent:  3.080712  loss_word:  282.42062  Time cost:  6.85577917098999\n",
      "Start and end (1415, 1420)\n",
      "idx:  1415  Epoch:  113  loss:  227.1877  loss_sent:  3.4652834  loss_word:  209.86127  Time cost:  6.685168504714966\n",
      "Start and end (1420, 1425)\n",
      "idx:  1420  Epoch:  113  loss:  322.59738  loss_sent:  2.8846097  loss_word:  308.17435  Time cost:  6.748680591583252\n",
      "Start and end (1425, 1430)\n",
      "idx:  1425  Epoch:  113  loss:  318.65765  loss_sent:  2.5145407  loss_word:  306.08496  Time cost:  6.950641393661499\n",
      "Start and end (1430, 1435)\n",
      "idx:  1430  Epoch:  113  loss:  327.75162  loss_sent:  2.465786  loss_word:  315.42267  Time cost:  6.760908842086792\n",
      "Start and end (1435, 1440)\n",
      "idx:  1435  Epoch:  113  loss:  283.9733  loss_sent:  2.5822568  loss_word:  271.06198  Time cost:  6.86420464515686\n",
      "Start and end (1440, 1445)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1440  Epoch:  113  loss:  201.33543  loss_sent:  3.064217  loss_word:  186.01436  Time cost:  6.723113775253296\n",
      "Start and end (1445, 1450)\n",
      "idx:  1445  Epoch:  113  loss:  316.1602  loss_sent:  3.1949031  loss_word:  300.1857  Time cost:  6.882862329483032\n",
      "Start and end (1450, 1455)\n",
      "idx:  1450  Epoch:  113  loss:  214.00539  loss_sent:  3.8781703  loss_word:  194.61452  Time cost:  6.664955139160156\n",
      "Start and end (1455, 1460)\n",
      "idx:  1455  Epoch:  113  loss:  297.05005  loss_sent:  2.6295288  loss_word:  283.9024  Time cost:  6.753477096557617\n",
      "Start and end (1460, 1465)\n",
      "idx:  1460  Epoch:  113  loss:  339.5487  loss_sent:  2.4883032  loss_word:  327.10718  Time cost:  6.837130546569824\n",
      "Start and end (1465, 1470)\n",
      "idx:  1465  Epoch:  113  loss:  242.75093  loss_sent:  3.0105286  loss_word:  227.69829  Time cost:  6.706668853759766\n",
      "Start and end (1470, 1475)\n",
      "idx:  1470  Epoch:  113  loss:  229.65067  loss_sent:  2.6336472  loss_word:  216.48245  Time cost:  6.8047401905059814\n",
      "Start and end (1475, 1480)\n",
      "idx:  1475  Epoch:  113  loss:  344.41812  loss_sent:  3.041637  loss_word:  329.21  Time cost:  6.693344593048096\n",
      "Start and end (1480, 1485)\n",
      "idx:  1480  Epoch:  113  loss:  260.89563  loss_sent:  2.8621078  loss_word:  246.5851  Time cost:  6.767130374908447\n",
      "Start and end (1485, 1490)\n",
      "idx:  1485  Epoch:  113  loss:  427.85358  loss_sent:  3.293739  loss_word:  411.38492  Time cost:  6.883773326873779\n",
      "Start and end (1490, 1495)\n",
      "idx:  1490  Epoch:  113  loss:  299.81302  loss_sent:  2.6377983  loss_word:  286.62405  Time cost:  6.700268268585205\n",
      "Start and end (1495, 1500)\n",
      "idx:  1495  Epoch:  113  loss:  317.09933  loss_sent:  2.8861878  loss_word:  302.66837  Time cost:  6.8945722579956055\n",
      "Start and end (1500, 1505)\n",
      "idx:  1500  Epoch:  113  loss:  193.24513  loss_sent:  2.8377473  loss_word:  179.05635  Time cost:  6.771142244338989\n",
      "Start and end (1505, 1510)\n",
      "idx:  1505  Epoch:  113  loss:  247.26898  loss_sent:  2.6232746  loss_word:  234.15259  Time cost:  6.821937561035156\n",
      "Start and end (1510, 1515)\n",
      "idx:  1510  Epoch:  113  loss:  292.02182  loss_sent:  2.4835553  loss_word:  279.60403  Time cost:  6.722876071929932\n",
      "Start and end (1515, 1520)\n",
      "idx:  1515  Epoch:  113  loss:  396.41678  loss_sent:  2.5495176  loss_word:  383.6692  Time cost:  6.952936887741089\n",
      "Start and end (1520, 1525)\n",
      "idx:  1520  Epoch:  113  loss:  292.4699  loss_sent:  2.5154493  loss_word:  279.89267  Time cost:  7.024216890335083\n",
      "Start and end (1525, 1530)\n",
      "idx:  1525  Epoch:  113  loss:  346.2274  loss_sent:  2.3848786  loss_word:  334.30295  Time cost:  6.8039820194244385\n",
      "Start and end (1530, 1535)\n",
      "idx:  1530  Epoch:  113  loss:  262.13538  loss_sent:  4.2678814  loss_word:  240.79599  Time cost:  6.885457515716553\n",
      "Start and end (1535, 1540)\n",
      "idx:  1535  Epoch:  113  loss:  257.2113  loss_sent:  3.015921  loss_word:  242.13168  Time cost:  6.779322385787964\n",
      "Start and end (1540, 1545)\n",
      "idx:  1540  Epoch:  113  loss:  290.23517  loss_sent:  4.24203  loss_word:  269.025  Time cost:  6.888243913650513\n",
      "Start and end (1545, 1550)\n",
      "idx:  1545  Epoch:  113  loss:  293.725  loss_sent:  3.2207732  loss_word:  277.6211  Time cost:  6.714609146118164\n",
      "Start and end (1550, 1555)\n",
      "idx:  1550  Epoch:  113  loss:  249.26869  loss_sent:  3.49295  loss_word:  231.80396  Time cost:  6.833639621734619\n",
      "Start and end (1555, 1560)\n",
      "idx:  1555  Epoch:  113  loss:  378.1563  loss_sent:  2.9438481  loss_word:  363.43707  Time cost:  6.8751513957977295\n",
      "Start and end (1560, 1565)\n",
      "idx:  1560  Epoch:  113  loss:  321.41418  loss_sent:  2.9021032  loss_word:  306.9037  Time cost:  6.74726939201355\n",
      "Start and end (1565, 1570)\n",
      "idx:  1565  Epoch:  113  loss:  250.61894  loss_sent:  2.9591799  loss_word:  235.82304  Time cost:  6.9011070728302\n",
      "Start and end (1570, 1575)\n",
      "idx:  1570  Epoch:  113  loss:  232.76077  loss_sent:  3.0310736  loss_word:  217.60542  Time cost:  6.7637457847595215\n",
      "Start and end (1575, 1580)\n",
      "idx:  1575  Epoch:  113  loss:  253.38345  loss_sent:  3.7214453  loss_word:  234.77623  Time cost:  6.988499879837036\n",
      "Start and end (1580, 1585)\n",
      "idx:  1580  Epoch:  113  loss:  221.21541  loss_sent:  2.644327  loss_word:  207.9938  Time cost:  6.810648202896118\n",
      "Start and end (1585, 1590)\n",
      "idx:  1585  Epoch:  113  loss:  285.63498  loss_sent:  2.6241083  loss_word:  272.5145  Time cost:  6.787136077880859\n",
      "Start and end (1590, 1595)\n",
      "idx:  1590  Epoch:  113  loss:  259.18173  loss_sent:  3.6658933  loss_word:  240.8523  Time cost:  6.824070453643799\n",
      "Start and end (1595, 1600)\n",
      "idx:  1595  Epoch:  113  loss:  481.7432  loss_sent:  2.7376208  loss_word:  468.05515  Time cost:  6.767340421676636\n",
      "Start and end (1600, 1605)\n",
      "idx:  1600  Epoch:  113  loss:  227.77423  loss_sent:  3.479308  loss_word:  210.37769  Time cost:  7.056194067001343\n",
      "Start and end (1605, 1610)\n",
      "idx:  1605  Epoch:  113  loss:  243.28444  loss_sent:  3.03244  loss_word:  228.12222  Time cost:  6.67134428024292\n",
      "Start and end (1610, 1615)\n",
      "idx:  1610  Epoch:  113  loss:  474.0222  loss_sent:  2.6605577  loss_word:  460.7194  Time cost:  6.889370441436768\n",
      "Start and end (1615, 1620)\n",
      "idx:  1615  Epoch:  113  loss:  193.15857  loss_sent:  3.2295418  loss_word:  177.01086  Time cost:  6.704785585403442\n",
      "Start and end (1620, 1625)\n",
      "idx:  1620  Epoch:  113  loss:  216.2262  loss_sent:  2.7894673  loss_word:  202.27884  Time cost:  6.72153639793396\n",
      "Start and end (1625, 1630)\n",
      "idx:  1625  Epoch:  113  loss:  249.45882  loss_sent:  3.321708  loss_word:  232.85028  Time cost:  6.651182174682617\n",
      "Start and end (1630, 1635)\n",
      "idx:  1630  Epoch:  113  loss:  372.0987  loss_sent:  2.6224124  loss_word:  358.98657  Time cost:  6.654537677764893\n",
      "Start and end (1635, 1640)\n",
      "idx:  1635  Epoch:  113  loss:  326.30148  loss_sent:  4.3156786  loss_word:  304.7231  Time cost:  6.8771960735321045\n",
      "Start and end (1640, 1645)\n",
      "idx:  1640  Epoch:  113  loss:  294.19904  loss_sent:  3.1255112  loss_word:  278.5715  Time cost:  6.658258676528931\n",
      "Start and end (1645, 1650)\n",
      "idx:  1645  Epoch:  113  loss:  314.71683  loss_sent:  2.6810026  loss_word:  301.31183  Time cost:  6.8468029499053955\n",
      "Start and end (1650, 1655)\n",
      "idx:  1650  Epoch:  113  loss:  272.09918  loss_sent:  2.632084  loss_word:  258.93878  Time cost:  6.779616355895996\n",
      "Start and end (1655, 1660)\n",
      "idx:  1655  Epoch:  113  loss:  200.32188  loss_sent:  2.6713548  loss_word:  186.9651  Time cost:  6.6905553340911865\n",
      "Start and end (1660, 1665)\n",
      "idx:  1660  Epoch:  113  loss:  238.29413  loss_sent:  2.8664236  loss_word:  223.96199  Time cost:  6.72324538230896\n",
      "Start and end (1665, 1670)\n",
      "idx:  1665  Epoch:  113  loss:  228.95036  loss_sent:  2.7713976  loss_word:  215.0934  Time cost:  6.645556211471558\n",
      "Start and end (1670, 1675)\n",
      "idx:  1670  Epoch:  113  loss:  283.67532  loss_sent:  3.2681992  loss_word:  267.33432  Time cost:  6.898746013641357\n",
      "Start and end (1675, 1680)\n",
      "idx:  1675  Epoch:  113  loss:  288.83392  loss_sent:  2.8109415  loss_word:  274.7792  Time cost:  6.697117567062378\n",
      "Start and end (1680, 1685)\n",
      "idx:  1680  Epoch:  113  loss:  306.03967  loss_sent:  2.661175  loss_word:  292.73383  Time cost:  6.83890962600708\n",
      "Start and end (1685, 1690)\n",
      "idx:  1685  Epoch:  113  loss:  229.30669  loss_sent:  3.0588984  loss_word:  214.01222  Time cost:  6.853463411331177\n",
      "Start and end (1690, 1695)\n",
      "idx:  1690  Epoch:  113  loss:  281.47092  loss_sent:  3.0769818  loss_word:  266.08606  Time cost:  6.9330902099609375\n",
      "Start and end (1695, 1700)\n",
      "idx:  1695  Epoch:  113  loss:  267.96796  loss_sent:  2.6452608  loss_word:  254.7417  Time cost:  6.840524911880493\n",
      "Start and end (1700, 1705)\n",
      "idx:  1700  Epoch:  113  loss:  235.59648  loss_sent:  3.1861653  loss_word:  219.66566  Time cost:  6.723040342330933\n",
      "Start and end (1705, 1710)\n",
      "idx:  1705  Epoch:  113  loss:  243.10007  loss_sent:  2.640139  loss_word:  229.89937  Time cost:  6.83672571182251\n",
      "Start and end (1710, 1715)\n",
      "idx:  1710  Epoch:  113  loss:  315.1505  loss_sent:  2.8634243  loss_word:  300.8334  Time cost:  6.748033285140991\n",
      "Start and end (1715, 1720)\n",
      "idx:  1715  Epoch:  113  loss:  359.92877  loss_sent:  2.7323172  loss_word:  346.26715  Time cost:  6.757772445678711\n",
      "Start and end (1720, 1725)\n",
      "idx:  1720  Epoch:  113  loss:  259.87714  loss_sent:  2.7783012  loss_word:  245.98564  Time cost:  6.863585948944092\n",
      "Start and end (1725, 1730)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1725  Epoch:  113  loss:  310.5453  loss_sent:  3.0935612  loss_word:  295.07748  Time cost:  6.704421758651733\n",
      "Start and end (1730, 1735)\n",
      "idx:  1730  Epoch:  113  loss:  275.82938  loss_sent:  2.669046  loss_word:  262.4842  Time cost:  6.845362901687622\n",
      "Start and end (1735, 1740)\n",
      "idx:  1735  Epoch:  113  loss:  326.9716  loss_sent:  2.1787963  loss_word:  316.07758  Time cost:  6.719499588012695\n",
      "Start and end (1740, 1745)\n",
      "idx:  1740  Epoch:  113  loss:  263.9364  loss_sent:  3.4668179  loss_word:  246.60236  Time cost:  6.920797109603882\n",
      "Start and end (1745, 1750)\n",
      "idx:  1745  Epoch:  113  loss:  318.79584  loss_sent:  2.5267658  loss_word:  306.162  Time cost:  6.642021894454956\n",
      "Start and end (1750, 1755)\n",
      "idx:  1750  Epoch:  113  loss:  281.68723  loss_sent:  2.8715205  loss_word:  267.32962  Time cost:  6.691251754760742\n",
      "Start and end (1755, 1760)\n",
      "idx:  1755  Epoch:  113  loss:  285.11084  loss_sent:  2.3711047  loss_word:  273.2553  Time cost:  6.784493923187256\n",
      "Start and end (1760, 1765)\n",
      "idx:  1760  Epoch:  113  loss:  414.64557  loss_sent:  2.5548394  loss_word:  401.8714  Time cost:  6.742526531219482\n",
      "Start and end (1765, 1770)\n",
      "idx:  1765  Epoch:  113  loss:  289.22656  loss_sent:  2.4182258  loss_word:  277.13544  Time cost:  6.9203784465789795\n",
      "Start and end (1770, 1775)\n",
      "idx:  1770  Epoch:  113  loss:  264.26373  loss_sent:  2.6276445  loss_word:  251.12552  Time cost:  6.651354789733887\n",
      "Start and end (1775, 1780)\n",
      "idx:  1775  Epoch:  113  loss:  369.55905  loss_sent:  2.4471526  loss_word:  357.32327  Time cost:  6.820122241973877\n",
      "Start and end (1780, 1785)\n",
      "idx:  1780  Epoch:  113  loss:  344.97763  loss_sent:  2.8937752  loss_word:  330.50876  Time cost:  6.865873098373413\n",
      "Start and end (1785, 1790)\n",
      "idx:  1785  Epoch:  113  loss:  269.52518  loss_sent:  2.47961  loss_word:  257.1271  Time cost:  6.6718199253082275\n",
      "Start and end (1790, 1795)\n",
      "idx:  1790  Epoch:  113  loss:  275.16922  loss_sent:  2.5408916  loss_word:  262.46472  Time cost:  6.8717122077941895\n",
      "Start and end (1795, 1800)\n",
      "idx:  1795  Epoch:  113  loss:  326.44873  loss_sent:  3.0735686  loss_word:  311.0809  Time cost:  6.732798099517822\n",
      "Start and end (1800, 1805)\n",
      "idx:  1800  Epoch:  113  loss:  365.88138  loss_sent:  2.3847013  loss_word:  353.95786  Time cost:  6.92522668838501\n",
      "Start and end (1805, 1810)\n",
      "idx:  1805  Epoch:  113  loss:  254.53723  loss_sent:  2.6019905  loss_word:  241.52728  Time cost:  6.706121444702148\n",
      "Start and end (1810, 1815)\n",
      "idx:  1810  Epoch:  113  loss:  220.19278  loss_sent:  2.428502  loss_word:  208.0503  Time cost:  6.9339494705200195\n",
      "Start and end (1815, 1820)\n",
      "idx:  1815  Epoch:  113  loss:  228.13768  loss_sent:  2.7028544  loss_word:  214.6234  Time cost:  6.735273599624634\n",
      "Start and end (1820, 1825)\n",
      "idx:  1820  Epoch:  113  loss:  300.19583  loss_sent:  2.855051  loss_word:  285.92053  Time cost:  6.661621809005737\n",
      "Start and end (1825, 1830)\n",
      "idx:  1825  Epoch:  113  loss:  191.98773  loss_sent:  3.0769014  loss_word:  176.60324  Time cost:  6.842608213424683\n",
      "Start and end (1830, 1835)\n",
      "idx:  1830  Epoch:  113  loss:  259.6529  loss_sent:  2.6021597  loss_word:  246.6421  Time cost:  6.6625330448150635\n",
      "Start and end (1835, 1840)\n",
      "idx:  1835  Epoch:  113  loss:  322.34067  loss_sent:  3.6474607  loss_word:  304.1033  Time cost:  6.939887046813965\n",
      "Start and end (1840, 1845)\n",
      "idx:  1840  Epoch:  113  loss:  206.0782  loss_sent:  2.9027505  loss_word:  191.56445  Time cost:  6.692437171936035\n",
      "Start and end (1845, 1850)\n",
      "idx:  1845  Epoch:  113  loss:  249.26149  loss_sent:  3.1197457  loss_word:  233.66277  Time cost:  6.666942358016968\n",
      "Start and end (1850, 1855)\n",
      "idx:  1850  Epoch:  113  loss:  246.54657  loss_sent:  3.377561  loss_word:  229.65875  Time cost:  7.231477499008179\n",
      "Start and end (1855, 1860)\n",
      "idx:  1855  Epoch:  113  loss:  211.07597  loss_sent:  2.8072457  loss_word:  197.03972  Time cost:  6.724167823791504\n",
      "Start and end (1860, 1865)\n",
      "idx:  1860  Epoch:  113  loss:  182.817  loss_sent:  3.2890193  loss_word:  166.37193  Time cost:  6.882898569107056\n",
      "Start and end (1865, 1870)\n",
      "idx:  1865  Epoch:  113  loss:  249.23402  loss_sent:  2.358008  loss_word:  237.44398  Time cost:  6.875192165374756\n",
      "Start and end (1870, 1875)\n",
      "idx:  1870  Epoch:  113  loss:  227.7662  loss_sent:  3.2412188  loss_word:  211.56013  Time cost:  6.929030895233154\n",
      "Start and end (1875, 1880)\n",
      "idx:  1875  Epoch:  113  loss:  250.74693  loss_sent:  3.2761853  loss_word:  234.36601  Time cost:  6.764124393463135\n",
      "Start and end (1880, 1885)\n",
      "idx:  1880  Epoch:  113  loss:  317.2163  loss_sent:  2.8372495  loss_word:  303.03003  Time cost:  6.795966625213623\n",
      "Start and end (1885, 1890)\n",
      "idx:  1885  Epoch:  113  loss:  257.70764  loss_sent:  3.0409706  loss_word:  242.50282  Time cost:  6.811567306518555\n",
      "Start and end (1890, 1895)\n",
      "idx:  1890  Epoch:  113  loss:  308.37265  loss_sent:  2.6056247  loss_word:  295.3445  Time cost:  6.707585096359253\n",
      "Start and end (1895, 1900)\n",
      "idx:  1895  Epoch:  113  loss:  280.72284  loss_sent:  3.0274558  loss_word:  265.58554  Time cost:  6.892754077911377\n",
      "Start and end (1900, 1905)\n",
      "idx:  1900  Epoch:  113  loss:  241.54576  loss_sent:  2.5286603  loss_word:  228.90247  Time cost:  6.6904137134552\n",
      "Start and end (1905, 1910)\n",
      "idx:  1905  Epoch:  113  loss:  280.2045  loss_sent:  2.5072598  loss_word:  267.6682  Time cost:  6.8965935707092285\n",
      "Start and end (1910, 1915)\n",
      "idx:  1910  Epoch:  113  loss:  305.44498  loss_sent:  2.1909237  loss_word:  294.49036  Time cost:  6.708287239074707\n",
      "Start and end (1915, 1920)\n",
      "idx:  1915  Epoch:  113  loss:  239.83548  loss_sent:  2.8636231  loss_word:  225.51735  Time cost:  6.695502042770386\n",
      "Start and end (1920, 1925)\n",
      "idx:  1920  Epoch:  113  loss:  274.27554  loss_sent:  2.5560987  loss_word:  261.4951  Time cost:  6.914071559906006\n",
      "Start and end (1925, 1930)\n",
      "idx:  1925  Epoch:  113  loss:  240.22455  loss_sent:  2.334491  loss_word:  228.5521  Time cost:  6.7774131298065186\n",
      "Start and end (1930, 1935)\n",
      "idx:  1930  Epoch:  113  loss:  204.82617  loss_sent:  2.904803  loss_word:  190.30215  Time cost:  6.900722980499268\n",
      "Start and end (1935, 1940)\n",
      "idx:  1935  Epoch:  113  loss:  176.9305  loss_sent:  3.879598  loss_word:  157.53252  Time cost:  6.750288248062134\n",
      "Start and end (1940, 1945)\n",
      "idx:  1940  Epoch:  113  loss:  342.564  loss_sent:  2.7516088  loss_word:  328.80594  Time cost:  6.982966661453247\n",
      "Start and end (1945, 1950)\n",
      "idx:  1945  Epoch:  113  loss:  231.53304  loss_sent:  4.1820693  loss_word:  210.62268  Time cost:  6.736034631729126\n",
      "Start and end (1950, 1955)\n",
      "idx:  1950  Epoch:  113  loss:  246.24797  loss_sent:  2.4463196  loss_word:  234.01636  Time cost:  6.684882402420044\n",
      "Start and end (1955, 1960)\n",
      "idx:  1955  Epoch:  113  loss:  208.80591  loss_sent:  2.237004  loss_word:  197.62086  Time cost:  6.9539124965667725\n",
      "Start and end (1960, 1965)\n",
      "idx:  1960  Epoch:  113  loss:  226.45828  loss_sent:  2.5704536  loss_word:  213.60599  Time cost:  6.699223518371582\n",
      "Start and end (1965, 1970)\n",
      "idx:  1965  Epoch:  113  loss:  252.83484  loss_sent:  2.6073117  loss_word:  239.79825  Time cost:  6.870828866958618\n",
      "Start and end (1970, 1975)\n",
      "idx:  1970  Epoch:  113  loss:  388.09927  loss_sent:  2.971324  loss_word:  373.24268  Time cost:  6.77827000617981\n",
      "Start and end (1975, 1980)\n",
      "idx:  1975  Epoch:  113  loss:  253.21825  loss_sent:  2.7695885  loss_word:  239.37029  Time cost:  6.8449647426605225\n",
      "Start and end (1980, 1985)\n",
      "idx:  1980  Epoch:  113  loss:  253.96735  loss_sent:  3.02922  loss_word:  238.82123  Time cost:  6.80099630355835\n",
      "Start and end (1985, 1990)\n",
      "idx:  1985  Epoch:  113  loss:  266.9102  loss_sent:  3.0044029  loss_word:  251.8882  Time cost:  6.676385879516602\n",
      "Start and end (1990, 1995)\n",
      "idx:  1990  Epoch:  113  loss:  265.5481  loss_sent:  2.965059  loss_word:  250.72278  Time cost:  6.767614364624023\n",
      "Start and end (1995, 2000)\n",
      "idx:  1995  Epoch:  113  loss:  248.59296  loss_sent:  2.8746686  loss_word:  234.2196  Time cost:  6.688294410705566\n",
      "Start and end (2000, 2005)\n",
      "idx:  2000  Epoch:  113  loss:  199.16154  loss_sent:  2.737915  loss_word:  185.47198  Time cost:  6.824886322021484\n",
      "Start and end (2005, 2010)\n",
      "idx:  2005  Epoch:  113  loss:  286.80563  loss_sent:  2.7102978  loss_word:  273.25415  Time cost:  6.766321897506714\n",
      "Start and end (2010, 2015)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2010  Epoch:  113  loss:  296.92255  loss_sent:  2.1798244  loss_word:  286.0234  Time cost:  6.893157482147217\n",
      "Start and end (2015, 2020)\n",
      "idx:  2015  Epoch:  113  loss:  334.26923  loss_sent:  3.6156576  loss_word:  316.19092  Time cost:  6.9587929248809814\n",
      "Start and end (2020, 2025)\n",
      "idx:  2020  Epoch:  113  loss:  175.40648  loss_sent:  2.6432881  loss_word:  162.19005  Time cost:  6.781388521194458\n",
      "Start and end (2025, 2030)\n",
      "idx:  2025  Epoch:  113  loss:  304.90347  loss_sent:  2.6047318  loss_word:  291.87985  Time cost:  6.81556248664856\n",
      "Start and end (2030, 2035)\n",
      "idx:  2030  Epoch:  113  loss:  292.72696  loss_sent:  2.522393  loss_word:  280.11505  Time cost:  6.7430219650268555\n",
      "Start and end (2035, 2040)\n",
      "idx:  2035  Epoch:  113  loss:  280.09937  loss_sent:  3.7900312  loss_word:  261.1492  Time cost:  6.873278379440308\n",
      "Start and end (2040, 2045)\n",
      "idx:  2040  Epoch:  113  loss:  335.56116  loss_sent:  4.172643  loss_word:  314.6979  Time cost:  6.6705427169799805\n",
      "Start and end (2045, 2050)\n",
      "idx:  2045  Epoch:  113  loss:  255.33371  loss_sent:  2.2109041  loss_word:  244.27919  Time cost:  7.026515007019043\n",
      "Start and end (2050, 2055)\n",
      "idx:  2050  Epoch:  113  loss:  380.7172  loss_sent:  2.8144488  loss_word:  366.6449  Time cost:  6.903300762176514\n",
      "Start and end (2055, 2060)\n",
      "idx:  2055  Epoch:  113  loss:  328.75085  loss_sent:  2.8404744  loss_word:  314.54846  Time cost:  7.280758380889893\n",
      "Start and end (2060, 2065)\n",
      "idx:  2060  Epoch:  113  loss:  313.87387  loss_sent:  3.2806103  loss_word:  297.4708  Time cost:  6.871735572814941\n",
      "Start and end (2065, 2070)\n",
      "idx:  2065  Epoch:  113  loss:  351.37  loss_sent:  3.8018293  loss_word:  332.3608  Time cost:  6.810319662094116\n",
      "Start and end (2070, 2075)\n",
      "idx:  2070  Epoch:  113  loss:  325.44537  loss_sent:  3.9396245  loss_word:  305.74716  Time cost:  6.818800926208496\n",
      "Start and end (2075, 2080)\n",
      "idx:  2075  Epoch:  113  loss:  283.02286  loss_sent:  2.855184  loss_word:  268.74695  Time cost:  6.7723681926727295\n",
      "Start and end (2080, 2085)\n",
      "idx:  2080  Epoch:  113  loss:  213.31273  loss_sent:  3.4446068  loss_word:  196.0897  Time cost:  7.2111499309539795\n",
      "Start and end (2085, 2090)\n",
      "idx:  2085  Epoch:  113  loss:  347.93814  loss_sent:  2.6177475  loss_word:  334.8494  Time cost:  6.9616382122039795\n",
      "Start and end (2090, 2095)\n",
      "idx:  2090  Epoch:  113  loss:  386.0082  loss_sent:  2.8103123  loss_word:  371.95667  Time cost:  6.8631227016448975\n",
      "Start and end (2095, 2100)\n",
      "idx:  2095  Epoch:  113  loss:  333.72345  loss_sent:  2.585917  loss_word:  320.79385  Time cost:  7.089816331863403\n",
      "Start and end (2100, 2105)\n",
      "idx:  2100  Epoch:  113  loss:  198.97456  loss_sent:  3.1534023  loss_word:  183.20755  Time cost:  6.948603391647339\n",
      "Start and end (2105, 2110)\n",
      "idx:  2105  Epoch:  113  loss:  204.78893  loss_sent:  2.5987203  loss_word:  191.79535  Time cost:  7.059372425079346\n",
      "Start and end (2110, 2115)\n",
      "idx:  2110  Epoch:  113  loss:  282.19443  loss_sent:  2.8293931  loss_word:  268.04752  Time cost:  6.9614667892456055\n",
      "Start and end (2115, 2120)\n",
      "idx:  2115  Epoch:  113  loss:  248.40369  loss_sent:  2.7442522  loss_word:  234.68243  Time cost:  7.111112594604492\n",
      "Start and end (2120, 2125)\n",
      "idx:  2120  Epoch:  113  loss:  288.63504  loss_sent:  2.5988343  loss_word:  275.6409  Time cost:  6.836414575576782\n",
      "Start and end (2125, 2130)\n",
      "idx:  2125  Epoch:  113  loss:  237.56108  loss_sent:  3.8318436  loss_word:  218.40189  Time cost:  6.8020219802856445\n",
      "Start and end (2130, 2135)\n",
      "idx:  2130  Epoch:  113  loss:  320.2875  loss_sent:  2.4162347  loss_word:  308.2063  Time cost:  7.005126237869263\n",
      "Start and end (2135, 2140)\n",
      "idx:  2135  Epoch:  113  loss:  251.47775  loss_sent:  2.618497  loss_word:  238.38525  Time cost:  6.656391859054565\n",
      "Start and end (2140, 2145)\n",
      "idx:  2140  Epoch:  113  loss:  237.32117  loss_sent:  3.2001853  loss_word:  221.32024  Time cost:  6.872685670852661\n",
      "Start and end (2145, 2150)\n",
      "idx:  2145  Epoch:  113  loss:  305.08353  loss_sent:  2.6346948  loss_word:  291.91006  Time cost:  6.774336814880371\n",
      "Start and end (2150, 2155)\n",
      "idx:  2150  Epoch:  113  loss:  233.8221  loss_sent:  3.00135  loss_word:  218.81532  Time cost:  6.7521069049835205\n",
      "Start and end (2155, 2160)\n",
      "idx:  2155  Epoch:  113  loss:  294.82083  loss_sent:  2.762625  loss_word:  281.00775  Time cost:  6.672224044799805\n",
      "Start and end (2160, 2165)\n",
      "idx:  2160  Epoch:  113  loss:  272.8593  loss_sent:  3.1587527  loss_word:  257.06558  Time cost:  6.719670534133911\n",
      "Start and end (2165, 2170)\n",
      "idx:  2165  Epoch:  113  loss:  285.11197  loss_sent:  2.6513605  loss_word:  271.85513  Time cost:  6.972744941711426\n",
      "Start and end (2170, 2175)\n",
      "idx:  2170  Epoch:  113  loss:  346.74918  loss_sent:  2.6022122  loss_word:  333.7381  Time cost:  6.74260139465332\n",
      "Start and end (2175, 2180)\n",
      "idx:  2175  Epoch:  113  loss:  281.45514  loss_sent:  2.6004941  loss_word:  268.45267  Time cost:  6.901274681091309\n",
      "Start and end (2180, 2185)\n",
      "idx:  2180  Epoch:  113  loss:  193.04375  loss_sent:  4.0500216  loss_word:  172.79366  Time cost:  6.701181650161743\n",
      "Start and end (2185, 2190)\n",
      "idx:  2185  Epoch:  113  loss:  237.9011  loss_sent:  2.170289  loss_word:  227.04965  Time cost:  6.746442794799805\n",
      "Start and end (2190, 2195)\n",
      "idx:  2190  Epoch:  113  loss:  205.33467  loss_sent:  2.587629  loss_word:  192.39651  Time cost:  6.843721389770508\n",
      "Start and end (2195, 2200)\n",
      "idx:  2195  Epoch:  113  loss:  291.1244  loss_sent:  2.6208708  loss_word:  278.02008  Time cost:  6.656026124954224\n",
      "Start and end (2200, 2205)\n",
      "idx:  2200  Epoch:  113  loss:  243.6468  loss_sent:  2.9106972  loss_word:  229.09332  Time cost:  6.75881814956665\n",
      "Start and end (2205, 2210)\n",
      "idx:  2205  Epoch:  113  loss:  252.28252  loss_sent:  2.9921694  loss_word:  237.3217  Time cost:  6.6879050731658936\n",
      "Start and end (2210, 2215)\n",
      "idx:  2210  Epoch:  113  loss:  180.49686  loss_sent:  2.4986339  loss_word:  168.0037  Time cost:  6.802850246429443\n",
      "Start and end (2215, 2220)\n",
      "idx:  2215  Epoch:  113  loss:  278.42664  loss_sent:  2.5868456  loss_word:  265.49237  Time cost:  6.67650294303894\n",
      "Start and end (2220, 2225)\n",
      "idx:  2220  Epoch:  113  loss:  359.3116  loss_sent:  2.1998901  loss_word:  348.31216  Time cost:  6.9758100509643555\n",
      "Start and end (2225, 2230)\n",
      "idx:  2225  Epoch:  113  loss:  289.90805  loss_sent:  2.2801242  loss_word:  278.50742  Time cost:  6.765123128890991\n",
      "Start and end (2230, 2235)\n",
      "idx:  2230  Epoch:  113  loss:  334.38113  loss_sent:  2.3045201  loss_word:  322.85855  Time cost:  6.634918451309204\n",
      "Start and end (2235, 2240)\n",
      "idx:  2235  Epoch:  113  loss:  271.2186  loss_sent:  4.0720916  loss_word:  250.85817  Time cost:  6.846714496612549\n",
      "Start and end (2240, 2245)\n",
      "idx:  2240  Epoch:  113  loss:  372.69565  loss_sent:  2.689646  loss_word:  359.2474  Time cost:  6.731944561004639\n",
      "Start and end (2245, 2250)\n",
      "idx:  2245  Epoch:  113  loss:  268.78223  loss_sent:  2.5009253  loss_word:  256.27765  Time cost:  6.91016697883606\n",
      "Start and end (2250, 2255)\n",
      "idx:  2250  Epoch:  113  loss:  241.87457  loss_sent:  2.2223835  loss_word:  230.76266  Time cost:  6.740192890167236\n",
      "Start and end (2255, 2260)\n",
      "idx:  2255  Epoch:  113  loss:  224.83801  loss_sent:  3.1077828  loss_word:  209.29912  Time cost:  6.899385213851929\n",
      "Start and end (2260, 2265)\n",
      "idx:  2260  Epoch:  113  loss:  254.45639  loss_sent:  2.405084  loss_word:  242.43098  Time cost:  6.668437957763672\n",
      "Start and end (2265, 2270)\n",
      "idx:  2265  Epoch:  113  loss:  205.70291  loss_sent:  3.0435548  loss_word:  190.48512  Time cost:  6.692081689834595\n",
      "Start and end (2270, 2275)\n",
      "idx:  2270  Epoch:  113  loss:  244.43169  loss_sent:  2.9999607  loss_word:  229.4319  Time cost:  6.848128318786621\n",
      "Start and end (2275, 2280)\n",
      "idx:  2275  Epoch:  113  loss:  233.70576  loss_sent:  2.506597  loss_word:  221.17278  Time cost:  6.757013320922852\n",
      "Start and end (2280, 2285)\n",
      "idx:  2280  Epoch:  113  loss:  268.2782  loss_sent:  2.8220546  loss_word:  254.16791  Time cost:  6.799279451370239\n",
      "Start and end (2285, 2290)\n",
      "idx:  2285  Epoch:  113  loss:  327.89133  loss_sent:  2.5824382  loss_word:  314.9792  Time cost:  6.692894220352173\n",
      "Start and end (2290, 2295)\n",
      "idx:  2290  Epoch:  113  loss:  229.10164  loss_sent:  3.7482998  loss_word:  210.36012  Time cost:  6.795072078704834\n",
      "Start and end (2295, 2300)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2295  Epoch:  113  loss:  229.55324  loss_sent:  2.1903324  loss_word:  218.6016  Time cost:  6.8650548458099365\n",
      "Start and end (2300, 2305)\n",
      "idx:  2300  Epoch:  113  loss:  263.84354  loss_sent:  2.1843386  loss_word:  252.92186  Time cost:  6.957227468490601\n",
      "Start and end (2305, 2310)\n",
      "idx:  2305  Epoch:  113  loss:  251.69379  loss_sent:  2.9693165  loss_word:  236.84721  Time cost:  7.066862344741821\n",
      "Start and end (2310, 2315)\n",
      "idx:  2310  Epoch:  113  loss:  318.12598  loss_sent:  2.467599  loss_word:  305.788  Time cost:  7.228207349777222\n",
      "Start and end (2315, 2320)\n",
      "idx:  2315  Epoch:  113  loss:  398.8962  loss_sent:  3.1844988  loss_word:  382.97372  Time cost:  7.4034583568573\n",
      "Start and end (2320, 2325)\n",
      "idx:  2320  Epoch:  113  loss:  284.66437  loss_sent:  3.6368477  loss_word:  266.4801  Time cost:  7.113168239593506\n",
      "Start and end (2325, 2330)\n",
      "idx:  2325  Epoch:  113  loss:  262.19263  loss_sent:  2.452576  loss_word:  249.92972  Time cost:  6.988577842712402\n",
      "Start and end (2330, 2335)\n",
      "idx:  2330  Epoch:  113  loss:  307.2237  loss_sent:  2.1047676  loss_word:  296.69986  Time cost:  7.313384294509888\n",
      "Start and end (2335, 2340)\n",
      "idx:  2335  Epoch:  113  loss:  279.23975  loss_sent:  2.6809497  loss_word:  265.83502  Time cost:  7.837671279907227\n",
      "Start and end (2340, 2345)\n",
      "idx:  2340  Epoch:  113  loss:  308.61456  loss_sent:  2.2578282  loss_word:  297.3254  Time cost:  8.032628774642944\n",
      "Start and end (2345, 2350)\n",
      "idx:  2345  Epoch:  113  loss:  244.4937  loss_sent:  4.3210835  loss_word:  222.88828  Time cost:  8.015475034713745\n",
      "Start and end (2350, 2355)\n",
      "idx:  2350  Epoch:  113  loss:  199.08762  loss_sent:  3.4391222  loss_word:  181.89201  Time cost:  7.356684923171997\n",
      "Start and end (2355, 2360)\n",
      "idx:  2355  Epoch:  113  loss:  321.94904  loss_sent:  4.9654684  loss_word:  297.12173  Time cost:  7.4094929695129395\n",
      "Start and end (2360, 2365)\n",
      "idx:  2360  Epoch:  113  loss:  343.38452  loss_sent:  2.738527  loss_word:  329.69193  Time cost:  6.979138374328613\n",
      "Start and end (2365, 2370)\n",
      "idx:  2365  Epoch:  113  loss:  367.26358  loss_sent:  2.5600166  loss_word:  354.46353  Time cost:  6.737873792648315\n",
      "Start and end (2370, 2375)\n",
      "idx:  2370  Epoch:  113  loss:  321.16217  loss_sent:  2.1659546  loss_word:  310.3324  Time cost:  6.7637763023376465\n",
      "Start and end (2375, 2380)\n",
      "idx:  2375  Epoch:  113  loss:  264.80066  loss_sent:  2.6769168  loss_word:  251.41608  Time cost:  6.57606053352356\n",
      "Start and end (2380, 2385)\n",
      "idx:  2380  Epoch:  113  loss:  134.7964  loss_sent:  2.7595406  loss_word:  120.99869  Time cost:  6.921296834945679\n",
      "Start and end (2385, 2390)\n",
      "idx:  2385  Epoch:  113  loss:  203.49232  loss_sent:  3.2595391  loss_word:  187.19464  Time cost:  6.67454981803894\n",
      "Start and end (2390, 2395)\n",
      "idx:  2390  Epoch:  113  loss:  257.1998  loss_sent:  3.1734538  loss_word:  241.3325  Time cost:  6.803440809249878\n",
      "Start and end (2395, 2400)\n",
      "idx:  2395  Epoch:  113  loss:  234.47343  loss_sent:  2.0654154  loss_word:  224.14636  Time cost:  6.794526815414429\n",
      "Start and end (2400, 2405)\n",
      "idx:  2400  Epoch:  113  loss:  295.89227  loss_sent:  2.8378038  loss_word:  281.70325  Time cost:  6.731711387634277\n",
      "Start and end (2405, 2410)\n",
      "idx:  2405  Epoch:  113  loss:  445.1229  loss_sent:  3.0072188  loss_word:  430.0868  Time cost:  6.783260107040405\n",
      "Start and end (2410, 2415)\n",
      "idx:  2410  Epoch:  113  loss:  423.31198  loss_sent:  3.0921404  loss_word:  407.85135  Time cost:  6.687660455703735\n",
      "Start and end (2415, 2420)\n",
      "idx:  2415  Epoch:  113  loss:  259.87808  loss_sent:  2.290339  loss_word:  248.42638  Time cost:  6.67230749130249\n",
      "Start and end (2420, 2425)\n",
      "idx:  2420  Epoch:  113  loss:  262.2894  loss_sent:  2.396141  loss_word:  250.3087  Time cost:  6.627747297286987\n",
      "Start and end (2425, 2430)\n",
      "idx:  2425  Epoch:  113  loss:  242.1723  loss_sent:  3.854826  loss_word:  222.89818  Time cost:  6.65903639793396\n",
      "Start and end (2430, 2435)\n",
      "idx:  2430  Epoch:  113  loss:  337.94455  loss_sent:  2.0782547  loss_word:  327.55325  Time cost:  6.926817893981934\n",
      "Start and end (2435, 2440)\n",
      "idx:  2435  Epoch:  113  loss:  269.37424  loss_sent:  2.1692438  loss_word:  258.52805  Time cost:  6.6769185066223145\n",
      "Start and end (2440, 2445)\n",
      "idx:  2440  Epoch:  113  loss:  191.25565  loss_sent:  3.5469782  loss_word:  173.52074  Time cost:  6.822598457336426\n",
      "Start and end (2445, 2450)\n",
      "idx:  2445  Epoch:  113  loss:  289.4192  loss_sent:  2.7577875  loss_word:  275.6303  Time cost:  6.797086000442505\n",
      "Start and end (2450, 2455)\n",
      "idx:  2450  Epoch:  113  loss:  245.62451  loss_sent:  2.112465  loss_word:  235.06218  Time cost:  6.68155312538147\n",
      "Start and end (2455, 2460)\n",
      "idx:  2455  Epoch:  113  loss:  409.95132  loss_sent:  2.338356  loss_word:  398.25952  Time cost:  6.840101957321167\n",
      "Start and end (2460, 2465)\n",
      "idx:  2460  Epoch:  113  loss:  275.92148  loss_sent:  2.3895314  loss_word:  263.97385  Time cost:  6.653002500534058\n",
      "Start and end (2465, 2470)\n",
      "idx:  2465  Epoch:  113  loss:  210.52307  loss_sent:  2.6363087  loss_word:  197.34154  Time cost:  6.647073745727539\n",
      "Start and end (2470, 2475)\n",
      "idx:  2470  Epoch:  113  loss:  376.62042  loss_sent:  2.872882  loss_word:  362.25604  Time cost:  6.607383489608765\n",
      "Start and end (2475, 2480)\n",
      "idx:  2475  Epoch:  113  loss:  198.01294  loss_sent:  2.3109388  loss_word:  186.45825  Time cost:  6.688430070877075\n",
      "Start and end (2480, 2485)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0724 10:09:09.613305 140197742454528 saver.py:1134] *******************************************************\n",
      "W0724 10:09:09.613727 140197742454528 saver.py:1135] TensorFlow's V1 checkpoint format has been deprecated.\n",
      "W0724 10:09:09.614110 140197742454528 saver.py:1136] Consider switching to the more efficient V2 format:\n",
      "W0724 10:09:09.614464 140197742454528 saver.py:1137]    `tf.train.Saver(write_version=tf.train.SaverDef.V2)`\n",
      "W0724 10:09:09.614828 140197742454528 saver.py:1138] now on by default.\n",
      "W0724 10:09:09.615251 140197742454528 saver.py:1139] *******************************************************\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2480  Epoch:  113  loss:  194.07774  loss_sent:  3.0841806  loss_word:  178.65685  Time cost:  6.697955131530762\n",
      "Successfully Written to temporary\n",
      "Epoch  113  is done. Saving the model ...\n",
      "Start and end (0, 5)\n",
      "idx:  0  Epoch:  114  loss:  263.93954  loss_sent:  2.1420796  loss_word:  253.22916  Time cost:  6.704392433166504\n",
      "Start and end (5, 10)\n",
      "idx:  5  Epoch:  114  loss:  278.46738  loss_sent:  2.3006606  loss_word:  266.96408  Time cost:  6.586405038833618\n",
      "Start and end (10, 15)\n",
      "idx:  10  Epoch:  114  loss:  273.84973  loss_sent:  2.351813  loss_word:  262.0906  Time cost:  6.475858449935913\n",
      "Start and end (15, 20)\n",
      "idx:  15  Epoch:  114  loss:  239.8149  loss_sent:  2.3208385  loss_word:  228.2107  Time cost:  6.626615762710571\n",
      "Start and end (20, 25)\n",
      "idx:  20  Epoch:  114  loss:  243.64311  loss_sent:  3.0705922  loss_word:  228.29016  Time cost:  6.4946980476379395\n",
      "Start and end (25, 30)\n",
      "idx:  25  Epoch:  114  loss:  269.88037  loss_sent:  2.7102666  loss_word:  256.3291  Time cost:  6.662930488586426\n",
      "Start and end (30, 35)\n",
      "idx:  30  Epoch:  114  loss:  237.29745  loss_sent:  2.1500633  loss_word:  226.54715  Time cost:  6.5159571170806885\n",
      "Start and end (35, 40)\n",
      "idx:  35  Epoch:  114  loss:  260.77686  loss_sent:  2.2892992  loss_word:  249.3304  Time cost:  6.588333368301392\n",
      "Start and end (40, 45)\n",
      "idx:  40  Epoch:  114  loss:  289.32883  loss_sent:  2.7476203  loss_word:  275.59073  Time cost:  6.728585720062256\n",
      "Start and end (45, 50)\n",
      "idx:  45  Epoch:  114  loss:  209.66158  loss_sent:  2.343757  loss_word:  197.94278  Time cost:  6.518690824508667\n",
      "Start and end (50, 55)\n",
      "idx:  50  Epoch:  114  loss:  255.63863  loss_sent:  2.6215396  loss_word:  242.53091  Time cost:  6.643179893493652\n",
      "Start and end (55, 60)\n",
      "idx:  55  Epoch:  114  loss:  198.10947  loss_sent:  1.8764172  loss_word:  188.72736  Time cost:  6.565732717514038\n",
      "Start and end (60, 65)\n",
      "idx:  60  Epoch:  114  loss:  317.77054  loss_sent:  2.3721268  loss_word:  305.90988  Time cost:  6.719760894775391\n",
      "Start and end (65, 70)\n",
      "idx:  65  Epoch:  114  loss:  194.92209  loss_sent:  2.8910053  loss_word:  180.46707  Time cost:  6.56584358215332\n",
      "Start and end (70, 75)\n",
      "idx:  70  Epoch:  114  loss:  210.10211  loss_sent:  2.4459121  loss_word:  197.87254  Time cost:  6.594735383987427\n",
      "Start and end (75, 80)\n",
      "idx:  75  Epoch:  114  loss:  309.8528  loss_sent:  2.2236438  loss_word:  298.7346  Time cost:  6.750370502471924\n",
      "Start and end (80, 85)\n",
      "idx:  80  Epoch:  114  loss:  215.78947  loss_sent:  2.8285902  loss_word:  201.64651  Time cost:  6.550562858581543\n",
      "Start and end (85, 90)\n",
      "idx:  85  Epoch:  114  loss:  276.3047  loss_sent:  2.2808528  loss_word:  264.90045  Time cost:  6.686158180236816\n",
      "Start and end (90, 95)\n",
      "idx:  90  Epoch:  114  loss:  268.00507  loss_sent:  2.9759128  loss_word:  253.12553  Time cost:  6.554614305496216\n",
      "Start and end (95, 100)\n",
      "idx:  95  Epoch:  114  loss:  333.9407  loss_sent:  2.208466  loss_word:  322.89838  Time cost:  6.565120458602905\n",
      "Start and end (100, 105)\n",
      "idx:  100  Epoch:  114  loss:  235.84834  loss_sent:  2.3383584  loss_word:  224.15654  Time cost:  6.722435474395752\n",
      "Start and end (105, 110)\n",
      "idx:  105  Epoch:  114  loss:  226.56602  loss_sent:  2.676212  loss_word:  213.185  Time cost:  6.575051307678223\n",
      "Start and end (110, 115)\n",
      "idx:  110  Epoch:  114  loss:  260.5741  loss_sent:  1.9089973  loss_word:  251.02914  Time cost:  6.700015544891357\n",
      "Start and end (115, 120)\n",
      "idx:  115  Epoch:  114  loss:  229.23914  loss_sent:  2.2690187  loss_word:  217.89404  Time cost:  6.556373596191406\n",
      "Start and end (120, 125)\n",
      "idx:  120  Epoch:  114  loss:  194.47745  loss_sent:  2.071509  loss_word:  184.1199  Time cost:  6.586076021194458\n",
      "Start and end (125, 130)\n",
      "idx:  125  Epoch:  114  loss:  182.27876  loss_sent:  2.3729851  loss_word:  170.41383  Time cost:  6.5113255977630615\n",
      "Start and end (130, 135)\n",
      "idx:  130  Epoch:  114  loss:  250.08176  loss_sent:  3.3781362  loss_word:  233.1911  Time cost:  6.5988688468933105\n",
      "Start and end (135, 140)\n",
      "idx:  135  Epoch:  114  loss:  251.7227  loss_sent:  3.1985464  loss_word:  235.72993  Time cost:  6.680274963378906\n",
      "Start and end (140, 145)\n",
      "idx:  140  Epoch:  114  loss:  192.3095  loss_sent:  2.40431  loss_word:  180.28795  Time cost:  6.568892955780029\n",
      "Start and end (145, 150)\n",
      "idx:  145  Epoch:  114  loss:  279.53973  loss_sent:  2.1074584  loss_word:  269.0024  Time cost:  6.688531398773193\n",
      "Start and end (150, 155)\n",
      "idx:  150  Epoch:  114  loss:  254.75848  loss_sent:  2.3545063  loss_word:  242.98595  Time cost:  6.488734722137451\n",
      "Start and end (155, 160)\n",
      "idx:  155  Epoch:  114  loss:  266.97235  loss_sent:  2.5017774  loss_word:  254.4635  Time cost:  6.56658148765564\n",
      "Start and end (160, 165)\n",
      "idx:  160  Epoch:  114  loss:  266.38593  loss_sent:  2.3525963  loss_word:  254.62294  Time cost:  6.679147243499756\n",
      "Start and end (165, 170)\n",
      "idx:  165  Epoch:  114  loss:  275.25635  loss_sent:  2.37492  loss_word:  263.38177  Time cost:  6.560340404510498\n",
      "Start and end (170, 175)\n",
      "idx:  170  Epoch:  114  loss:  236.9702  loss_sent:  2.6811466  loss_word:  223.56448  Time cost:  6.511199474334717\n",
      "Start and end (175, 180)\n",
      "idx:  175  Epoch:  114  loss:  267.6785  loss_sent:  2.3646104  loss_word:  255.85548  Time cost:  6.574557542800903\n",
      "Start and end (180, 185)\n",
      "idx:  180  Epoch:  114  loss:  227.91902  loss_sent:  2.4485297  loss_word:  215.67639  Time cost:  6.538697242736816\n",
      "Start and end (185, 190)\n",
      "idx:  185  Epoch:  114  loss:  336.9673  loss_sent:  2.3489707  loss_word:  325.2224  Time cost:  6.580099582672119\n",
      "Start and end (190, 195)\n",
      "idx:  190  Epoch:  114  loss:  324.73364  loss_sent:  2.7788708  loss_word:  310.83926  Time cost:  6.528263330459595\n",
      "Start and end (195, 200)\n",
      "idx:  195  Epoch:  114  loss:  228.82855  loss_sent:  2.3438814  loss_word:  217.10913  Time cost:  6.576756954193115\n",
      "Start and end (200, 205)\n",
      "idx:  200  Epoch:  114  loss:  239.69168  loss_sent:  3.5923147  loss_word:  221.73013  Time cost:  6.5654730796813965\n",
      "Start and end (205, 210)\n",
      "idx:  205  Epoch:  114  loss:  344.77042  loss_sent:  2.0936854  loss_word:  334.302  Time cost:  6.574354887008667\n",
      "Start and end (210, 215)\n",
      "idx:  210  Epoch:  114  loss:  248.99083  loss_sent:  2.024611  loss_word:  238.8678  Time cost:  6.943676948547363\n",
      "Start and end (215, 220)\n",
      "idx:  215  Epoch:  114  loss:  231.05989  loss_sent:  1.8257903  loss_word:  221.93095  Time cost:  7.921396017074585\n",
      "Start and end (220, 225)\n",
      "idx:  220  Epoch:  114  loss:  327.02527  loss_sent:  2.2284102  loss_word:  315.88318  Time cost:  7.508135080337524\n",
      "Start and end (225, 230)\n",
      "idx:  225  Epoch:  114  loss:  242.74373  loss_sent:  2.8088326  loss_word:  228.69955  Time cost:  7.772573709487915\n",
      "Start and end (230, 235)\n",
      "idx:  230  Epoch:  114  loss:  362.97794  loss_sent:  2.606983  loss_word:  349.94302  Time cost:  7.713158845901489\n",
      "Start and end (235, 240)\n",
      "idx:  235  Epoch:  114  loss:  247.62766  loss_sent:  3.2750235  loss_word:  231.25255  Time cost:  6.649390459060669\n",
      "Start and end (240, 245)\n",
      "idx:  240  Epoch:  114  loss:  252.31374  loss_sent:  2.1216872  loss_word:  241.70534  Time cost:  6.621993064880371\n",
      "Start and end (245, 250)\n",
      "idx:  245  Epoch:  114  loss:  161.76994  loss_sent:  2.456087  loss_word:  149.48952  Time cost:  6.441892623901367\n",
      "Start and end (250, 255)\n",
      "idx:  250  Epoch:  114  loss:  309.20154  loss_sent:  3.8470867  loss_word:  289.96613  Time cost:  6.543310642242432\n",
      "Start and end (255, 260)\n",
      "idx:  255  Epoch:  114  loss:  199.26346  loss_sent:  3.0642695  loss_word:  183.94212  Time cost:  6.598412036895752\n",
      "Start and end (260, 265)\n",
      "idx:  260  Epoch:  114  loss:  266.62814  loss_sent:  2.3333735  loss_word:  254.96124  Time cost:  6.607621431350708\n",
      "Start and end (265, 270)\n",
      "idx:  265  Epoch:  114  loss:  229.999  loss_sent:  2.54698  loss_word:  217.26408  Time cost:  6.47503137588501\n",
      "Start and end (270, 275)\n",
      "idx:  270  Epoch:  114  loss:  217.96327  loss_sent:  3.1152213  loss_word:  202.38719  Time cost:  6.55815052986145\n",
      "Start and end (275, 280)\n",
      "idx:  275  Epoch:  114  loss:  289.4534  loss_sent:  2.1116712  loss_word:  278.89505  Time cost:  6.546214818954468\n",
      "Start and end (280, 285)\n",
      "idx:  280  Epoch:  114  loss:  263.03705  loss_sent:  1.9479783  loss_word:  253.29715  Time cost:  6.534805536270142\n",
      "Start and end (285, 290)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  285  Epoch:  114  loss:  197.1803  loss_sent:  2.495774  loss_word:  184.70142  Time cost:  6.5290350914001465\n",
      "Start and end (290, 295)\n",
      "idx:  290  Epoch:  114  loss:  234.1469  loss_sent:  3.3449595  loss_word:  217.4221  Time cost:  6.5470569133758545\n",
      "Start and end (295, 300)\n",
      "idx:  295  Epoch:  114  loss:  299.0423  loss_sent:  2.529513  loss_word:  286.39468  Time cost:  6.53415584564209\n",
      "Start and end (300, 305)\n",
      "idx:  300  Epoch:  114  loss:  301.73077  loss_sent:  1.9413693  loss_word:  292.02396  Time cost:  6.576545715332031\n",
      "Start and end (305, 310)\n",
      "idx:  305  Epoch:  114  loss:  184.5377  loss_sent:  2.8845356  loss_word:  170.11502  Time cost:  6.56409478187561\n",
      "Start and end (310, 315)\n",
      "idx:  310  Epoch:  114  loss:  343.08585  loss_sent:  1.7202566  loss_word:  334.48456  Time cost:  6.572478771209717\n",
      "Start and end (315, 320)\n",
      "idx:  315  Epoch:  114  loss:  281.5258  loss_sent:  2.2845075  loss_word:  270.10327  Time cost:  6.532787084579468\n",
      "Start and end (320, 325)\n",
      "idx:  320  Epoch:  114  loss:  249.94737  loss_sent:  2.5618815  loss_word:  237.13799  Time cost:  6.540684223175049\n",
      "Start and end (325, 330)\n",
      "idx:  325  Epoch:  114  loss:  256.71643  loss_sent:  2.3807783  loss_word:  244.81258  Time cost:  6.506886720657349\n",
      "Start and end (330, 335)\n",
      "idx:  330  Epoch:  114  loss:  208.43715  loss_sent:  2.3401449  loss_word:  196.7364  Time cost:  6.503141403198242\n",
      "Start and end (335, 340)\n",
      "idx:  335  Epoch:  114  loss:  260.13544  loss_sent:  1.9953059  loss_word:  250.15894  Time cost:  6.542024374008179\n",
      "Start and end (340, 345)\n",
      "idx:  340  Epoch:  114  loss:  293.08347  loss_sent:  1.6794918  loss_word:  284.68597  Time cost:  6.583132743835449\n",
      "Start and end (345, 350)\n",
      "idx:  345  Epoch:  114  loss:  257.32007  loss_sent:  2.5783787  loss_word:  244.42812  Time cost:  7.229051351547241\n",
      "Start and end (350, 355)\n",
      "idx:  350  Epoch:  114  loss:  247.34746  loss_sent:  2.3667874  loss_word:  235.51353  Time cost:  6.875873327255249\n",
      "Start and end (355, 360)\n",
      "idx:  355  Epoch:  114  loss:  257.23648  loss_sent:  1.9795176  loss_word:  247.33885  Time cost:  6.686509847640991\n",
      "Start and end (360, 365)\n",
      "idx:  360  Epoch:  114  loss:  268.85504  loss_sent:  1.8400943  loss_word:  259.65457  Time cost:  6.479562520980835\n",
      "Start and end (365, 370)\n",
      "idx:  365  Epoch:  114  loss:  273.7718  loss_sent:  2.7058442  loss_word:  260.24258  Time cost:  6.514328241348267\n",
      "Start and end (370, 375)\n",
      "idx:  370  Epoch:  114  loss:  203.86404  loss_sent:  2.2165203  loss_word:  192.78146  Time cost:  6.86850905418396\n",
      "Start and end (375, 380)\n",
      "idx:  375  Epoch:  114  loss:  225.33633  loss_sent:  2.3681529  loss_word:  213.49556  Time cost:  7.464426279067993\n",
      "Start and end (380, 385)\n",
      "idx:  380  Epoch:  114  loss:  237.89311  loss_sent:  2.7486186  loss_word:  224.15001  Time cost:  6.5844407081604\n",
      "Start and end (385, 390)\n",
      "idx:  385  Epoch:  114  loss:  187.77347  loss_sent:  2.0780683  loss_word:  177.38312  Time cost:  6.564818382263184\n",
      "Start and end (390, 395)\n",
      "idx:  390  Epoch:  114  loss:  280.60284  loss_sent:  2.7354853  loss_word:  266.92538  Time cost:  7.09868860244751\n",
      "Start and end (395, 400)\n",
      "idx:  395  Epoch:  114  loss:  223.63881  loss_sent:  2.1415377  loss_word:  212.93108  Time cost:  7.051708459854126\n",
      "Start and end (400, 405)\n",
      "idx:  400  Epoch:  114  loss:  213.26683  loss_sent:  2.7217855  loss_word:  199.65794  Time cost:  6.535442113876343\n",
      "Start and end (405, 410)\n",
      "idx:  405  Epoch:  114  loss:  328.77402  loss_sent:  2.5673528  loss_word:  315.93726  Time cost:  6.477759122848511\n",
      "Start and end (410, 415)\n",
      "idx:  410  Epoch:  114  loss:  302.47916  loss_sent:  2.3501196  loss_word:  290.72855  Time cost:  6.56718373298645\n",
      "Start and end (415, 420)\n",
      "idx:  415  Epoch:  114  loss:  276.42706  loss_sent:  2.6562452  loss_word:  263.14587  Time cost:  6.52143120765686\n",
      "Start and end (420, 425)\n",
      "idx:  420  Epoch:  114  loss:  214.59145  loss_sent:  2.7865396  loss_word:  200.65874  Time cost:  6.468605995178223\n",
      "Start and end (425, 430)\n",
      "idx:  425  Epoch:  114  loss:  190.71832  loss_sent:  2.6187658  loss_word:  177.62451  Time cost:  6.508667469024658\n",
      "Start and end (430, 435)\n",
      "idx:  430  Epoch:  114  loss:  282.78235  loss_sent:  2.6163135  loss_word:  269.7008  Time cost:  6.639283895492554\n",
      "Start and end (435, 440)\n",
      "idx:  435  Epoch:  114  loss:  361.66046  loss_sent:  3.057561  loss_word:  346.37268  Time cost:  6.466781139373779\n",
      "Start and end (440, 445)\n",
      "idx:  440  Epoch:  114  loss:  350.97864  loss_sent:  2.5073957  loss_word:  338.44165  Time cost:  6.634949207305908\n",
      "Start and end (445, 450)\n",
      "idx:  445  Epoch:  114  loss:  227.37273  loss_sent:  2.1301758  loss_word:  216.72185  Time cost:  6.451947212219238\n",
      "Start and end (450, 455)\n",
      "idx:  450  Epoch:  114  loss:  278.24915  loss_sent:  3.9548018  loss_word:  258.4751  Time cost:  6.535090684890747\n",
      "Start and end (455, 460)\n",
      "idx:  455  Epoch:  114  loss:  186.0354  loss_sent:  3.4219494  loss_word:  168.92563  Time cost:  6.517048597335815\n",
      "Start and end (460, 465)\n",
      "idx:  460  Epoch:  114  loss:  209.5999  loss_sent:  2.5747874  loss_word:  196.72598  Time cost:  6.404980659484863\n",
      "Start and end (465, 470)\n",
      "idx:  465  Epoch:  114  loss:  253.8016  loss_sent:  2.347571  loss_word:  242.06375  Time cost:  6.5197389125823975\n",
      "Start and end (470, 475)\n",
      "idx:  470  Epoch:  114  loss:  320.21164  loss_sent:  2.8948247  loss_word:  305.73752  Time cost:  6.527169942855835\n",
      "Start and end (475, 480)\n",
      "idx:  475  Epoch:  114  loss:  293.90466  loss_sent:  2.8896816  loss_word:  279.45624  Time cost:  6.5108137130737305\n",
      "Start and end (480, 485)\n",
      "idx:  480  Epoch:  114  loss:  340.39667  loss_sent:  1.9747882  loss_word:  330.5227  Time cost:  6.408391714096069\n",
      "Start and end (485, 490)\n",
      "idx:  485  Epoch:  114  loss:  293.09012  loss_sent:  2.1355724  loss_word:  282.4122  Time cost:  6.5677807331085205\n",
      "Start and end (490, 495)\n",
      "idx:  490  Epoch:  114  loss:  313.89194  loss_sent:  2.260486  loss_word:  302.5895  Time cost:  6.592797756195068\n",
      "Start and end (495, 500)\n",
      "idx:  495  Epoch:  114  loss:  265.69424  loss_sent:  2.3432715  loss_word:  253.9779  Time cost:  6.560377836227417\n",
      "Start and end (500, 505)\n",
      "idx:  500  Epoch:  114  loss:  225.7089  loss_sent:  2.5619004  loss_word:  212.89938  Time cost:  6.435641050338745\n",
      "Start and end (505, 510)\n",
      "idx:  505  Epoch:  114  loss:  223.81976  loss_sent:  2.0132506  loss_word:  213.75351  Time cost:  6.520283937454224\n",
      "Start and end (510, 515)\n",
      "idx:  510  Epoch:  114  loss:  286.49988  loss_sent:  2.5161948  loss_word:  273.9189  Time cost:  6.569298982620239\n",
      "Start and end (515, 520)\n",
      "idx:  515  Epoch:  114  loss:  275.11346  loss_sent:  2.2375867  loss_word:  263.92554  Time cost:  6.447113513946533\n",
      "Start and end (520, 525)\n",
      "idx:  520  Epoch:  114  loss:  338.75494  loss_sent:  2.5029979  loss_word:  326.23996  Time cost:  6.55928897857666\n",
      "Start and end (525, 530)\n",
      "idx:  525  Epoch:  114  loss:  334.78973  loss_sent:  2.393255  loss_word:  322.82346  Time cost:  6.555083513259888\n",
      "Start and end (530, 535)\n",
      "idx:  530  Epoch:  114  loss:  431.4757  loss_sent:  2.509072  loss_word:  418.93036  Time cost:  6.608292579650879\n",
      "Start and end (535, 540)\n",
      "idx:  535  Epoch:  114  loss:  254.19638  loss_sent:  2.0539608  loss_word:  243.92657  Time cost:  6.493811845779419\n",
      "Start and end (540, 545)\n",
      "idx:  540  Epoch:  114  loss:  306.08856  loss_sent:  2.374532  loss_word:  294.21594  Time cost:  6.705459356307983\n",
      "Start and end (545, 550)\n",
      "idx:  545  Epoch:  114  loss:  224.49416  loss_sent:  2.10107  loss_word:  213.98882  Time cost:  6.5719404220581055\n",
      "Start and end (550, 555)\n",
      "idx:  550  Epoch:  114  loss:  217.98195  loss_sent:  2.2609963  loss_word:  206.67697  Time cost:  6.519832611083984\n",
      "Start and end (555, 560)\n",
      "idx:  555  Epoch:  114  loss:  178.48839  loss_sent:  2.256794  loss_word:  167.2044  Time cost:  6.534584999084473\n",
      "Start and end (560, 565)\n",
      "idx:  560  Epoch:  114  loss:  215.42697  loss_sent:  2.3195524  loss_word:  203.82921  Time cost:  6.508867979049683\n",
      "Start and end (565, 570)\n",
      "idx:  565  Epoch:  114  loss:  247.16035  loss_sent:  2.5189898  loss_word:  234.56538  Time cost:  6.416352033615112\n",
      "Start and end (570, 575)\n",
      "idx:  570  Epoch:  114  loss:  290.3531  loss_sent:  2.116679  loss_word:  279.76968  Time cost:  6.5437846183776855\n",
      "Start and end (575, 580)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  575  Epoch:  114  loss:  268.87256  loss_sent:  2.0021124  loss_word:  258.86197  Time cost:  6.51788067817688\n",
      "Start and end (580, 585)\n",
      "idx:  580  Epoch:  114  loss:  263.12943  loss_sent:  2.4230154  loss_word:  251.01437  Time cost:  6.54920506477356\n",
      "Start and end (585, 590)\n",
      "idx:  585  Epoch:  114  loss:  274.0552  loss_sent:  2.0856848  loss_word:  263.62677  Time cost:  6.501956939697266\n",
      "Start and end (590, 595)\n",
      "idx:  590  Epoch:  114  loss:  216.9658  loss_sent:  2.1345117  loss_word:  206.29324  Time cost:  6.532183647155762\n",
      "Start and end (595, 600)\n",
      "idx:  595  Epoch:  114  loss:  249.91379  loss_sent:  2.2390366  loss_word:  238.7186  Time cost:  6.548753023147583\n",
      "Start and end (600, 605)\n",
      "idx:  600  Epoch:  114  loss:  271.1223  loss_sent:  2.6808283  loss_word:  257.71823  Time cost:  6.584240913391113\n",
      "Start and end (605, 610)\n",
      "idx:  605  Epoch:  114  loss:  230.58629  loss_sent:  2.5169287  loss_word:  218.00166  Time cost:  6.432028532028198\n",
      "Start and end (610, 615)\n",
      "idx:  610  Epoch:  114  loss:  249.19005  loss_sent:  1.8165405  loss_word:  240.10735  Time cost:  6.480851888656616\n",
      "Start and end (615, 620)\n",
      "idx:  615  Epoch:  114  loss:  305.08157  loss_sent:  2.0815609  loss_word:  294.67374  Time cost:  6.531888008117676\n",
      "Start and end (620, 625)\n",
      "idx:  620  Epoch:  114  loss:  279.2528  loss_sent:  1.966565  loss_word:  269.42  Time cost:  6.4805309772491455\n",
      "Start and end (625, 630)\n",
      "idx:  625  Epoch:  114  loss:  271.42773  loss_sent:  2.2244778  loss_word:  260.30536  Time cost:  6.476056337356567\n",
      "Start and end (630, 635)\n",
      "idx:  630  Epoch:  114  loss:  253.06961  loss_sent:  2.388248  loss_word:  241.12836  Time cost:  6.5049355030059814\n",
      "Start and end (635, 640)\n",
      "idx:  635  Epoch:  114  loss:  383.4304  loss_sent:  3.075471  loss_word:  368.05307  Time cost:  6.4809486865997314\n",
      "Start and end (640, 645)\n",
      "idx:  640  Epoch:  114  loss:  279.81442  loss_sent:  2.237957  loss_word:  268.62466  Time cost:  6.524064540863037\n",
      "Start and end (645, 650)\n",
      "idx:  645  Epoch:  114  loss:  239.20097  loss_sent:  2.3624382  loss_word:  227.3888  Time cost:  6.5029661655426025\n",
      "Start and end (650, 655)\n",
      "idx:  650  Epoch:  114  loss:  410.571  loss_sent:  2.1576185  loss_word:  399.78293  Time cost:  6.56158185005188\n",
      "Start and end (655, 660)\n",
      "idx:  655  Epoch:  114  loss:  226.46767  loss_sent:  3.2638004  loss_word:  210.14867  Time cost:  6.408161163330078\n",
      "Start and end (660, 665)\n",
      "idx:  660  Epoch:  114  loss:  284.86017  loss_sent:  4.6825705  loss_word:  261.44733  Time cost:  6.4086713790893555\n",
      "Start and end (665, 670)\n",
      "idx:  665  Epoch:  114  loss:  276.79617  loss_sent:  2.9105837  loss_word:  262.24326  Time cost:  6.512293577194214\n",
      "Start and end (670, 675)\n",
      "idx:  670  Epoch:  114  loss:  266.0864  loss_sent:  2.6895342  loss_word:  252.63873  Time cost:  6.58309531211853\n",
      "Start and end (675, 680)\n",
      "idx:  675  Epoch:  114  loss:  249.98663  loss_sent:  1.8777184  loss_word:  240.59804  Time cost:  6.456973075866699\n",
      "Start and end (680, 685)\n",
      "idx:  680  Epoch:  114  loss:  291.1401  loss_sent:  2.6940017  loss_word:  277.67007  Time cost:  6.405693531036377\n",
      "Start and end (685, 690)\n",
      "idx:  685  Epoch:  114  loss:  263.70456  loss_sent:  3.016037  loss_word:  248.62437  Time cost:  6.52649712562561\n",
      "Start and end (690, 695)\n",
      "idx:  690  Epoch:  114  loss:  229.56796  loss_sent:  2.3864388  loss_word:  217.63576  Time cost:  6.5644731521606445\n",
      "Start and end (695, 700)\n",
      "idx:  695  Epoch:  114  loss:  311.33554  loss_sent:  2.513513  loss_word:  298.768  Time cost:  6.484561443328857\n",
      "Start and end (700, 705)\n",
      "idx:  700  Epoch:  114  loss:  268.89825  loss_sent:  3.6370902  loss_word:  250.71288  Time cost:  6.45702862739563\n",
      "Start and end (705, 710)\n",
      "idx:  705  Epoch:  114  loss:  168.55386  loss_sent:  1.8450686  loss_word:  159.32852  Time cost:  6.403562307357788\n",
      "Start and end (710, 715)\n",
      "idx:  710  Epoch:  114  loss:  235.57341  loss_sent:  2.412303  loss_word:  223.5119  Time cost:  6.428446292877197\n",
      "Start and end (715, 720)\n",
      "idx:  715  Epoch:  114  loss:  195.79216  loss_sent:  2.2889237  loss_word:  184.34753  Time cost:  6.5179808139801025\n",
      "Start and end (720, 725)\n",
      "idx:  720  Epoch:  114  loss:  252.78696  loss_sent:  2.3120315  loss_word:  241.22678  Time cost:  6.567267179489136\n",
      "Start and end (725, 730)\n",
      "idx:  725  Epoch:  114  loss:  232.7414  loss_sent:  2.1227622  loss_word:  222.12758  Time cost:  6.520174503326416\n",
      "Start and end (730, 735)\n",
      "idx:  730  Epoch:  114  loss:  261.09778  loss_sent:  2.7112675  loss_word:  247.54147  Time cost:  6.572999477386475\n",
      "Start and end (735, 740)\n",
      "idx:  735  Epoch:  114  loss:  286.3363  loss_sent:  2.1219997  loss_word:  275.7263  Time cost:  6.475531339645386\n",
      "Start and end (740, 745)\n",
      "idx:  740  Epoch:  114  loss:  298.97217  loss_sent:  3.628549  loss_word:  280.82935  Time cost:  6.549971580505371\n",
      "Start and end (745, 750)\n",
      "idx:  745  Epoch:  114  loss:  215.33443  loss_sent:  2.180591  loss_word:  204.43149  Time cost:  6.398023366928101\n",
      "Start and end (750, 755)\n",
      "idx:  750  Epoch:  114  loss:  342.00735  loss_sent:  2.019517  loss_word:  331.90976  Time cost:  6.436585187911987\n",
      "Start and end (755, 760)\n",
      "idx:  755  Epoch:  114  loss:  221.1244  loss_sent:  1.8566383  loss_word:  211.84122  Time cost:  6.519165992736816\n",
      "Start and end (760, 765)\n",
      "idx:  760  Epoch:  114  loss:  332.37012  loss_sent:  2.4882643  loss_word:  319.92877  Time cost:  6.5764548778533936\n",
      "Start and end (765, 770)\n",
      "idx:  765  Epoch:  114  loss:  339.99216  loss_sent:  2.4856434  loss_word:  327.56393  Time cost:  6.50349760055542\n",
      "Start and end (770, 775)\n",
      "idx:  770  Epoch:  114  loss:  178.1747  loss_sent:  2.5008318  loss_word:  165.67052  Time cost:  6.436397552490234\n",
      "Start and end (775, 780)\n",
      "idx:  775  Epoch:  114  loss:  319.08157  loss_sent:  1.8916099  loss_word:  309.62357  Time cost:  6.542182683944702\n",
      "Start and end (780, 785)\n",
      "idx:  780  Epoch:  114  loss:  243.97687  loss_sent:  2.2814777  loss_word:  232.56946  Time cost:  6.553765058517456\n",
      "Start and end (785, 790)\n",
      "idx:  785  Epoch:  114  loss:  254.00203  loss_sent:  1.7684534  loss_word:  245.15977  Time cost:  6.427162170410156\n",
      "Start and end (790, 795)\n",
      "idx:  790  Epoch:  114  loss:  217.81516  loss_sent:  2.0090148  loss_word:  207.77005  Time cost:  6.484112977981567\n",
      "Start and end (795, 800)\n",
      "idx:  795  Epoch:  114  loss:  350.2764  loss_sent:  2.145548  loss_word:  339.54865  Time cost:  6.538984537124634\n",
      "Start and end (800, 805)\n",
      "idx:  800  Epoch:  114  loss:  207.12466  loss_sent:  3.3902144  loss_word:  190.17358  Time cost:  6.450977802276611\n",
      "Start and end (805, 810)\n",
      "idx:  805  Epoch:  114  loss:  295.23184  loss_sent:  2.292472  loss_word:  283.7695  Time cost:  6.425680637359619\n",
      "Start and end (810, 815)\n",
      "idx:  810  Epoch:  114  loss:  265.75113  loss_sent:  2.2545135  loss_word:  254.47853  Time cost:  6.476011753082275\n",
      "Start and end (815, 820)\n",
      "idx:  815  Epoch:  114  loss:  216.98135  loss_sent:  2.5149674  loss_word:  204.40654  Time cost:  6.498136281967163\n",
      "Start and end (820, 825)\n",
      "idx:  820  Epoch:  114  loss:  292.39008  loss_sent:  2.3346798  loss_word:  280.7167  Time cost:  6.428835868835449\n",
      "Start and end (825, 830)\n",
      "idx:  825  Epoch:  114  loss:  253.5927  loss_sent:  2.0101745  loss_word:  243.54182  Time cost:  6.5142951011657715\n",
      "Start and end (830, 835)\n",
      "idx:  830  Epoch:  114  loss:  250.92027  loss_sent:  1.9624882  loss_word:  241.10785  Time cost:  6.521812677383423\n",
      "Start and end (835, 840)\n",
      "idx:  835  Epoch:  114  loss:  341.2198  loss_sent:  2.4153035  loss_word:  329.14328  Time cost:  6.467557907104492\n",
      "Start and end (840, 845)\n",
      "idx:  840  Epoch:  114  loss:  285.45056  loss_sent:  2.0128403  loss_word:  275.3864  Time cost:  6.548367738723755\n",
      "Start and end (845, 850)\n",
      "idx:  845  Epoch:  114  loss:  262.44952  loss_sent:  2.445274  loss_word:  250.22316  Time cost:  6.548161745071411\n",
      "Start and end (850, 855)\n",
      "idx:  850  Epoch:  114  loss:  209.07886  loss_sent:  2.0825813  loss_word:  198.66594  Time cost:  6.515569448471069\n",
      "Start and end (855, 860)\n",
      "idx:  855  Epoch:  114  loss:  271.87973  loss_sent:  1.9544818  loss_word:  262.10733  Time cost:  6.5036773681640625\n",
      "Start and end (860, 865)\n",
      "idx:  860  Epoch:  114  loss:  216.27736  loss_sent:  1.8372985  loss_word:  207.09087  Time cost:  6.516161203384399\n",
      "Start and end (865, 870)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  865  Epoch:  114  loss:  266.794  loss_sent:  2.092166  loss_word:  256.33316  Time cost:  6.466049432754517\n",
      "Start and end (870, 875)\n",
      "idx:  870  Epoch:  114  loss:  325.29248  loss_sent:  2.1743062  loss_word:  314.42096  Time cost:  6.581722021102905\n",
      "Start and end (875, 880)\n",
      "idx:  875  Epoch:  114  loss:  283.01608  loss_sent:  1.9532193  loss_word:  273.24997  Time cost:  6.553986310958862\n",
      "Start and end (880, 885)\n",
      "idx:  880  Epoch:  114  loss:  222.43887  loss_sent:  2.351213  loss_word:  210.68282  Time cost:  6.5137619972229\n",
      "Start and end (885, 890)\n",
      "idx:  885  Epoch:  114  loss:  331.18826  loss_sent:  2.2377372  loss_word:  319.99957  Time cost:  6.53497314453125\n",
      "Start and end (890, 895)\n",
      "idx:  890  Epoch:  114  loss:  211.98907  loss_sent:  2.9278407  loss_word:  197.34987  Time cost:  6.46548056602478\n",
      "Start and end (895, 900)\n",
      "idx:  895  Epoch:  114  loss:  348.43112  loss_sent:  2.1320174  loss_word:  337.77106  Time cost:  6.49746561050415\n",
      "Start and end (900, 905)\n",
      "idx:  900  Epoch:  114  loss:  256.64282  loss_sent:  1.8630761  loss_word:  247.32742  Time cost:  6.506471872329712\n",
      "Start and end (905, 910)\n",
      "idx:  905  Epoch:  114  loss:  187.11113  loss_sent:  2.0566282  loss_word:  176.828  Time cost:  6.5282087326049805\n",
      "Start and end (910, 915)\n",
      "idx:  910  Epoch:  114  loss:  323.86212  loss_sent:  1.9345572  loss_word:  314.18936  Time cost:  6.4355552196502686\n",
      "Start and end (915, 920)\n",
      "idx:  915  Epoch:  114  loss:  213.37936  loss_sent:  2.2611027  loss_word:  202.07385  Time cost:  6.495218276977539\n",
      "Start and end (920, 925)\n",
      "idx:  920  Epoch:  114  loss:  302.41324  loss_sent:  2.240245  loss_word:  291.21204  Time cost:  6.557423114776611\n",
      "Start and end (925, 930)\n",
      "idx:  925  Epoch:  114  loss:  229.21318  loss_sent:  2.4840066  loss_word:  216.79314  Time cost:  6.442646026611328\n",
      "Start and end (930, 935)\n",
      "idx:  930  Epoch:  114  loss:  252.00603  loss_sent:  2.2938263  loss_word:  240.53691  Time cost:  6.4441139698028564\n",
      "Start and end (935, 940)\n",
      "idx:  935  Epoch:  114  loss:  269.95197  loss_sent:  2.0440016  loss_word:  259.73196  Time cost:  6.519140720367432\n",
      "Start and end (940, 945)\n",
      "idx:  940  Epoch:  114  loss:  154.60219  loss_sent:  2.4007156  loss_word:  142.59863  Time cost:  6.511582136154175\n",
      "Start and end (945, 950)\n",
      "idx:  945  Epoch:  114  loss:  221.69518  loss_sent:  2.4338484  loss_word:  209.52594  Time cost:  6.513799667358398\n",
      "Start and end (950, 955)\n",
      "idx:  950  Epoch:  114  loss:  277.8516  loss_sent:  2.4719064  loss_word:  265.49213  Time cost:  6.507369756698608\n",
      "Start and end (955, 960)\n",
      "idx:  955  Epoch:  114  loss:  250.01753  loss_sent:  2.0953407  loss_word:  239.54085  Time cost:  6.523514032363892\n",
      "Start and end (960, 965)\n",
      "idx:  960  Epoch:  114  loss:  190.37807  loss_sent:  3.3574972  loss_word:  173.59056  Time cost:  6.442065238952637\n",
      "Start and end (965, 970)\n",
      "idx:  965  Epoch:  114  loss:  253.4873  loss_sent:  2.4818738  loss_word:  241.07794  Time cost:  6.532042741775513\n",
      "Start and end (970, 975)\n",
      "idx:  970  Epoch:  114  loss:  265.99603  loss_sent:  2.555462  loss_word:  253.21872  Time cost:  6.446530818939209\n",
      "Start and end (975, 980)\n",
      "idx:  975  Epoch:  114  loss:  278.3362  loss_sent:  2.0773165  loss_word:  267.94962  Time cost:  6.521822929382324\n",
      "Start and end (980, 985)\n",
      "idx:  980  Epoch:  114  loss:  154.68907  loss_sent:  2.2503595  loss_word:  143.43729  Time cost:  6.446771144866943\n",
      "Start and end (985, 990)\n",
      "idx:  985  Epoch:  114  loss:  184.25218  loss_sent:  2.046071  loss_word:  174.02184  Time cost:  6.500364065170288\n",
      "Start and end (990, 995)\n",
      "idx:  990  Epoch:  114  loss:  275.12222  loss_sent:  2.1764839  loss_word:  264.23978  Time cost:  6.508967161178589\n",
      "Start and end (995, 1000)\n",
      "idx:  995  Epoch:  114  loss:  250.88902  loss_sent:  2.3247669  loss_word:  239.2652  Time cost:  6.52466082572937\n",
      "Start and end (1000, 1005)\n",
      "idx:  1000  Epoch:  114  loss:  263.91302  loss_sent:  2.21239  loss_word:  252.8511  Time cost:  6.473715543746948\n",
      "Start and end (1005, 1010)\n",
      "idx:  1005  Epoch:  114  loss:  263.8891  loss_sent:  1.8165019  loss_word:  254.80656  Time cost:  6.535984754562378\n",
      "Start and end (1010, 1015)\n",
      "idx:  1010  Epoch:  114  loss:  290.157  loss_sent:  2.0445163  loss_word:  279.93445  Time cost:  6.550436735153198\n",
      "Start and end (1015, 1020)\n",
      "idx:  1015  Epoch:  114  loss:  271.32425  loss_sent:  1.9764948  loss_word:  261.44177  Time cost:  6.479907274246216\n",
      "Start and end (1020, 1025)\n",
      "idx:  1020  Epoch:  114  loss:  269.81302  loss_sent:  1.8270993  loss_word:  260.67752  Time cost:  6.4982006549835205\n",
      "Start and end (1025, 1030)\n",
      "idx:  1025  Epoch:  114  loss:  232.46185  loss_sent:  2.1211305  loss_word:  221.85619  Time cost:  6.516316652297974\n",
      "Start and end (1030, 1035)\n",
      "idx:  1030  Epoch:  114  loss:  213.58571  loss_sent:  2.082607  loss_word:  203.17265  Time cost:  6.48374342918396\n",
      "Start and end (1035, 1040)\n",
      "idx:  1035  Epoch:  114  loss:  336.6693  loss_sent:  2.2581382  loss_word:  325.37863  Time cost:  6.530063152313232\n",
      "Start and end (1040, 1045)\n",
      "idx:  1040  Epoch:  114  loss:  209.41003  loss_sent:  1.718729  loss_word:  200.81639  Time cost:  6.4644458293914795\n",
      "Start and end (1045, 1050)\n",
      "idx:  1045  Epoch:  114  loss:  264.2767  loss_sent:  3.684538  loss_word:  245.85399  Time cost:  6.477471590042114\n",
      "Start and end (1050, 1055)\n",
      "idx:  1050  Epoch:  114  loss:  289.50354  loss_sent:  2.85367  loss_word:  275.23517  Time cost:  6.551081895828247\n",
      "Start and end (1055, 1060)\n",
      "idx:  1055  Epoch:  114  loss:  189.78682  loss_sent:  2.337171  loss_word:  178.10097  Time cost:  6.513814926147461\n",
      "Start and end (1060, 1065)\n",
      "idx:  1060  Epoch:  114  loss:  315.16537  loss_sent:  2.2561412  loss_word:  303.88467  Time cost:  6.539283752441406\n",
      "Start and end (1065, 1070)\n",
      "idx:  1065  Epoch:  114  loss:  269.54556  loss_sent:  2.6025996  loss_word:  256.53256  Time cost:  6.527017593383789\n",
      "Start and end (1070, 1075)\n",
      "idx:  1070  Epoch:  114  loss:  375.40067  loss_sent:  1.8728065  loss_word:  366.03662  Time cost:  6.44794225692749\n",
      "Start and end (1075, 1080)\n",
      "idx:  1075  Epoch:  114  loss:  270.33273  loss_sent:  2.6131277  loss_word:  257.26715  Time cost:  6.527952432632446\n",
      "Start and end (1080, 1085)\n",
      "idx:  1080  Epoch:  114  loss:  254.12027  loss_sent:  3.2953682  loss_word:  237.6434  Time cost:  6.44197154045105\n",
      "Start and end (1085, 1090)\n",
      "idx:  1085  Epoch:  114  loss:  259.6042  loss_sent:  2.4375474  loss_word:  247.41646  Time cost:  6.579420566558838\n",
      "Start and end (1090, 1095)\n",
      "idx:  1090  Epoch:  114  loss:  303.96817  loss_sent:  2.557794  loss_word:  291.1792  Time cost:  6.493576526641846\n",
      "Start and end (1095, 1100)\n",
      "idx:  1095  Epoch:  114  loss:  314.78583  loss_sent:  2.3147488  loss_word:  303.2121  Time cost:  6.534149885177612\n",
      "Start and end (1100, 1105)\n",
      "idx:  1100  Epoch:  114  loss:  245.63295  loss_sent:  1.8943403  loss_word:  236.16125  Time cost:  6.498010873794556\n",
      "Start and end (1105, 1110)\n",
      "idx:  1105  Epoch:  114  loss:  301.73746  loss_sent:  2.414876  loss_word:  289.66312  Time cost:  6.424275875091553\n",
      "Start and end (1110, 1115)\n",
      "idx:  1110  Epoch:  114  loss:  281.64856  loss_sent:  2.126857  loss_word:  271.01428  Time cost:  6.5427844524383545\n",
      "Start and end (1115, 1120)\n",
      "idx:  1115  Epoch:  114  loss:  224.75049  loss_sent:  2.3522062  loss_word:  212.98946  Time cost:  6.501518964767456\n",
      "Start and end (1120, 1125)\n",
      "idx:  1120  Epoch:  114  loss:  260.02585  loss_sent:  1.9437258  loss_word:  250.30717  Time cost:  6.480303049087524\n",
      "Start and end (1125, 1130)\n",
      "idx:  1125  Epoch:  114  loss:  187.29301  loss_sent:  4.060262  loss_word:  166.99171  Time cost:  6.432905435562134\n",
      "Start and end (1130, 1135)\n",
      "idx:  1130  Epoch:  114  loss:  229.64769  loss_sent:  2.5997443  loss_word:  216.64897  Time cost:  6.42982292175293\n",
      "Start and end (1135, 1140)\n",
      "idx:  1135  Epoch:  114  loss:  326.4833  loss_sent:  3.5318222  loss_word:  308.8242  Time cost:  6.539555311203003\n",
      "Start and end (1140, 1145)\n",
      "idx:  1140  Epoch:  114  loss:  190.87138  loss_sent:  2.8469644  loss_word:  176.63657  Time cost:  6.56493616104126\n",
      "Start and end (1145, 1150)\n",
      "idx:  1145  Epoch:  114  loss:  293.2263  loss_sent:  2.477644  loss_word:  280.8381  Time cost:  6.530463457107544\n",
      "Start and end (1150, 1155)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1150  Epoch:  114  loss:  262.62073  loss_sent:  2.2934241  loss_word:  251.15364  Time cost:  6.529861927032471\n",
      "Start and end (1155, 1160)\n",
      "idx:  1155  Epoch:  114  loss:  337.6861  loss_sent:  2.56524  loss_word:  324.8598  Time cost:  6.5192718505859375\n",
      "Start and end (1160, 1165)\n",
      "idx:  1160  Epoch:  114  loss:  261.55286  loss_sent:  2.3616486  loss_word:  249.74458  Time cost:  6.536913871765137\n",
      "Start and end (1165, 1170)\n",
      "idx:  1165  Epoch:  114  loss:  208.38551  loss_sent:  2.6495814  loss_word:  195.13759  Time cost:  6.503604173660278\n",
      "Start and end (1170, 1175)\n",
      "idx:  1170  Epoch:  114  loss:  299.9939  loss_sent:  2.467617  loss_word:  287.6558  Time cost:  6.453360557556152\n",
      "Start and end (1175, 1180)\n",
      "idx:  1175  Epoch:  114  loss:  191.71312  loss_sent:  2.697415  loss_word:  178.22603  Time cost:  6.482064962387085\n",
      "Start and end (1180, 1185)\n",
      "idx:  1180  Epoch:  114  loss:  242.46872  loss_sent:  2.7831693  loss_word:  228.55287  Time cost:  6.427148103713989\n",
      "Start and end (1185, 1190)\n",
      "idx:  1185  Epoch:  114  loss:  255.92627  loss_sent:  2.3069916  loss_word:  244.39133  Time cost:  6.537073850631714\n",
      "Start and end (1190, 1195)\n",
      "idx:  1190  Epoch:  114  loss:  334.91776  loss_sent:  1.993434  loss_word:  324.95062  Time cost:  6.415403366088867\n",
      "Start and end (1195, 1200)\n",
      "idx:  1195  Epoch:  114  loss:  252.39934  loss_sent:  2.4751592  loss_word:  240.02353  Time cost:  6.427995443344116\n",
      "Start and end (1200, 1205)\n",
      "idx:  1200  Epoch:  114  loss:  274.27032  loss_sent:  2.0477874  loss_word:  264.0314  Time cost:  6.421722412109375\n",
      "Start and end (1205, 1210)\n",
      "idx:  1205  Epoch:  114  loss:  255.36821  loss_sent:  2.963553  loss_word:  240.55045  Time cost:  6.500349283218384\n",
      "Start and end (1210, 1215)\n",
      "idx:  1210  Epoch:  114  loss:  238.64609  loss_sent:  3.0767589  loss_word:  223.2623  Time cost:  6.488382816314697\n",
      "Start and end (1215, 1220)\n",
      "idx:  1215  Epoch:  114  loss:  225.40529  loss_sent:  2.8840349  loss_word:  210.98512  Time cost:  6.402094602584839\n",
      "Start and end (1220, 1225)\n",
      "idx:  1220  Epoch:  114  loss:  191.80373  loss_sent:  2.7266872  loss_word:  178.17027  Time cost:  6.499981880187988\n",
      "Start and end (1225, 1230)\n",
      "idx:  1225  Epoch:  114  loss:  256.80402  loss_sent:  2.2382023  loss_word:  245.61302  Time cost:  6.514466285705566\n",
      "Start and end (1230, 1235)\n",
      "idx:  1230  Epoch:  114  loss:  230.2329  loss_sent:  2.1900249  loss_word:  219.28278  Time cost:  6.402481555938721\n",
      "Start and end (1235, 1240)\n",
      "idx:  1235  Epoch:  114  loss:  301.1353  loss_sent:  2.3372934  loss_word:  289.44888  Time cost:  6.512852191925049\n",
      "Start and end (1240, 1245)\n",
      "idx:  1240  Epoch:  114  loss:  218.05911  loss_sent:  2.841308  loss_word:  203.85258  Time cost:  6.431985378265381\n",
      "Start and end (1245, 1250)\n",
      "idx:  1245  Epoch:  114  loss:  372.30713  loss_sent:  3.0185971  loss_word:  357.21417  Time cost:  6.537392616271973\n",
      "Start and end (1250, 1255)\n",
      "idx:  1250  Epoch:  114  loss:  232.32278  loss_sent:  2.9617338  loss_word:  217.51411  Time cost:  6.49873685836792\n",
      "Start and end (1255, 1260)\n",
      "idx:  1255  Epoch:  114  loss:  195.68996  loss_sent:  2.0723884  loss_word:  185.328  Time cost:  6.459705591201782\n",
      "Start and end (1260, 1265)\n",
      "idx:  1260  Epoch:  114  loss:  294.2561  loss_sent:  1.9128563  loss_word:  284.6918  Time cost:  6.578929424285889\n",
      "Start and end (1265, 1270)\n",
      "idx:  1265  Epoch:  114  loss:  263.4896  loss_sent:  2.5202036  loss_word:  250.88857  Time cost:  6.492272853851318\n",
      "Start and end (1270, 1275)\n",
      "idx:  1270  Epoch:  114  loss:  213.45346  loss_sent:  2.7915454  loss_word:  199.49576  Time cost:  6.57145619392395\n",
      "Start and end (1275, 1280)\n",
      "idx:  1275  Epoch:  114  loss:  267.05444  loss_sent:  2.1503003  loss_word:  256.30292  Time cost:  6.517667531967163\n",
      "Start and end (1280, 1285)\n",
      "idx:  1280  Epoch:  114  loss:  263.06934  loss_sent:  1.9315519  loss_word:  253.41154  Time cost:  6.42983341217041\n",
      "Start and end (1285, 1290)\n",
      "idx:  1285  Epoch:  114  loss:  246.30612  loss_sent:  2.400295  loss_word:  234.30464  Time cost:  6.5103535652160645\n",
      "Start and end (1290, 1295)\n",
      "idx:  1290  Epoch:  114  loss:  302.2666  loss_sent:  1.9944699  loss_word:  292.29428  Time cost:  6.488537549972534\n",
      "Start and end (1295, 1300)\n",
      "idx:  1295  Epoch:  114  loss:  234.40166  loss_sent:  2.2992861  loss_word:  222.90526  Time cost:  6.5028345584869385\n",
      "Start and end (1300, 1305)\n",
      "idx:  1300  Epoch:  114  loss:  285.77322  loss_sent:  1.8134553  loss_word:  276.70596  Time cost:  6.5263659954071045\n",
      "Start and end (1305, 1310)\n",
      "idx:  1305  Epoch:  114  loss:  268.76535  loss_sent:  2.0335937  loss_word:  258.5974  Time cost:  6.44116735458374\n",
      "Start and end (1310, 1315)\n",
      "idx:  1310  Epoch:  114  loss:  215.02072  loss_sent:  2.1226702  loss_word:  204.40736  Time cost:  6.398535966873169\n",
      "Start and end (1315, 1320)\n",
      "idx:  1315  Epoch:  114  loss:  213.42969  loss_sent:  2.426514  loss_word:  201.29709  Time cost:  6.501202583312988\n",
      "Start and end (1320, 1325)\n",
      "idx:  1320  Epoch:  114  loss:  227.91335  loss_sent:  3.7838988  loss_word:  208.99388  Time cost:  6.647192001342773\n",
      "Start and end (1325, 1330)\n",
      "idx:  1325  Epoch:  114  loss:  226.93588  loss_sent:  1.9231791  loss_word:  217.31998  Time cost:  6.557190179824829\n",
      "Start and end (1330, 1335)\n",
      "idx:  1330  Epoch:  114  loss:  252.89996  loss_sent:  1.9603226  loss_word:  243.09836  Time cost:  6.539475440979004\n",
      "Start and end (1335, 1340)\n",
      "idx:  1335  Epoch:  114  loss:  175.50697  loss_sent:  3.1088257  loss_word:  159.96286  Time cost:  6.494441747665405\n",
      "Start and end (1340, 1345)\n",
      "idx:  1340  Epoch:  114  loss:  284.71515  loss_sent:  1.8560231  loss_word:  275.43494  Time cost:  6.5232462882995605\n",
      "Start and end (1345, 1350)\n",
      "idx:  1345  Epoch:  114  loss:  287.80917  loss_sent:  2.6352766  loss_word:  274.63284  Time cost:  6.5236334800720215\n",
      "Start and end (1350, 1355)\n",
      "idx:  1350  Epoch:  114  loss:  281.1513  loss_sent:  2.481148  loss_word:  268.74557  Time cost:  6.543587923049927\n",
      "Start and end (1355, 1360)\n",
      "idx:  1355  Epoch:  114  loss:  170.2983  loss_sent:  2.3160763  loss_word:  158.71791  Time cost:  6.5412609577178955\n",
      "Start and end (1360, 1365)\n",
      "idx:  1360  Epoch:  114  loss:  228.20886  loss_sent:  2.1816866  loss_word:  217.30045  Time cost:  6.559156894683838\n",
      "Start and end (1365, 1370)\n",
      "idx:  1365  Epoch:  114  loss:  210.3695  loss_sent:  2.1685257  loss_word:  199.5269  Time cost:  6.515672922134399\n",
      "Start and end (1370, 1375)\n",
      "idx:  1370  Epoch:  114  loss:  218.76608  loss_sent:  2.0474966  loss_word:  208.52864  Time cost:  6.4652862548828125\n",
      "Start and end (1375, 1380)\n",
      "idx:  1375  Epoch:  114  loss:  272.81854  loss_sent:  2.3093863  loss_word:  261.2716  Time cost:  6.579784870147705\n",
      "Start and end (1380, 1385)\n",
      "idx:  1380  Epoch:  114  loss:  206.83556  loss_sent:  2.0732055  loss_word:  196.46953  Time cost:  6.513131380081177\n",
      "Start and end (1385, 1390)\n",
      "idx:  1385  Epoch:  114  loss:  209.45242  loss_sent:  2.064542  loss_word:  199.12971  Time cost:  6.536440134048462\n",
      "Start and end (1390, 1395)\n",
      "idx:  1390  Epoch:  114  loss:  247.61076  loss_sent:  2.1916723  loss_word:  236.6524  Time cost:  6.529855012893677\n",
      "Start and end (1395, 1400)\n",
      "idx:  1395  Epoch:  114  loss:  286.58755  loss_sent:  3.5541947  loss_word:  268.8166  Time cost:  6.54233193397522\n",
      "Start and end (1400, 1405)\n",
      "idx:  1400  Epoch:  114  loss:  273.0593  loss_sent:  3.1603615  loss_word:  257.2575  Time cost:  6.5274200439453125\n",
      "Start and end (1405, 1410)\n",
      "idx:  1405  Epoch:  114  loss:  257.78772  loss_sent:  1.8793224  loss_word:  248.39107  Time cost:  6.502202749252319\n",
      "Start and end (1410, 1415)\n",
      "idx:  1410  Epoch:  114  loss:  215.10915  loss_sent:  2.9961176  loss_word:  200.12857  Time cost:  6.511068820953369\n",
      "Start and end (1415, 1420)\n",
      "idx:  1415  Epoch:  114  loss:  341.1148  loss_sent:  1.8841643  loss_word:  331.694  Time cost:  6.5376877784729\n",
      "Start and end (1420, 1425)\n",
      "idx:  1420  Epoch:  114  loss:  270.66913  loss_sent:  2.5306556  loss_word:  258.0158  Time cost:  6.541473865509033\n",
      "Start and end (1425, 1430)\n",
      "idx:  1425  Epoch:  114  loss:  312.83307  loss_sent:  2.353499  loss_word:  301.06555  Time cost:  6.479177474975586\n",
      "Start and end (1430, 1435)\n",
      "idx:  1430  Epoch:  114  loss:  224.00635  loss_sent:  2.2167428  loss_word:  212.92265  Time cost:  6.411308288574219\n",
      "Start and end (1435, 1440)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1435  Epoch:  114  loss:  242.57356  loss_sent:  2.8444788  loss_word:  228.35118  Time cost:  6.546400547027588\n",
      "Start and end (1440, 1445)\n",
      "idx:  1440  Epoch:  114  loss:  219.22083  loss_sent:  2.0856526  loss_word:  208.79254  Time cost:  6.458518743515015\n",
      "Start and end (1445, 1450)\n",
      "idx:  1445  Epoch:  114  loss:  277.6162  loss_sent:  2.5422952  loss_word:  264.90472  Time cost:  6.443972826004028\n",
      "Start and end (1450, 1455)\n",
      "idx:  1450  Epoch:  114  loss:  222.65004  loss_sent:  2.5101602  loss_word:  210.09921  Time cost:  6.51537823677063\n",
      "Start and end (1455, 1460)\n",
      "idx:  1455  Epoch:  114  loss:  219.38957  loss_sent:  2.6351683  loss_word:  206.21375  Time cost:  6.518052816390991\n",
      "Start and end (1460, 1465)\n",
      "idx:  1460  Epoch:  114  loss:  247.43489  loss_sent:  2.2211633  loss_word:  236.32907  Time cost:  6.404594421386719\n",
      "Start and end (1465, 1470)\n",
      "idx:  1465  Epoch:  114  loss:  226.21078  loss_sent:  2.1860561  loss_word:  215.2805  Time cost:  6.534444808959961\n",
      "Start and end (1470, 1475)\n",
      "idx:  1470  Epoch:  114  loss:  280.51138  loss_sent:  2.4053693  loss_word:  268.4846  Time cost:  6.543834209442139\n",
      "Start and end (1475, 1480)\n",
      "idx:  1475  Epoch:  114  loss:  172.23508  loss_sent:  3.207652  loss_word:  156.19681  Time cost:  6.4979212284088135\n",
      "Start and end (1480, 1485)\n",
      "idx:  1480  Epoch:  114  loss:  236.08203  loss_sent:  2.296884  loss_word:  224.59763  Time cost:  6.4089813232421875\n",
      "Start and end (1485, 1490)\n",
      "idx:  1485  Epoch:  114  loss:  225.89337  loss_sent:  1.956625  loss_word:  216.11024  Time cost:  6.5239174365997314\n",
      "Start and end (1490, 1495)\n",
      "idx:  1490  Epoch:  114  loss:  231.42561  loss_sent:  2.088118  loss_word:  220.98502  Time cost:  6.491112947463989\n",
      "Start and end (1495, 1500)\n",
      "idx:  1495  Epoch:  114  loss:  271.45224  loss_sent:  2.108812  loss_word:  260.90823  Time cost:  6.453875303268433\n",
      "Start and end (1500, 1505)\n",
      "idx:  1500  Epoch:  114  loss:  244.08673  loss_sent:  2.119544  loss_word:  233.48901  Time cost:  6.503887176513672\n",
      "Start and end (1505, 1510)\n",
      "idx:  1505  Epoch:  114  loss:  265.05078  loss_sent:  2.9161384  loss_word:  250.4701  Time cost:  6.468520164489746\n",
      "Start and end (1510, 1515)\n",
      "idx:  1510  Epoch:  114  loss:  383.323  loss_sent:  2.212535  loss_word:  372.26038  Time cost:  6.495153903961182\n",
      "Start and end (1515, 1520)\n",
      "idx:  1515  Epoch:  114  loss:  217.83305  loss_sent:  2.3818965  loss_word:  205.92354  Time cost:  6.551462888717651\n",
      "Start and end (1520, 1525)\n",
      "idx:  1520  Epoch:  114  loss:  223.55394  loss_sent:  2.5212297  loss_word:  210.94778  Time cost:  6.517914533615112\n",
      "Start and end (1525, 1530)\n",
      "idx:  1525  Epoch:  114  loss:  324.99194  loss_sent:  2.633585  loss_word:  311.82404  Time cost:  6.464338064193726\n",
      "Start and end (1530, 1535)\n",
      "idx:  1530  Epoch:  114  loss:  265.8234  loss_sent:  2.6749003  loss_word:  252.4489  Time cost:  6.470462322235107\n",
      "Start and end (1535, 1540)\n",
      "idx:  1535  Epoch:  114  loss:  266.62592  loss_sent:  2.433125  loss_word:  254.46031  Time cost:  6.516423225402832\n",
      "Start and end (1540, 1545)\n",
      "idx:  1540  Epoch:  114  loss:  223.54492  loss_sent:  2.006987  loss_word:  213.50998  Time cost:  6.506136178970337\n",
      "Start and end (1545, 1550)\n",
      "idx:  1545  Epoch:  114  loss:  227.63148  loss_sent:  2.2225616  loss_word:  216.51866  Time cost:  6.5065484046936035\n",
      "Start and end (1550, 1555)\n",
      "idx:  1550  Epoch:  114  loss:  180.97906  loss_sent:  2.46247  loss_word:  168.6667  Time cost:  6.530717849731445\n",
      "Start and end (1555, 1560)\n",
      "idx:  1555  Epoch:  114  loss:  203.6153  loss_sent:  1.9021664  loss_word:  194.10446  Time cost:  6.495842218399048\n",
      "Start and end (1560, 1565)\n",
      "idx:  1560  Epoch:  114  loss:  203.03595  loss_sent:  1.8280907  loss_word:  193.8955  Time cost:  6.532120704650879\n",
      "Start and end (1565, 1570)\n",
      "idx:  1565  Epoch:  114  loss:  207.10513  loss_sent:  2.191597  loss_word:  196.14713  Time cost:  6.588673114776611\n",
      "Start and end (1570, 1575)\n",
      "idx:  1570  Epoch:  114  loss:  244.7468  loss_sent:  2.7379093  loss_word:  231.05724  Time cost:  6.516342639923096\n",
      "Start and end (1575, 1580)\n",
      "idx:  1575  Epoch:  114  loss:  278.2434  loss_sent:  4.6990566  loss_word:  254.74817  Time cost:  6.444089651107788\n",
      "Start and end (1580, 1585)\n",
      "idx:  1580  Epoch:  114  loss:  273.54956  loss_sent:  2.6201808  loss_word:  260.4486  Time cost:  6.5512473583221436\n",
      "Start and end (1585, 1590)\n",
      "idx:  1585  Epoch:  114  loss:  228.72696  loss_sent:  2.883446  loss_word:  214.30972  Time cost:  6.420034885406494\n",
      "Start and end (1590, 1595)\n",
      "idx:  1590  Epoch:  114  loss:  327.02374  loss_sent:  1.9294102  loss_word:  317.3767  Time cost:  6.540563583374023\n",
      "Start and end (1595, 1600)\n",
      "idx:  1595  Epoch:  114  loss:  239.75658  loss_sent:  2.5281303  loss_word:  227.11592  Time cost:  6.5389580726623535\n",
      "Start and end (1600, 1605)\n",
      "idx:  1600  Epoch:  114  loss:  291.38638  loss_sent:  2.3146243  loss_word:  279.81326  Time cost:  6.499474763870239\n",
      "Start and end (1605, 1610)\n",
      "idx:  1605  Epoch:  114  loss:  220.89278  loss_sent:  1.9528329  loss_word:  211.12862  Time cost:  6.474968194961548\n",
      "Start and end (1610, 1615)\n",
      "idx:  1610  Epoch:  114  loss:  237.59088  loss_sent:  4.2940784  loss_word:  216.1205  Time cost:  6.527097702026367\n",
      "Start and end (1615, 1620)\n",
      "idx:  1615  Epoch:  114  loss:  259.32706  loss_sent:  1.9408267  loss_word:  249.6229  Time cost:  6.490746736526489\n",
      "Start and end (1620, 1625)\n",
      "idx:  1620  Epoch:  114  loss:  274.69714  loss_sent:  2.252859  loss_word:  263.4329  Time cost:  6.512164115905762\n",
      "Start and end (1625, 1630)\n",
      "idx:  1625  Epoch:  114  loss:  228.26492  loss_sent:  2.3241498  loss_word:  216.64418  Time cost:  6.460419416427612\n",
      "Start and end (1630, 1635)\n",
      "idx:  1630  Epoch:  114  loss:  170.9999  loss_sent:  2.6578531  loss_word:  157.71063  Time cost:  6.377164125442505\n",
      "Start and end (1635, 1640)\n",
      "idx:  1635  Epoch:  114  loss:  255.4963  loss_sent:  2.2753465  loss_word:  244.11958  Time cost:  6.429948806762695\n",
      "Start and end (1640, 1645)\n",
      "idx:  1640  Epoch:  114  loss:  241.0157  loss_sent:  2.290228  loss_word:  229.56459  Time cost:  6.520652770996094\n",
      "Start and end (1645, 1650)\n",
      "idx:  1645  Epoch:  114  loss:  224.97688  loss_sent:  2.4761312  loss_word:  212.59622  Time cost:  6.395881652832031\n",
      "Start and end (1650, 1655)\n",
      "idx:  1650  Epoch:  114  loss:  309.87646  loss_sent:  2.6643467  loss_word:  296.5547  Time cost:  6.514614820480347\n",
      "Start and end (1655, 1660)\n",
      "idx:  1655  Epoch:  114  loss:  218.64444  loss_sent:  2.3068628  loss_word:  207.11012  Time cost:  6.524784326553345\n",
      "Start and end (1660, 1665)\n",
      "idx:  1660  Epoch:  114  loss:  263.00363  loss_sent:  2.8996727  loss_word:  248.50531  Time cost:  6.4704790115356445\n",
      "Start and end (1665, 1670)\n",
      "idx:  1665  Epoch:  114  loss:  301.37543  loss_sent:  3.4125698  loss_word:  284.31253  Time cost:  6.5465288162231445\n",
      "Start and end (1670, 1675)\n",
      "idx:  1670  Epoch:  114  loss:  175.84335  loss_sent:  3.6170948  loss_word:  157.75786  Time cost:  6.438262462615967\n",
      "Start and end (1675, 1680)\n",
      "idx:  1675  Epoch:  114  loss:  322.23117  loss_sent:  2.2189333  loss_word:  311.1365  Time cost:  6.545586109161377\n",
      "Start and end (1680, 1685)\n",
      "idx:  1680  Epoch:  114  loss:  211.78119  loss_sent:  2.9678674  loss_word:  196.94186  Time cost:  6.553726673126221\n",
      "Start and end (1685, 1690)\n",
      "idx:  1685  Epoch:  114  loss:  233.52336  loss_sent:  2.5594149  loss_word:  220.7263  Time cost:  6.437238931655884\n",
      "Start and end (1690, 1695)\n",
      "idx:  1690  Epoch:  114  loss:  236.4133  loss_sent:  2.1132612  loss_word:  225.84697  Time cost:  6.46132755279541\n",
      "Start and end (1695, 1700)\n",
      "idx:  1695  Epoch:  114  loss:  297.88873  loss_sent:  2.1883688  loss_word:  286.94684  Time cost:  6.501070976257324\n",
      "Start and end (1700, 1705)\n",
      "idx:  1700  Epoch:  114  loss:  241.25967  loss_sent:  2.320376  loss_word:  229.65779  Time cost:  6.59122896194458\n",
      "Start and end (1705, 1710)\n",
      "idx:  1705  Epoch:  114  loss:  249.43185  loss_sent:  2.413326  loss_word:  237.36523  Time cost:  6.514620780944824\n",
      "Start and end (1710, 1715)\n",
      "idx:  1710  Epoch:  114  loss:  234.05751  loss_sent:  2.562938  loss_word:  221.24284  Time cost:  6.443201541900635\n",
      "Start and end (1715, 1720)\n",
      "idx:  1715  Epoch:  114  loss:  196.13033  loss_sent:  2.8007865  loss_word:  182.12639  Time cost:  6.51922607421875\n",
      "Start and end (1720, 1725)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1720  Epoch:  114  loss:  184.20197  loss_sent:  2.5053036  loss_word:  171.67546  Time cost:  6.426969051361084\n",
      "Start and end (1725, 1730)\n",
      "idx:  1725  Epoch:  114  loss:  217.86493  loss_sent:  2.80241  loss_word:  203.85287  Time cost:  6.5381011962890625\n",
      "Start and end (1730, 1735)\n",
      "idx:  1730  Epoch:  114  loss:  279.04123  loss_sent:  2.4673183  loss_word:  266.70465  Time cost:  6.487589359283447\n",
      "Start and end (1735, 1740)\n",
      "idx:  1735  Epoch:  114  loss:  223.6926  loss_sent:  2.2132711  loss_word:  212.62624  Time cost:  6.472128868103027\n",
      "Start and end (1740, 1745)\n",
      "idx:  1740  Epoch:  114  loss:  264.97784  loss_sent:  2.2680159  loss_word:  253.63777  Time cost:  6.5431437492370605\n",
      "Start and end (1745, 1750)\n",
      "idx:  1745  Epoch:  114  loss:  304.29984  loss_sent:  3.1744597  loss_word:  288.4275  Time cost:  6.464349269866943\n",
      "Start and end (1750, 1755)\n",
      "idx:  1750  Epoch:  114  loss:  207.41975  loss_sent:  2.9163618  loss_word:  192.83795  Time cost:  6.548246622085571\n",
      "Start and end (1755, 1760)\n",
      "idx:  1755  Epoch:  114  loss:  264.7575  loss_sent:  2.2536657  loss_word:  253.48915  Time cost:  6.529189109802246\n",
      "Start and end (1760, 1765)\n",
      "idx:  1760  Epoch:  114  loss:  305.37115  loss_sent:  2.2062209  loss_word:  294.3401  Time cost:  6.593087911605835\n",
      "Start and end (1765, 1770)\n",
      "idx:  1765  Epoch:  114  loss:  342.3779  loss_sent:  2.0394473  loss_word:  332.18063  Time cost:  6.535627603530884\n",
      "Start and end (1770, 1775)\n",
      "idx:  1770  Epoch:  114  loss:  261.17392  loss_sent:  2.1270163  loss_word:  250.53885  Time cost:  6.397099733352661\n",
      "Start and end (1775, 1780)\n",
      "idx:  1775  Epoch:  114  loss:  288.21466  loss_sent:  2.8590522  loss_word:  273.91937  Time cost:  6.473407983779907\n",
      "Start and end (1780, 1785)\n",
      "idx:  1780  Epoch:  114  loss:  275.86868  loss_sent:  1.7877  loss_word:  266.93018  Time cost:  6.577259540557861\n",
      "Start and end (1785, 1790)\n",
      "idx:  1785  Epoch:  114  loss:  288.3713  loss_sent:  2.1894608  loss_word:  277.42404  Time cost:  6.501544237136841\n",
      "Start and end (1790, 1795)\n",
      "idx:  1790  Epoch:  114  loss:  213.93965  loss_sent:  2.423248  loss_word:  201.82343  Time cost:  6.400853872299194\n",
      "Start and end (1795, 1800)\n",
      "idx:  1795  Epoch:  114  loss:  231.6711  loss_sent:  2.0521543  loss_word:  221.41031  Time cost:  7.099230051040649\n",
      "Start and end (1800, 1805)\n",
      "idx:  1800  Epoch:  114  loss:  270.97232  loss_sent:  3.232837  loss_word:  254.80807  Time cost:  6.7062859535217285\n",
      "Start and end (1805, 1810)\n",
      "idx:  1805  Epoch:  114  loss:  246.14732  loss_sent:  2.2417204  loss_word:  234.93872  Time cost:  6.800460577011108\n",
      "Start and end (1810, 1815)\n",
      "idx:  1810  Epoch:  114  loss:  248.423  loss_sent:  2.3673382  loss_word:  236.58632  Time cost:  6.793280601501465\n",
      "Start and end (1815, 1820)\n",
      "idx:  1815  Epoch:  114  loss:  217.36494  loss_sent:  3.4815652  loss_word:  199.95712  Time cost:  6.72675633430481\n",
      "Start and end (1820, 1825)\n",
      "idx:  1820  Epoch:  114  loss:  300.58435  loss_sent:  2.8485374  loss_word:  286.34164  Time cost:  6.913707733154297\n",
      "Start and end (1825, 1830)\n",
      "idx:  1825  Epoch:  114  loss:  250.24649  loss_sent:  2.193026  loss_word:  239.28136  Time cost:  8.849369764328003\n",
      "Start and end (1830, 1835)\n",
      "idx:  1830  Epoch:  114  loss:  269.27307  loss_sent:  2.831986  loss_word:  255.11316  Time cost:  7.602038383483887\n",
      "Start and end (1835, 1840)\n",
      "idx:  1835  Epoch:  114  loss:  258.01764  loss_sent:  2.4947019  loss_word:  245.54416  Time cost:  6.854883670806885\n",
      "Start and end (1840, 1845)\n",
      "idx:  1840  Epoch:  114  loss:  279.737  loss_sent:  2.9109268  loss_word:  265.18237  Time cost:  7.525717496871948\n",
      "Start and end (1845, 1850)\n",
      "idx:  1845  Epoch:  114  loss:  203.7838  loss_sent:  2.360549  loss_word:  191.98103  Time cost:  6.699460029602051\n",
      "Start and end (1850, 1855)\n",
      "idx:  1850  Epoch:  114  loss:  346.2645  loss_sent:  2.8234282  loss_word:  332.14734  Time cost:  6.629051685333252\n",
      "Start and end (1855, 1860)\n",
      "idx:  1855  Epoch:  114  loss:  236.86705  loss_sent:  2.1852033  loss_word:  225.94104  Time cost:  6.650419473648071\n",
      "Start and end (1860, 1865)\n",
      "idx:  1860  Epoch:  114  loss:  361.35107  loss_sent:  2.3990111  loss_word:  349.35602  Time cost:  6.660524129867554\n",
      "Start and end (1865, 1870)\n",
      "idx:  1865  Epoch:  114  loss:  226.17557  loss_sent:  1.9452648  loss_word:  216.44925  Time cost:  6.6624650955200195\n",
      "Start and end (1870, 1875)\n",
      "idx:  1870  Epoch:  114  loss:  195.41074  loss_sent:  2.220998  loss_word:  184.30573  Time cost:  7.521485328674316\n",
      "Start and end (1875, 1880)\n",
      "idx:  1875  Epoch:  114  loss:  371.95935  loss_sent:  2.1316643  loss_word:  361.30103  Time cost:  7.916043758392334\n",
      "Start and end (1880, 1885)\n",
      "idx:  1880  Epoch:  114  loss:  395.22992  loss_sent:  1.9219184  loss_word:  385.62033  Time cost:  7.6436121463775635\n",
      "Start and end (1885, 1890)\n",
      "idx:  1885  Epoch:  114  loss:  199.76624  loss_sent:  4.139951  loss_word:  179.06648  Time cost:  7.720864534378052\n",
      "Start and end (1890, 1895)\n",
      "idx:  1890  Epoch:  114  loss:  317.1444  loss_sent:  1.7619419  loss_word:  308.33472  Time cost:  7.978534698486328\n",
      "Start and end (1895, 1900)\n",
      "idx:  1895  Epoch:  114  loss:  283.48886  loss_sent:  2.4306188  loss_word:  271.33575  Time cost:  7.914976119995117\n",
      "Start and end (1900, 1905)\n",
      "idx:  1900  Epoch:  114  loss:  239.56563  loss_sent:  1.7066762  loss_word:  231.03226  Time cost:  7.750469923019409\n",
      "Start and end (1905, 1910)\n",
      "idx:  1905  Epoch:  114  loss:  254.2508  loss_sent:  2.767025  loss_word:  240.41566  Time cost:  7.798016786575317\n",
      "Start and end (1910, 1915)\n",
      "idx:  1910  Epoch:  114  loss:  234.31418  loss_sent:  2.390289  loss_word:  222.36273  Time cost:  8.025464057922363\n",
      "Start and end (1915, 1920)\n",
      "idx:  1915  Epoch:  114  loss:  265.7866  loss_sent:  2.5173826  loss_word:  253.19965  Time cost:  8.211458444595337\n",
      "Start and end (1920, 1925)\n",
      "idx:  1920  Epoch:  114  loss:  209.2639  loss_sent:  2.0164185  loss_word:  199.18182  Time cost:  8.959532022476196\n",
      "Start and end (1925, 1930)\n",
      "idx:  1925  Epoch:  114  loss:  224.04166  loss_sent:  2.5784893  loss_word:  211.14922  Time cost:  8.048123836517334\n",
      "Start and end (1930, 1935)\n",
      "idx:  1930  Epoch:  114  loss:  235.86343  loss_sent:  2.5700064  loss_word:  223.01343  Time cost:  7.945621490478516\n",
      "Start and end (1935, 1940)\n",
      "idx:  1935  Epoch:  114  loss:  344.4239  loss_sent:  2.2225375  loss_word:  333.3112  Time cost:  6.837879419326782\n",
      "Start and end (1940, 1945)\n",
      "idx:  1940  Epoch:  114  loss:  241.32056  loss_sent:  2.6726782  loss_word:  227.95718  Time cost:  6.646340847015381\n",
      "Start and end (1945, 1950)\n",
      "idx:  1945  Epoch:  114  loss:  239.45622  loss_sent:  2.6863256  loss_word:  226.02463  Time cost:  6.5672547817230225\n",
      "Start and end (1950, 1955)\n",
      "idx:  1950  Epoch:  114  loss:  213.59294  loss_sent:  2.408445  loss_word:  201.55074  Time cost:  6.676645278930664\n",
      "Start and end (1955, 1960)\n",
      "idx:  1955  Epoch:  114  loss:  277.5749  loss_sent:  2.6066375  loss_word:  264.54172  Time cost:  6.855365514755249\n",
      "Start and end (1960, 1965)\n",
      "idx:  1960  Epoch:  114  loss:  214.25076  loss_sent:  2.767746  loss_word:  200.41202  Time cost:  6.657377243041992\n",
      "Start and end (1965, 1970)\n",
      "idx:  1965  Epoch:  114  loss:  194.41417  loss_sent:  2.532802  loss_word:  181.75017  Time cost:  6.7152204513549805\n",
      "Start and end (1970, 1975)\n",
      "idx:  1970  Epoch:  114  loss:  247.059  loss_sent:  2.2362938  loss_word:  235.87755  Time cost:  6.7710981369018555\n",
      "Start and end (1975, 1980)\n",
      "idx:  1975  Epoch:  114  loss:  258.5798  loss_sent:  2.533581  loss_word:  245.91193  Time cost:  6.887091398239136\n",
      "Start and end (1980, 1985)\n",
      "idx:  1980  Epoch:  114  loss:  281.3816  loss_sent:  2.2784722  loss_word:  269.98926  Time cost:  6.762570381164551\n",
      "Start and end (1985, 1990)\n",
      "idx:  1985  Epoch:  114  loss:  273.94897  loss_sent:  3.0022202  loss_word:  258.93787  Time cost:  6.739163875579834\n",
      "Start and end (1990, 1995)\n",
      "idx:  1990  Epoch:  114  loss:  267.96124  loss_sent:  2.265573  loss_word:  256.6334  Time cost:  6.715528964996338\n",
      "Start and end (1995, 2000)\n",
      "idx:  1995  Epoch:  114  loss:  244.91664  loss_sent:  2.1186612  loss_word:  234.32333  Time cost:  6.796521902084351\n",
      "Start and end (2000, 2005)\n",
      "idx:  2000  Epoch:  114  loss:  301.47763  loss_sent:  2.7836165  loss_word:  287.55954  Time cost:  6.7555625438690186\n",
      "Start and end (2005, 2010)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2005  Epoch:  114  loss:  165.74709  loss_sent:  1.768556  loss_word:  156.9043  Time cost:  6.668513774871826\n",
      "Start and end (2010, 2015)\n",
      "idx:  2010  Epoch:  114  loss:  186.68106  loss_sent:  2.3915  loss_word:  174.72354  Time cost:  6.562763929367065\n",
      "Start and end (2015, 2020)\n",
      "idx:  2015  Epoch:  114  loss:  245.80042  loss_sent:  1.9716413  loss_word:  235.9422  Time cost:  6.631821393966675\n",
      "Start and end (2020, 2025)\n",
      "idx:  2020  Epoch:  114  loss:  206.98773  loss_sent:  2.2360296  loss_word:  195.80759  Time cost:  6.657048463821411\n",
      "Start and end (2025, 2030)\n",
      "idx:  2025  Epoch:  114  loss:  365.096  loss_sent:  2.5588017  loss_word:  352.30203  Time cost:  6.740338325500488\n",
      "Start and end (2030, 2035)\n",
      "idx:  2030  Epoch:  114  loss:  255.1575  loss_sent:  2.1449986  loss_word:  244.43254  Time cost:  7.778806686401367\n",
      "Start and end (2035, 2040)\n",
      "idx:  2035  Epoch:  114  loss:  216.9164  loss_sent:  2.3050852  loss_word:  205.39095  Time cost:  6.970146656036377\n",
      "Start and end (2040, 2045)\n",
      "idx:  2040  Epoch:  114  loss:  287.14597  loss_sent:  2.1523113  loss_word:  276.38443  Time cost:  6.748619556427002\n",
      "Start and end (2045, 2050)\n",
      "idx:  2045  Epoch:  114  loss:  203.59236  loss_sent:  2.4598045  loss_word:  191.29333  Time cost:  6.608890771865845\n",
      "Start and end (2050, 2055)\n",
      "idx:  2050  Epoch:  114  loss:  293.6703  loss_sent:  3.116859  loss_word:  278.08597  Time cost:  6.503690719604492\n",
      "Start and end (2055, 2060)\n",
      "idx:  2055  Epoch:  114  loss:  319.2033  loss_sent:  1.8078945  loss_word:  310.16388  Time cost:  6.631685733795166\n",
      "Start and end (2060, 2065)\n",
      "idx:  2060  Epoch:  114  loss:  267.90247  loss_sent:  2.1193929  loss_word:  257.30557  Time cost:  6.5814573764801025\n",
      "Start and end (2065, 2070)\n",
      "idx:  2065  Epoch:  114  loss:  193.76888  loss_sent:  2.4513876  loss_word:  181.51196  Time cost:  6.668928384780884\n",
      "Start and end (2070, 2075)\n",
      "idx:  2070  Epoch:  114  loss:  293.8172  loss_sent:  2.452345  loss_word:  281.55548  Time cost:  7.018213748931885\n",
      "Start and end (2075, 2080)\n",
      "idx:  2075  Epoch:  114  loss:  213.68095  loss_sent:  2.093957  loss_word:  203.21115  Time cost:  6.762155771255493\n",
      "Start and end (2080, 2085)\n",
      "idx:  2080  Epoch:  114  loss:  227.44446  loss_sent:  2.1334963  loss_word:  216.77695  Time cost:  6.619217395782471\n",
      "Start and end (2085, 2090)\n",
      "idx:  2085  Epoch:  114  loss:  268.80884  loss_sent:  2.0648277  loss_word:  258.4847  Time cost:  6.574199438095093\n",
      "Start and end (2090, 2095)\n",
      "idx:  2090  Epoch:  114  loss:  179.68242  loss_sent:  3.5730746  loss_word:  161.81705  Time cost:  6.751920461654663\n",
      "Start and end (2095, 2100)\n",
      "idx:  2095  Epoch:  114  loss:  126.937546  loss_sent:  2.9068608  loss_word:  112.403244  Time cost:  6.561429023742676\n",
      "Start and end (2100, 2105)\n",
      "idx:  2100  Epoch:  114  loss:  343.48087  loss_sent:  2.210607  loss_word:  332.4278  Time cost:  6.538996458053589\n",
      "Start and end (2105, 2110)\n",
      "idx:  2105  Epoch:  114  loss:  187.40224  loss_sent:  2.215219  loss_word:  176.32613  Time cost:  6.468716382980347\n",
      "Start and end (2110, 2115)\n",
      "idx:  2110  Epoch:  114  loss:  227.41573  loss_sent:  2.0863163  loss_word:  216.98418  Time cost:  6.495907783508301\n",
      "Start and end (2115, 2120)\n",
      "idx:  2115  Epoch:  114  loss:  255.38788  loss_sent:  2.0799375  loss_word:  244.9882  Time cost:  6.562605619430542\n",
      "Start and end (2120, 2125)\n",
      "idx:  2120  Epoch:  114  loss:  201.17218  loss_sent:  2.6263018  loss_word:  188.04066  Time cost:  6.558409214019775\n",
      "Start and end (2125, 2130)\n",
      "idx:  2125  Epoch:  114  loss:  242.20851  loss_sent:  2.3666146  loss_word:  230.37543  Time cost:  6.800373554229736\n",
      "Start and end (2130, 2135)\n",
      "idx:  2130  Epoch:  114  loss:  375.71323  loss_sent:  2.6116874  loss_word:  362.65475  Time cost:  6.969867944717407\n",
      "Start and end (2135, 2140)\n",
      "idx:  2135  Epoch:  114  loss:  269.94858  loss_sent:  2.3848827  loss_word:  258.02417  Time cost:  6.600759267807007\n",
      "Start and end (2140, 2145)\n",
      "idx:  2140  Epoch:  114  loss:  263.4119  loss_sent:  2.197344  loss_word:  252.42516  Time cost:  6.644005060195923\n",
      "Start and end (2145, 2150)\n",
      "idx:  2145  Epoch:  114  loss:  254.50014  loss_sent:  3.1373162  loss_word:  238.81357  Time cost:  6.590278148651123\n",
      "Start and end (2150, 2155)\n",
      "idx:  2150  Epoch:  114  loss:  190.78017  loss_sent:  3.603264  loss_word:  172.76384  Time cost:  6.5701072216033936\n",
      "Start and end (2155, 2160)\n",
      "idx:  2155  Epoch:  114  loss:  223.31946  loss_sent:  2.5693755  loss_word:  210.47258  Time cost:  6.508282423019409\n",
      "Start and end (2160, 2165)\n",
      "idx:  2160  Epoch:  114  loss:  224.67531  loss_sent:  2.6159039  loss_word:  211.5958  Time cost:  6.440542221069336\n",
      "Start and end (2165, 2170)\n",
      "idx:  2165  Epoch:  114  loss:  193.54341  loss_sent:  1.994091  loss_word:  183.57295  Time cost:  6.600992202758789\n",
      "Start and end (2170, 2175)\n",
      "idx:  2170  Epoch:  114  loss:  254.99548  loss_sent:  2.324144  loss_word:  243.37476  Time cost:  6.464043378829956\n",
      "Start and end (2175, 2180)\n",
      "idx:  2175  Epoch:  114  loss:  315.7491  loss_sent:  1.8926985  loss_word:  306.28564  Time cost:  6.701058626174927\n",
      "Start and end (2180, 2185)\n",
      "idx:  2180  Epoch:  114  loss:  271.02527  loss_sent:  2.2647128  loss_word:  259.7017  Time cost:  6.479041576385498\n",
      "Start and end (2185, 2190)\n",
      "idx:  2185  Epoch:  114  loss:  169.07382  loss_sent:  2.051626  loss_word:  158.8157  Time cost:  6.7910590171813965\n",
      "Start and end (2190, 2195)\n",
      "idx:  2190  Epoch:  114  loss:  279.18008  loss_sent:  2.2088442  loss_word:  268.13593  Time cost:  7.0394792556762695\n",
      "Start and end (2195, 2200)\n",
      "idx:  2195  Epoch:  114  loss:  330.95056  loss_sent:  2.4544055  loss_word:  318.67853  Time cost:  6.990635395050049\n",
      "Start and end (2200, 2205)\n",
      "idx:  2200  Epoch:  114  loss:  237.46306  loss_sent:  2.3498542  loss_word:  225.71376  Time cost:  6.5756471157073975\n",
      "Start and end (2205, 2210)\n",
      "idx:  2205  Epoch:  114  loss:  368.11404  loss_sent:  1.9756348  loss_word:  358.23584  Time cost:  6.841504335403442\n",
      "Start and end (2210, 2215)\n",
      "idx:  2210  Epoch:  114  loss:  276.34177  loss_sent:  2.5898783  loss_word:  263.39236  Time cost:  7.0625810623168945\n",
      "Start and end (2215, 2220)\n",
      "idx:  2215  Epoch:  114  loss:  225.85274  loss_sent:  1.9620409  loss_word:  216.04254  Time cost:  7.4854395389556885\n",
      "Start and end (2220, 2225)\n",
      "idx:  2220  Epoch:  114  loss:  282.74527  loss_sent:  2.0318046  loss_word:  272.58627  Time cost:  6.5904364585876465\n",
      "Start and end (2225, 2230)\n",
      "idx:  2225  Epoch:  114  loss:  297.94397  loss_sent:  1.9880841  loss_word:  288.00357  Time cost:  7.46421480178833\n",
      "Start and end (2230, 2235)\n",
      "idx:  2230  Epoch:  114  loss:  236.49379  loss_sent:  1.8613994  loss_word:  227.18681  Time cost:  7.327829122543335\n",
      "Start and end (2235, 2240)\n",
      "idx:  2235  Epoch:  114  loss:  313.6197  loss_sent:  4.391602  loss_word:  291.66165  Time cost:  7.475943088531494\n",
      "Start and end (2240, 2245)\n",
      "idx:  2240  Epoch:  114  loss:  267.36258  loss_sent:  2.421102  loss_word:  255.257  Time cost:  7.2403318881988525\n",
      "Start and end (2245, 2250)\n",
      "idx:  2245  Epoch:  114  loss:  183.16916  loss_sent:  2.3525226  loss_word:  171.40657  Time cost:  6.610537528991699\n",
      "Start and end (2250, 2255)\n",
      "idx:  2250  Epoch:  114  loss:  250.01265  loss_sent:  2.3299868  loss_word:  238.36272  Time cost:  6.542848348617554\n",
      "Start and end (2255, 2260)\n",
      "idx:  2255  Epoch:  114  loss:  243.88344  loss_sent:  2.6263757  loss_word:  230.75156  Time cost:  6.6765360832214355\n",
      "Start and end (2260, 2265)\n",
      "idx:  2260  Epoch:  114  loss:  261.0819  loss_sent:  2.149736  loss_word:  250.33322  Time cost:  6.55778169631958\n",
      "Start and end (2265, 2270)\n",
      "idx:  2265  Epoch:  114  loss:  250.40346  loss_sent:  2.5651164  loss_word:  237.57787  Time cost:  6.525646448135376\n",
      "Start and end (2270, 2275)\n",
      "idx:  2270  Epoch:  114  loss:  387.85788  loss_sent:  2.105396  loss_word:  377.3309  Time cost:  6.515867233276367\n",
      "Start and end (2275, 2280)\n",
      "idx:  2275  Epoch:  114  loss:  228.77623  loss_sent:  1.8328102  loss_word:  219.61217  Time cost:  6.6002357006073\n",
      "Start and end (2280, 2285)\n",
      "idx:  2280  Epoch:  114  loss:  314.30222  loss_sent:  2.7268448  loss_word:  300.66803  Time cost:  6.644564628601074\n",
      "Start and end (2285, 2290)\n",
      "idx:  2285  Epoch:  114  loss:  286.24  loss_sent:  2.2281291  loss_word:  275.0993  Time cost:  6.611356496810913\n",
      "Start and end (2290, 2295)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2290  Epoch:  114  loss:  230.2557  loss_sent:  2.71044  loss_word:  216.7035  Time cost:  6.685961484909058\n",
      "Start and end (2295, 2300)\n",
      "idx:  2295  Epoch:  114  loss:  234.54071  loss_sent:  2.6185203  loss_word:  221.44814  Time cost:  6.870643854141235\n",
      "Start and end (2300, 2305)\n",
      "idx:  2300  Epoch:  114  loss:  242.9182  loss_sent:  2.0337775  loss_word:  232.74933  Time cost:  6.708220958709717\n",
      "Start and end (2305, 2310)\n",
      "idx:  2305  Epoch:  114  loss:  187.96138  loss_sent:  2.4506953  loss_word:  175.7079  Time cost:  7.123295545578003\n",
      "Start and end (2310, 2315)\n",
      "idx:  2310  Epoch:  114  loss:  265.97354  loss_sent:  2.5586398  loss_word:  253.18044  Time cost:  7.031448602676392\n",
      "Start and end (2315, 2320)\n",
      "idx:  2315  Epoch:  114  loss:  265.67853  loss_sent:  2.0783768  loss_word:  255.28664  Time cost:  7.296682119369507\n",
      "Start and end (2320, 2325)\n",
      "idx:  2320  Epoch:  114  loss:  286.40198  loss_sent:  1.6958834  loss_word:  277.92258  Time cost:  6.808310270309448\n",
      "Start and end (2325, 2330)\n",
      "idx:  2325  Epoch:  114  loss:  204.27216  loss_sent:  1.6758069  loss_word:  195.89313  Time cost:  7.161728858947754\n",
      "Start and end (2330, 2335)\n",
      "idx:  2330  Epoch:  114  loss:  222.75706  loss_sent:  2.635041  loss_word:  209.58186  Time cost:  6.785375118255615\n",
      "Start and end (2335, 2340)\n",
      "idx:  2335  Epoch:  114  loss:  257.84973  loss_sent:  2.4783893  loss_word:  245.4578  Time cost:  6.9081830978393555\n",
      "Start and end (2340, 2345)\n",
      "idx:  2340  Epoch:  114  loss:  299.4976  loss_sent:  2.9909918  loss_word:  284.54272  Time cost:  6.926035404205322\n",
      "Start and end (2345, 2350)\n",
      "idx:  2345  Epoch:  114  loss:  344.01685  loss_sent:  3.156999  loss_word:  328.23187  Time cost:  6.66339635848999\n",
      "Start and end (2350, 2355)\n",
      "idx:  2350  Epoch:  114  loss:  272.7422  loss_sent:  2.340778  loss_word:  261.0383  Time cost:  6.599472761154175\n",
      "Start and end (2355, 2360)\n",
      "idx:  2355  Epoch:  114  loss:  315.5675  loss_sent:  2.300355  loss_word:  304.0658  Time cost:  6.5972981452941895\n",
      "Start and end (2360, 2365)\n",
      "idx:  2360  Epoch:  114  loss:  215.4089  loss_sent:  2.153218  loss_word:  204.64282  Time cost:  6.446123838424683\n",
      "Start and end (2365, 2370)\n",
      "idx:  2365  Epoch:  114  loss:  272.75522  loss_sent:  2.2747324  loss_word:  261.3815  Time cost:  6.560790538787842\n",
      "Start and end (2370, 2375)\n",
      "idx:  2370  Epoch:  114  loss:  196.83795  loss_sent:  3.2471838  loss_word:  180.60204  Time cost:  6.576952695846558\n",
      "Start and end (2375, 2380)\n",
      "idx:  2375  Epoch:  114  loss:  235.9269  loss_sent:  2.6430144  loss_word:  222.71182  Time cost:  6.544439792633057\n",
      "Start and end (2380, 2385)\n",
      "idx:  2380  Epoch:  114  loss:  222.71042  loss_sent:  2.127947  loss_word:  212.07068  Time cost:  7.209444761276245\n",
      "Start and end (2385, 2390)\n",
      "idx:  2385  Epoch:  114  loss:  340.93698  loss_sent:  1.9177537  loss_word:  331.3483  Time cost:  6.697101593017578\n",
      "Start and end (2390, 2395)\n",
      "idx:  2390  Epoch:  114  loss:  317.49396  loss_sent:  2.4602573  loss_word:  305.1927  Time cost:  6.550745248794556\n",
      "Start and end (2395, 2400)\n",
      "idx:  2395  Epoch:  114  loss:  208.84985  loss_sent:  2.378105  loss_word:  196.95929  Time cost:  6.504615068435669\n",
      "Start and end (2400, 2405)\n",
      "idx:  2400  Epoch:  114  loss:  189.8416  loss_sent:  3.3291714  loss_word:  173.19576  Time cost:  6.567817211151123\n",
      "Start and end (2405, 2410)\n",
      "idx:  2405  Epoch:  114  loss:  194.60509  loss_sent:  2.1718514  loss_word:  183.74582  Time cost:  6.596932888031006\n",
      "Start and end (2410, 2415)\n",
      "idx:  2410  Epoch:  114  loss:  234.28366  loss_sent:  2.1060965  loss_word:  223.75316  Time cost:  6.709609746932983\n",
      "Start and end (2415, 2420)\n",
      "idx:  2415  Epoch:  114  loss:  308.15506  loss_sent:  2.1133223  loss_word:  297.58844  Time cost:  11.562278032302856\n",
      "Start and end (2420, 2425)\n",
      "idx:  2420  Epoch:  114  loss:  184.76839  loss_sent:  2.6022725  loss_word:  171.75703  Time cost:  9.716648578643799\n",
      "Start and end (2425, 2430)\n",
      "idx:  2425  Epoch:  114  loss:  257.141  loss_sent:  1.8470404  loss_word:  247.90579  Time cost:  6.752060651779175\n",
      "Start and end (2430, 2435)\n",
      "idx:  2430  Epoch:  114  loss:  277.00778  loss_sent:  2.741438  loss_word:  263.3006  Time cost:  6.755948543548584\n",
      "Start and end (2435, 2440)\n",
      "idx:  2435  Epoch:  114  loss:  212.41751  loss_sent:  1.8399693  loss_word:  203.21765  Time cost:  6.5826416015625\n",
      "Start and end (2440, 2445)\n",
      "idx:  2440  Epoch:  114  loss:  277.53964  loss_sent:  2.7409604  loss_word:  263.83484  Time cost:  6.644681930541992\n",
      "Start and end (2445, 2450)\n",
      "idx:  2445  Epoch:  114  loss:  279.0465  loss_sent:  2.116074  loss_word:  268.46613  Time cost:  6.555948495864868\n",
      "Start and end (2450, 2455)\n",
      "idx:  2450  Epoch:  114  loss:  222.25958  loss_sent:  3.4894829  loss_word:  204.8122  Time cost:  6.751636981964111\n",
      "Start and end (2455, 2460)\n",
      "idx:  2455  Epoch:  114  loss:  219.45659  loss_sent:  1.9763148  loss_word:  209.57504  Time cost:  6.7967705726623535\n",
      "Start and end (2460, 2465)\n",
      "idx:  2460  Epoch:  114  loss:  271.09348  loss_sent:  1.8751408  loss_word:  261.71777  Time cost:  6.5719897747039795\n",
      "Start and end (2465, 2470)\n",
      "idx:  2465  Epoch:  114  loss:  258.21527  loss_sent:  2.22561  loss_word:  247.08723  Time cost:  6.795682907104492\n",
      "Start and end (2470, 2475)\n",
      "idx:  2470  Epoch:  114  loss:  231.89627  loss_sent:  2.5112493  loss_word:  219.34003  Time cost:  6.972305536270142\n",
      "Start and end (2475, 2480)\n",
      "idx:  2475  Epoch:  114  loss:  246.34317  loss_sent:  1.8453813  loss_word:  237.11626  Time cost:  7.268611669540405\n",
      "Start and end (2480, 2485)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0724 11:04:20.977010 140197742454528 saver.py:1134] *******************************************************\n",
      "W0724 11:04:20.977496 140197742454528 saver.py:1135] TensorFlow's V1 checkpoint format has been deprecated.\n",
      "W0724 11:04:20.978115 140197742454528 saver.py:1136] Consider switching to the more efficient V2 format:\n",
      "W0724 11:04:20.978400 140197742454528 saver.py:1137]    `tf.train.Saver(write_version=tf.train.SaverDef.V2)`\n",
      "W0724 11:04:20.978742 140197742454528 saver.py:1138] now on by default.\n",
      "W0724 11:04:20.979074 140197742454528 saver.py:1139] *******************************************************\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2480  Epoch:  114  loss:  193.55753  loss_sent:  3.17797  loss_word:  177.6677  Time cost:  6.714325189590454\n",
      "Successfully Written to temporary\n",
      "Epoch  114  is done. Saving the model ...\n",
      "Start and end (0, 5)\n",
      "idx:  0  Epoch:  115  loss:  264.27545  loss_sent:  1.8729893  loss_word:  254.91055  Time cost:  6.634373664855957\n",
      "Start and end (5, 10)\n",
      "idx:  5  Epoch:  115  loss:  316.0386  loss_sent:  1.6690073  loss_word:  307.69357  Time cost:  6.540422201156616\n",
      "Start and end (10, 15)\n",
      "idx:  10  Epoch:  115  loss:  215.32098  loss_sent:  2.0326111  loss_word:  205.15793  Time cost:  6.662522792816162\n",
      "Start and end (15, 20)\n",
      "idx:  15  Epoch:  115  loss:  302.78003  loss_sent:  1.5698578  loss_word:  294.93073  Time cost:  6.666482210159302\n",
      "Start and end (20, 25)\n",
      "idx:  20  Epoch:  115  loss:  192.62936  loss_sent:  1.8083738  loss_word:  183.5875  Time cost:  6.59878134727478\n",
      "Start and end (25, 30)\n",
      "idx:  25  Epoch:  115  loss:  226.66733  loss_sent:  2.0441759  loss_word:  216.44643  Time cost:  7.099603176116943\n",
      "Start and end (30, 35)\n",
      "idx:  30  Epoch:  115  loss:  340.8013  loss_sent:  1.8613095  loss_word:  331.49475  Time cost:  6.711044073104858\n",
      "Start and end (35, 40)\n",
      "idx:  35  Epoch:  115  loss:  272.45026  loss_sent:  1.7074692  loss_word:  263.9129  Time cost:  6.6133246421813965\n",
      "Start and end (40, 45)\n",
      "idx:  40  Epoch:  115  loss:  153.66806  loss_sent:  1.8516197  loss_word:  144.40994  Time cost:  6.7659711837768555\n",
      "Start and end (45, 50)\n",
      "idx:  45  Epoch:  115  loss:  287.41306  loss_sent:  1.8585745  loss_word:  278.12018  Time cost:  6.6293792724609375\n",
      "Start and end (50, 55)\n",
      "idx:  50  Epoch:  115  loss:  253.27975  loss_sent:  1.8518014  loss_word:  244.02075  Time cost:  6.7738189697265625\n",
      "Start and end (55, 60)\n",
      "idx:  55  Epoch:  115  loss:  414.30722  loss_sent:  2.5126357  loss_word:  401.74408  Time cost:  7.131847858428955\n",
      "Start and end (60, 65)\n",
      "idx:  60  Epoch:  115  loss:  232.78546  loss_sent:  1.5298188  loss_word:  225.13637  Time cost:  6.960323333740234\n",
      "Start and end (65, 70)\n",
      "idx:  65  Epoch:  115  loss:  240.53429  loss_sent:  2.268823  loss_word:  229.19016  Time cost:  6.600281476974487\n",
      "Start and end (70, 75)\n",
      "idx:  70  Epoch:  115  loss:  249.38547  loss_sent:  1.7364833  loss_word:  240.70303  Time cost:  6.706477642059326\n",
      "Start and end (75, 80)\n",
      "idx:  75  Epoch:  115  loss:  234.7161  loss_sent:  2.1176305  loss_word:  224.12793  Time cost:  6.5753045082092285\n",
      "Start and end (80, 85)\n",
      "idx:  80  Epoch:  115  loss:  221.00583  loss_sent:  1.906988  loss_word:  211.4709  Time cost:  6.522662878036499\n",
      "Start and end (85, 90)\n",
      "idx:  85  Epoch:  115  loss:  222.93227  loss_sent:  2.1126087  loss_word:  212.36925  Time cost:  6.587422609329224\n",
      "Start and end (90, 95)\n",
      "idx:  90  Epoch:  115  loss:  201.44466  loss_sent:  1.9825422  loss_word:  191.53194  Time cost:  6.688256025314331\n",
      "Start and end (95, 100)\n",
      "idx:  95  Epoch:  115  loss:  216.2576  loss_sent:  2.0770822  loss_word:  205.8722  Time cost:  6.554678201675415\n",
      "Start and end (100, 105)\n",
      "idx:  100  Epoch:  115  loss:  290.53564  loss_sent:  1.7400683  loss_word:  281.83533  Time cost:  6.759073495864868\n",
      "Start and end (105, 110)\n",
      "idx:  105  Epoch:  115  loss:  199.8137  loss_sent:  1.7013104  loss_word:  191.30714  Time cost:  6.548046112060547\n",
      "Start and end (110, 115)\n",
      "idx:  110  Epoch:  115  loss:  271.757  loss_sent:  2.698662  loss_word:  258.2636  Time cost:  6.547609090805054\n",
      "Start and end (115, 120)\n",
      "idx:  115  Epoch:  115  loss:  208.81314  loss_sent:  1.6946546  loss_word:  200.33987  Time cost:  6.445173978805542\n",
      "Start and end (120, 125)\n",
      "idx:  120  Epoch:  115  loss:  234.50194  loss_sent:  2.2644296  loss_word:  223.17981  Time cost:  6.65129828453064\n",
      "Start and end (125, 130)\n",
      "idx:  125  Epoch:  115  loss:  231.20602  loss_sent:  1.6448362  loss_word:  222.98186  Time cost:  6.527484178543091\n",
      "Start and end (130, 135)\n",
      "idx:  130  Epoch:  115  loss:  332.53592  loss_sent:  3.3262987  loss_word:  315.90442  Time cost:  6.5957722663879395\n",
      "Start and end (135, 140)\n",
      "idx:  135  Epoch:  115  loss:  178.41405  loss_sent:  1.6373093  loss_word:  170.22751  Time cost:  6.5732505321502686\n",
      "Start and end (140, 145)\n",
      "idx:  140  Epoch:  115  loss:  202.7022  loss_sent:  3.1323292  loss_word:  187.0406  Time cost:  6.752899408340454\n",
      "Start and end (145, 150)\n",
      "idx:  145  Epoch:  115  loss:  244.46878  loss_sent:  1.9321612  loss_word:  234.80798  Time cost:  6.5515828132629395\n",
      "Start and end (150, 155)\n",
      "idx:  150  Epoch:  115  loss:  218.28593  loss_sent:  1.9305532  loss_word:  208.63318  Time cost:  6.529663324356079\n",
      "Start and end (155, 160)\n",
      "idx:  155  Epoch:  115  loss:  206.50522  loss_sent:  1.5659512  loss_word:  198.67546  Time cost:  6.499868154525757\n",
      "Start and end (160, 165)\n",
      "idx:  160  Epoch:  115  loss:  207.33188  loss_sent:  1.7720758  loss_word:  198.4715  Time cost:  6.469765901565552\n",
      "Start and end (165, 170)\n",
      "idx:  165  Epoch:  115  loss:  153.00291  loss_sent:  1.6135234  loss_word:  144.93529  Time cost:  6.522761344909668\n",
      "Start and end (170, 175)\n",
      "idx:  170  Epoch:  115  loss:  191.12303  loss_sent:  1.6064138  loss_word:  183.09093  Time cost:  6.551296234130859\n",
      "Start and end (175, 180)\n",
      "idx:  175  Epoch:  115  loss:  257.77008  loss_sent:  1.6452872  loss_word:  249.54366  Time cost:  6.640872001647949\n",
      "Start and end (180, 185)\n",
      "idx:  180  Epoch:  115  loss:  343.63824  loss_sent:  2.2115307  loss_word:  332.5806  Time cost:  6.623399496078491\n",
      "Start and end (185, 190)\n",
      "idx:  185  Epoch:  115  loss:  209.80913  loss_sent:  1.9974909  loss_word:  199.82167  Time cost:  6.580275535583496\n",
      "Start and end (190, 195)\n",
      "idx:  190  Epoch:  115  loss:  247.10077  loss_sent:  2.4783618  loss_word:  234.70897  Time cost:  6.8040454387664795\n",
      "Start and end (195, 200)\n",
      "idx:  195  Epoch:  115  loss:  287.9889  loss_sent:  2.0101209  loss_word:  277.93832  Time cost:  6.73788595199585\n",
      "Start and end (200, 205)\n",
      "idx:  200  Epoch:  115  loss:  235.47382  loss_sent:  1.833344  loss_word:  226.30707  Time cost:  6.630720138549805\n",
      "Start and end (205, 210)\n",
      "idx:  205  Epoch:  115  loss:  214.75017  loss_sent:  1.9186783  loss_word:  205.15677  Time cost:  6.70872688293457\n",
      "Start and end (210, 215)\n",
      "idx:  210  Epoch:  115  loss:  223.26547  loss_sent:  2.3908868  loss_word:  211.31104  Time cost:  6.6382787227630615\n",
      "Start and end (215, 220)\n",
      "idx:  215  Epoch:  115  loss:  326.01273  loss_sent:  2.8245754  loss_word:  311.88983  Time cost:  6.59200119972229\n",
      "Start and end (220, 225)\n",
      "idx:  220  Epoch:  115  loss:  227.59802  loss_sent:  1.8281345  loss_word:  218.45734  Time cost:  6.506920337677002\n",
      "Start and end (225, 230)\n",
      "idx:  225  Epoch:  115  loss:  253.73102  loss_sent:  2.0093865  loss_word:  243.68408  Time cost:  6.512263774871826\n",
      "Start and end (230, 235)\n",
      "idx:  230  Epoch:  115  loss:  238.71031  loss_sent:  1.8086095  loss_word:  229.66728  Time cost:  6.615635871887207\n",
      "Start and end (235, 240)\n",
      "idx:  235  Epoch:  115  loss:  213.92369  loss_sent:  2.4409246  loss_word:  201.71909  Time cost:  6.848788738250732\n",
      "Start and end (240, 245)\n",
      "idx:  240  Epoch:  115  loss:  358.35156  loss_sent:  1.9482753  loss_word:  348.61014  Time cost:  6.5490195751190186\n",
      "Start and end (245, 250)\n",
      "idx:  245  Epoch:  115  loss:  332.46548  loss_sent:  1.9568168  loss_word:  322.6814  Time cost:  6.580406188964844\n",
      "Start and end (250, 255)\n",
      "idx:  250  Epoch:  115  loss:  187.16087  loss_sent:  1.8483891  loss_word:  177.91893  Time cost:  6.521601676940918\n",
      "Start and end (255, 260)\n",
      "idx:  255  Epoch:  115  loss:  217.3033  loss_sent:  1.7444175  loss_word:  208.58122  Time cost:  6.6088480949401855\n",
      "Start and end (260, 265)\n",
      "idx:  260  Epoch:  115  loss:  203.54364  loss_sent:  1.8670526  loss_word:  194.20839  Time cost:  6.425674676895142\n",
      "Start and end (265, 270)\n",
      "idx:  265  Epoch:  115  loss:  305.40097  loss_sent:  2.0885541  loss_word:  294.95816  Time cost:  6.563949823379517\n",
      "Start and end (270, 275)\n",
      "idx:  270  Epoch:  115  loss:  223.28554  loss_sent:  1.5659515  loss_word:  215.45576  Time cost:  6.497436046600342\n",
      "Start and end (275, 280)\n",
      "idx:  275  Epoch:  115  loss:  247.05801  loss_sent:  1.643851  loss_word:  238.83875  Time cost:  6.544228553771973\n",
      "Start and end (280, 285)\n",
      "idx:  280  Epoch:  115  loss:  236.59178  loss_sent:  1.9918506  loss_word:  226.63249  Time cost:  6.540398836135864\n",
      "Start and end (285, 290)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  285  Epoch:  115  loss:  227.10129  loss_sent:  1.4031343  loss_word:  220.08563  Time cost:  6.544416904449463\n",
      "Start and end (290, 295)\n",
      "idx:  290  Epoch:  115  loss:  199.70818  loss_sent:  2.322212  loss_word:  188.0971  Time cost:  6.550437688827515\n",
      "Start and end (295, 300)\n",
      "idx:  295  Epoch:  115  loss:  225.34775  loss_sent:  1.97396  loss_word:  215.47795  Time cost:  6.54757022857666\n",
      "Start and end (300, 305)\n",
      "idx:  300  Epoch:  115  loss:  226.48012  loss_sent:  1.9856952  loss_word:  216.55164  Time cost:  6.5871922969818115\n",
      "Start and end (305, 310)\n",
      "idx:  305  Epoch:  115  loss:  190.08072  loss_sent:  1.5730655  loss_word:  182.21541  Time cost:  6.548961877822876\n",
      "Start and end (310, 315)\n",
      "idx:  310  Epoch:  115  loss:  210.22987  loss_sent:  2.1646614  loss_word:  199.40656  Time cost:  6.551495313644409\n",
      "Start and end (315, 320)\n",
      "idx:  315  Epoch:  115  loss:  313.97433  loss_sent:  2.5659733  loss_word:  301.14447  Time cost:  6.514719247817993\n",
      "Start and end (320, 325)\n",
      "idx:  320  Epoch:  115  loss:  220.81514  loss_sent:  1.4243283  loss_word:  213.69348  Time cost:  6.556912899017334\n",
      "Start and end (325, 330)\n",
      "idx:  325  Epoch:  115  loss:  326.61176  loss_sent:  1.503448  loss_word:  319.0945  Time cost:  6.468524694442749\n",
      "Start and end (330, 335)\n",
      "idx:  330  Epoch:  115  loss:  276.8919  loss_sent:  2.2591505  loss_word:  265.59616  Time cost:  6.512090444564819\n",
      "Start and end (335, 340)\n",
      "idx:  335  Epoch:  115  loss:  300.28137  loss_sent:  1.7688038  loss_word:  291.43738  Time cost:  6.50284218788147\n",
      "Start and end (340, 345)\n",
      "idx:  340  Epoch:  115  loss:  164.05217  loss_sent:  1.8392601  loss_word:  154.85587  Time cost:  6.602706670761108\n",
      "Start and end (345, 350)\n",
      "idx:  345  Epoch:  115  loss:  216.3318  loss_sent:  2.16226  loss_word:  205.52052  Time cost:  6.684878826141357\n",
      "Start and end (350, 355)\n",
      "idx:  350  Epoch:  115  loss:  246.38467  loss_sent:  1.6114671  loss_word:  238.32735  Time cost:  6.606720209121704\n",
      "Start and end (355, 360)\n",
      "idx:  355  Epoch:  115  loss:  292.73608  loss_sent:  1.955481  loss_word:  282.95868  Time cost:  6.582614898681641\n",
      "Start and end (360, 365)\n",
      "idx:  360  Epoch:  115  loss:  207.91975  loss_sent:  1.6317756  loss_word:  199.76086  Time cost:  6.556362628936768\n",
      "Start and end (365, 370)\n",
      "idx:  365  Epoch:  115  loss:  238.67688  loss_sent:  1.7827022  loss_word:  229.76332  Time cost:  6.596062660217285\n",
      "Start and end (370, 375)\n",
      "idx:  370  Epoch:  115  loss:  254.67603  loss_sent:  2.1027246  loss_word:  244.16241  Time cost:  6.567529916763306\n",
      "Start and end (375, 380)\n",
      "idx:  375  Epoch:  115  loss:  278.44217  loss_sent:  1.6966921  loss_word:  269.9587  Time cost:  6.5795838832855225\n",
      "Start and end (380, 385)\n",
      "idx:  380  Epoch:  115  loss:  216.51706  loss_sent:  1.81314  loss_word:  207.45139  Time cost:  6.54676079750061\n",
      "Start and end (385, 390)\n",
      "idx:  385  Epoch:  115  loss:  222.3934  loss_sent:  1.981362  loss_word:  212.48662  Time cost:  6.542364597320557\n",
      "Start and end (390, 395)\n",
      "idx:  390  Epoch:  115  loss:  251.49568  loss_sent:  1.8066461  loss_word:  242.46242  Time cost:  6.631108999252319\n",
      "Start and end (395, 400)\n",
      "idx:  395  Epoch:  115  loss:  197.66255  loss_sent:  1.8195952  loss_word:  188.5646  Time cost:  6.547134876251221\n",
      "Start and end (400, 405)\n",
      "idx:  400  Epoch:  115  loss:  212.33221  loss_sent:  1.6221179  loss_word:  204.22165  Time cost:  6.564903497695923\n",
      "Start and end (405, 410)\n",
      "idx:  405  Epoch:  115  loss:  263.3345  loss_sent:  1.4648695  loss_word:  256.01022  Time cost:  6.468766450881958\n",
      "Start and end (410, 415)\n",
      "idx:  410  Epoch:  115  loss:  174.34904  loss_sent:  1.5529509  loss_word:  166.58429  Time cost:  6.549022674560547\n",
      "Start and end (415, 420)\n",
      "idx:  415  Epoch:  115  loss:  233.36052  loss_sent:  1.7680535  loss_word:  224.52028  Time cost:  6.467564821243286\n",
      "Start and end (420, 425)\n",
      "idx:  420  Epoch:  115  loss:  181.00786  loss_sent:  2.6396146  loss_word:  167.80978  Time cost:  6.530359745025635\n",
      "Start and end (425, 430)\n",
      "idx:  425  Epoch:  115  loss:  316.28928  loss_sent:  2.4321303  loss_word:  304.12863  Time cost:  6.579704284667969\n",
      "Start and end (430, 435)\n",
      "idx:  430  Epoch:  115  loss:  212.61186  loss_sent:  1.4678364  loss_word:  205.27269  Time cost:  6.525418996810913\n",
      "Start and end (435, 440)\n",
      "idx:  435  Epoch:  115  loss:  277.9454  loss_sent:  2.35552  loss_word:  266.16776  Time cost:  6.607743263244629\n",
      "Start and end (440, 445)\n",
      "idx:  440  Epoch:  115  loss:  221.09056  loss_sent:  1.4798282  loss_word:  213.69144  Time cost:  6.4726057052612305\n",
      "Start and end (445, 450)\n",
      "idx:  445  Epoch:  115  loss:  251.46895  loss_sent:  1.5769935  loss_word:  243.58398  Time cost:  6.525395631790161\n",
      "Start and end (450, 455)\n",
      "idx:  450  Epoch:  115  loss:  270.9906  loss_sent:  2.78881  loss_word:  257.04654  Time cost:  6.626106262207031\n",
      "Start and end (455, 460)\n",
      "idx:  455  Epoch:  115  loss:  211.39922  loss_sent:  2.0912519  loss_word:  200.94298  Time cost:  6.5261147022247314\n",
      "Start and end (460, 465)\n",
      "idx:  460  Epoch:  115  loss:  228.90623  loss_sent:  1.3961872  loss_word:  221.92531  Time cost:  6.552801609039307\n",
      "Start and end (465, 470)\n",
      "idx:  465  Epoch:  115  loss:  239.97519  loss_sent:  2.1628299  loss_word:  229.16104  Time cost:  6.550593852996826\n",
      "Start and end (470, 475)\n",
      "idx:  470  Epoch:  115  loss:  220.97011  loss_sent:  2.6353085  loss_word:  207.7936  Time cost:  6.584441423416138\n",
      "Start and end (475, 480)\n",
      "idx:  475  Epoch:  115  loss:  249.21866  loss_sent:  1.7882947  loss_word:  240.27719  Time cost:  6.592432737350464\n",
      "Start and end (480, 485)\n",
      "idx:  480  Epoch:  115  loss:  252.38676  loss_sent:  2.0717056  loss_word:  242.02823  Time cost:  6.569320917129517\n",
      "Start and end (485, 490)\n",
      "idx:  485  Epoch:  115  loss:  253.1351  loss_sent:  1.7787025  loss_word:  244.24158  Time cost:  6.609017372131348\n",
      "Start and end (490, 495)\n",
      "idx:  490  Epoch:  115  loss:  288.49277  loss_sent:  2.0401225  loss_word:  278.2922  Time cost:  6.461894273757935\n",
      "Start and end (495, 500)\n",
      "idx:  495  Epoch:  115  loss:  306.9889  loss_sent:  2.220231  loss_word:  295.88776  Time cost:  6.503926515579224\n",
      "Start and end (500, 505)\n",
      "idx:  500  Epoch:  115  loss:  204.40544  loss_sent:  2.173199  loss_word:  193.53943  Time cost:  6.5039873123168945\n",
      "Start and end (505, 510)\n",
      "idx:  505  Epoch:  115  loss:  202.20535  loss_sent:  1.8299115  loss_word:  193.05579  Time cost:  6.573160409927368\n",
      "Start and end (510, 515)\n",
      "idx:  510  Epoch:  115  loss:  274.47046  loss_sent:  1.4170043  loss_word:  267.38544  Time cost:  6.54200553894043\n",
      "Start and end (515, 520)\n",
      "idx:  515  Epoch:  115  loss:  284.0361  loss_sent:  1.780343  loss_word:  275.13437  Time cost:  6.613509654998779\n",
      "Start and end (520, 525)\n",
      "idx:  520  Epoch:  115  loss:  230.22536  loss_sent:  2.0990403  loss_word:  219.73018  Time cost:  6.493567943572998\n",
      "Start and end (525, 530)\n",
      "idx:  525  Epoch:  115  loss:  253.32666  loss_sent:  1.8245822  loss_word:  244.20375  Time cost:  6.5627663135528564\n",
      "Start and end (530, 535)\n",
      "idx:  530  Epoch:  115  loss:  207.82817  loss_sent:  1.9072608  loss_word:  198.29187  Time cost:  6.518000841140747\n",
      "Start and end (535, 540)\n",
      "idx:  535  Epoch:  115  loss:  217.6668  loss_sent:  1.8365054  loss_word:  208.48425  Time cost:  6.582961320877075\n",
      "Start and end (540, 545)\n",
      "idx:  540  Epoch:  115  loss:  208.2496  loss_sent:  1.9106972  loss_word:  198.69615  Time cost:  6.541187286376953\n",
      "Start and end (545, 550)\n",
      "idx:  545  Epoch:  115  loss:  261.37335  loss_sent:  1.964541  loss_word:  251.55066  Time cost:  6.592541933059692\n",
      "Start and end (550, 555)\n",
      "idx:  550  Epoch:  115  loss:  222.23325  loss_sent:  2.2976463  loss_word:  210.745  Time cost:  6.505530834197998\n",
      "Start and end (555, 560)\n",
      "idx:  555  Epoch:  115  loss:  187.95616  loss_sent:  1.7819225  loss_word:  179.04654  Time cost:  6.4521849155426025\n",
      "Start and end (560, 565)\n",
      "idx:  560  Epoch:  115  loss:  227.03923  loss_sent:  2.0659132  loss_word:  216.70966  Time cost:  6.530263423919678\n",
      "Start and end (565, 570)\n",
      "idx:  565  Epoch:  115  loss:  231.69441  loss_sent:  2.066874  loss_word:  221.36003  Time cost:  6.5088419914245605\n",
      "Start and end (570, 575)\n",
      "idx:  570  Epoch:  115  loss:  227.84116  loss_sent:  1.7984526  loss_word:  218.84889  Time cost:  6.479828357696533\n",
      "Start and end (575, 580)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  575  Epoch:  115  loss:  242.19508  loss_sent:  1.8207023  loss_word:  233.09157  Time cost:  6.610602378845215\n",
      "Start and end (580, 585)\n",
      "idx:  580  Epoch:  115  loss:  294.96393  loss_sent:  1.8543894  loss_word:  285.692  Time cost:  6.532600164413452\n",
      "Start and end (585, 590)\n",
      "idx:  585  Epoch:  115  loss:  232.20868  loss_sent:  2.1417499  loss_word:  221.49991  Time cost:  6.608145713806152\n",
      "Start and end (590, 595)\n",
      "idx:  590  Epoch:  115  loss:  228.55351  loss_sent:  3.4392667  loss_word:  211.35718  Time cost:  6.562000036239624\n",
      "Start and end (595, 600)\n",
      "idx:  595  Epoch:  115  loss:  259.59753  loss_sent:  1.8479135  loss_word:  250.35796  Time cost:  6.580857276916504\n",
      "Start and end (600, 605)\n",
      "idx:  600  Epoch:  115  loss:  211.09381  loss_sent:  2.0770216  loss_word:  200.70871  Time cost:  6.557347059249878\n",
      "Start and end (605, 610)\n",
      "idx:  605  Epoch:  115  loss:  280.23077  loss_sent:  1.9085093  loss_word:  270.68823  Time cost:  6.489459276199341\n",
      "Start and end (610, 615)\n",
      "idx:  610  Epoch:  115  loss:  311.53098  loss_sent:  1.5202731  loss_word:  303.92957  Time cost:  6.561004161834717\n",
      "Start and end (615, 620)\n",
      "idx:  615  Epoch:  115  loss:  238.57689  loss_sent:  1.923313  loss_word:  228.96031  Time cost:  7.2332823276519775\n",
      "Start and end (620, 625)\n",
      "idx:  620  Epoch:  115  loss:  280.96024  loss_sent:  1.9384325  loss_word:  271.2681  Time cost:  7.866427898406982\n",
      "Start and end (625, 630)\n",
      "idx:  625  Epoch:  115  loss:  184.54744  loss_sent:  1.8591702  loss_word:  175.25159  Time cost:  6.654487133026123\n",
      "Start and end (630, 635)\n",
      "idx:  630  Epoch:  115  loss:  217.52176  loss_sent:  1.8274014  loss_word:  208.38478  Time cost:  6.500935316085815\n",
      "Start and end (635, 640)\n",
      "idx:  635  Epoch:  115  loss:  217.80138  loss_sent:  1.7328392  loss_word:  209.13718  Time cost:  6.500303268432617\n",
      "Start and end (640, 645)\n",
      "idx:  640  Epoch:  115  loss:  186.37807  loss_sent:  1.7451319  loss_word:  177.65242  Time cost:  6.492330551147461\n",
      "Start and end (645, 650)\n",
      "idx:  645  Epoch:  115  loss:  250.0157  loss_sent:  2.1141667  loss_word:  239.44489  Time cost:  6.557389974594116\n",
      "Start and end (650, 655)\n",
      "idx:  650  Epoch:  115  loss:  221.30959  loss_sent:  1.7937416  loss_word:  212.34088  Time cost:  6.467353105545044\n",
      "Start and end (655, 660)\n",
      "idx:  655  Epoch:  115  loss:  247.09181  loss_sent:  1.5719256  loss_word:  239.2322  Time cost:  6.589840412139893\n",
      "Start and end (660, 665)\n",
      "idx:  660  Epoch:  115  loss:  166.51062  loss_sent:  1.7017602  loss_word:  158.00183  Time cost:  6.468738079071045\n",
      "Start and end (665, 670)\n",
      "idx:  665  Epoch:  115  loss:  225.57127  loss_sent:  1.354721  loss_word:  218.79768  Time cost:  6.622057676315308\n",
      "Start and end (670, 675)\n",
      "idx:  670  Epoch:  115  loss:  213.04747  loss_sent:  3.3290095  loss_word:  196.40244  Time cost:  6.650406360626221\n",
      "Start and end (675, 680)\n",
      "idx:  675  Epoch:  115  loss:  269.13287  loss_sent:  1.7798643  loss_word:  260.23358  Time cost:  6.615220546722412\n",
      "Start and end (680, 685)\n",
      "idx:  680  Epoch:  115  loss:  245.18141  loss_sent:  1.8041115  loss_word:  236.16086  Time cost:  6.6335694789886475\n",
      "Start and end (685, 690)\n",
      "idx:  685  Epoch:  115  loss:  265.20322  loss_sent:  1.6972694  loss_word:  256.71683  Time cost:  6.481216192245483\n",
      "Start and end (690, 695)\n",
      "idx:  690  Epoch:  115  loss:  191.45078  loss_sent:  1.4744884  loss_word:  184.07832  Time cost:  6.589273929595947\n",
      "Start and end (695, 700)\n",
      "idx:  695  Epoch:  115  loss:  252.93442  loss_sent:  1.9133189  loss_word:  243.3678  Time cost:  6.599924564361572\n",
      "Start and end (700, 705)\n",
      "idx:  700  Epoch:  115  loss:  228.93501  loss_sent:  1.9939429  loss_word:  218.96529  Time cost:  6.606741428375244\n",
      "Start and end (705, 710)\n",
      "idx:  705  Epoch:  115  loss:  239.71388  loss_sent:  1.9682747  loss_word:  229.8725  Time cost:  6.727277517318726\n",
      "Start and end (710, 715)\n",
      "idx:  710  Epoch:  115  loss:  235.263  loss_sent:  2.0034456  loss_word:  225.24574  Time cost:  6.533960819244385\n",
      "Start and end (715, 720)\n",
      "idx:  715  Epoch:  115  loss:  245.78865  loss_sent:  1.474603  loss_word:  238.41563  Time cost:  6.489158630371094\n",
      "Start and end (720, 725)\n",
      "idx:  720  Epoch:  115  loss:  199.06395  loss_sent:  2.3064806  loss_word:  187.53154  Time cost:  6.546464920043945\n",
      "Start and end (725, 730)\n",
      "idx:  725  Epoch:  115  loss:  226.94398  loss_sent:  2.3421476  loss_word:  215.23326  Time cost:  6.4593284130096436\n",
      "Start and end (730, 735)\n",
      "idx:  730  Epoch:  115  loss:  191.66486  loss_sent:  1.5369633  loss_word:  183.98004  Time cost:  6.537261962890625\n",
      "Start and end (735, 740)\n",
      "idx:  735  Epoch:  115  loss:  266.29263  loss_sent:  1.8434925  loss_word:  257.07516  Time cost:  6.5533130168914795\n",
      "Start and end (740, 745)\n",
      "idx:  740  Epoch:  115  loss:  231.81995  loss_sent:  2.7036703  loss_word:  218.30157  Time cost:  6.6573240756988525\n",
      "Start and end (745, 750)\n",
      "idx:  745  Epoch:  115  loss:  206.23093  loss_sent:  2.380503  loss_word:  194.32841  Time cost:  6.440550088882446\n",
      "Start and end (750, 755)\n",
      "idx:  750  Epoch:  115  loss:  275.11996  loss_sent:  1.5718927  loss_word:  267.2605  Time cost:  6.520642995834351\n",
      "Start and end (755, 760)\n",
      "idx:  755  Epoch:  115  loss:  257.66336  loss_sent:  1.7625717  loss_word:  248.85045  Time cost:  6.591115951538086\n",
      "Start and end (760, 765)\n",
      "idx:  760  Epoch:  115  loss:  395.40036  loss_sent:  2.278597  loss_word:  384.0074  Time cost:  6.589413642883301\n",
      "Start and end (765, 770)\n",
      "idx:  765  Epoch:  115  loss:  152.31267  loss_sent:  1.4762782  loss_word:  144.93129  Time cost:  6.527722597122192\n",
      "Start and end (770, 775)\n",
      "idx:  770  Epoch:  115  loss:  267.61203  loss_sent:  2.0510063  loss_word:  257.357  Time cost:  6.504171371459961\n",
      "Start and end (775, 780)\n",
      "idx:  775  Epoch:  115  loss:  216.6346  loss_sent:  1.823234  loss_word:  207.51842  Time cost:  6.492674827575684\n",
      "Start and end (780, 785)\n",
      "idx:  780  Epoch:  115  loss:  228.09332  loss_sent:  1.9837952  loss_word:  218.17433  Time cost:  6.503352880477905\n",
      "Start and end (785, 790)\n",
      "idx:  785  Epoch:  115  loss:  300.784  loss_sent:  1.9148569  loss_word:  291.20972  Time cost:  6.488408803939819\n",
      "Start and end (790, 795)\n",
      "idx:  790  Epoch:  115  loss:  193.60446  loss_sent:  1.4964968  loss_word:  186.122  Time cost:  6.554333686828613\n",
      "Start and end (795, 800)\n",
      "idx:  795  Epoch:  115  loss:  177.72932  loss_sent:  2.6525385  loss_word:  164.46663  Time cost:  6.5176780223846436\n",
      "Start and end (800, 805)\n",
      "idx:  800  Epoch:  115  loss:  294.09866  loss_sent:  2.0367863  loss_word:  283.91473  Time cost:  6.602144241333008\n",
      "Start and end (805, 810)\n",
      "idx:  805  Epoch:  115  loss:  315.6449  loss_sent:  1.6495407  loss_word:  307.39722  Time cost:  6.569754123687744\n",
      "Start and end (810, 815)\n",
      "idx:  810  Epoch:  115  loss:  219.27556  loss_sent:  1.6565334  loss_word:  210.99287  Time cost:  6.599113702774048\n",
      "Start and end (815, 820)\n",
      "idx:  815  Epoch:  115  loss:  239.30592  loss_sent:  1.6767923  loss_word:  230.92197  Time cost:  6.498109817504883\n",
      "Start and end (820, 825)\n",
      "idx:  820  Epoch:  115  loss:  207.37018  loss_sent:  1.6246866  loss_word:  199.24675  Time cost:  6.594387769699097\n",
      "Start and end (825, 830)\n",
      "idx:  825  Epoch:  115  loss:  224.53946  loss_sent:  1.9792774  loss_word:  214.64305  Time cost:  6.54120397567749\n",
      "Start and end (830, 835)\n",
      "idx:  830  Epoch:  115  loss:  286.79526  loss_sent:  1.6941165  loss_word:  278.32462  Time cost:  6.567474603652954\n",
      "Start and end (835, 840)\n",
      "idx:  835  Epoch:  115  loss:  246.1243  loss_sent:  2.5721753  loss_word:  233.26341  Time cost:  6.632546663284302\n",
      "Start and end (840, 845)\n",
      "idx:  840  Epoch:  115  loss:  227.12895  loss_sent:  1.3229845  loss_word:  220.51404  Time cost:  6.539978742599487\n",
      "Start and end (845, 850)\n",
      "idx:  845  Epoch:  115  loss:  211.46721  loss_sent:  2.4292753  loss_word:  199.32082  Time cost:  6.562519788742065\n",
      "Start and end (850, 855)\n",
      "idx:  850  Epoch:  115  loss:  230.52104  loss_sent:  1.3645371  loss_word:  223.69836  Time cost:  6.554129123687744\n",
      "Start and end (855, 860)\n",
      "idx:  855  Epoch:  115  loss:  171.99977  loss_sent:  1.9765949  loss_word:  162.1168  Time cost:  6.545170068740845\n",
      "Start and end (860, 865)\n",
      "idx:  860  Epoch:  115  loss:  202.00928  loss_sent:  1.5269626  loss_word:  194.37445  Time cost:  6.461559534072876\n",
      "Start and end (865, 870)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  865  Epoch:  115  loss:  213.87404  loss_sent:  1.6006912  loss_word:  205.87059  Time cost:  6.573620080947876\n",
      "Start and end (870, 875)\n",
      "idx:  870  Epoch:  115  loss:  194.88557  loss_sent:  1.8284088  loss_word:  185.7435  Time cost:  6.560470819473267\n",
      "Start and end (875, 880)\n",
      "idx:  875  Epoch:  115  loss:  203.58351  loss_sent:  1.7108375  loss_word:  195.02933  Time cost:  6.61773157119751\n",
      "Start and end (880, 885)\n",
      "idx:  880  Epoch:  115  loss:  228.98384  loss_sent:  1.8646425  loss_word:  219.66063  Time cost:  6.540525436401367\n",
      "Start and end (885, 890)\n",
      "idx:  885  Epoch:  115  loss:  178.51343  loss_sent:  1.3977183  loss_word:  171.52484  Time cost:  6.562628984451294\n",
      "Start and end (890, 895)\n",
      "idx:  890  Epoch:  115  loss:  241.43037  loss_sent:  2.4066086  loss_word:  229.39732  Time cost:  6.578070163726807\n",
      "Start and end (895, 900)\n",
      "idx:  895  Epoch:  115  loss:  237.60583  loss_sent:  1.4297636  loss_word:  230.45702  Time cost:  6.531609296798706\n",
      "Start and end (900, 905)\n",
      "idx:  900  Epoch:  115  loss:  190.11758  loss_sent:  1.9265916  loss_word:  180.48463  Time cost:  6.558412075042725\n",
      "Start and end (905, 910)\n",
      "idx:  905  Epoch:  115  loss:  198.2967  loss_sent:  2.0405293  loss_word:  188.09406  Time cost:  6.493652582168579\n",
      "Start and end (910, 915)\n",
      "idx:  910  Epoch:  115  loss:  142.1938  loss_sent:  1.5965811  loss_word:  134.21089  Time cost:  6.489806413650513\n",
      "Start and end (915, 920)\n",
      "idx:  915  Epoch:  115  loss:  268.61398  loss_sent:  1.9428821  loss_word:  258.89957  Time cost:  6.552495002746582\n",
      "Start and end (920, 925)\n",
      "idx:  920  Epoch:  115  loss:  175.02171  loss_sent:  1.87218  loss_word:  165.66081  Time cost:  6.498023748397827\n",
      "Start and end (925, 930)\n",
      "idx:  925  Epoch:  115  loss:  190.71175  loss_sent:  1.7591879  loss_word:  181.91579  Time cost:  6.456774473190308\n",
      "Start and end (930, 935)\n",
      "idx:  930  Epoch:  115  loss:  260.4341  loss_sent:  1.8216314  loss_word:  251.32593  Time cost:  6.619681358337402\n",
      "Start and end (935, 940)\n",
      "idx:  935  Epoch:  115  loss:  269.61667  loss_sent:  2.1116464  loss_word:  259.0584  Time cost:  6.492242097854614\n",
      "Start and end (940, 945)\n",
      "idx:  940  Epoch:  115  loss:  286.45523  loss_sent:  1.5831192  loss_word:  278.53964  Time cost:  6.6393797397613525\n",
      "Start and end (945, 950)\n",
      "idx:  945  Epoch:  115  loss:  186.76462  loss_sent:  1.8012676  loss_word:  177.75826  Time cost:  6.5803656578063965\n",
      "Start and end (950, 955)\n",
      "idx:  950  Epoch:  115  loss:  233.79732  loss_sent:  1.7345656  loss_word:  225.12451  Time cost:  6.58075475692749\n",
      "Start and end (955, 960)\n",
      "idx:  955  Epoch:  115  loss:  291.78098  loss_sent:  1.579426  loss_word:  283.88385  Time cost:  6.5107715129852295\n",
      "Start and end (960, 965)\n",
      "idx:  960  Epoch:  115  loss:  371.8492  loss_sent:  1.8388733  loss_word:  362.65485  Time cost:  6.4851624965667725\n",
      "Start and end (965, 970)\n",
      "idx:  965  Epoch:  115  loss:  162.6702  loss_sent:  1.4501495  loss_word:  155.41945  Time cost:  6.523627042770386\n",
      "Start and end (970, 975)\n",
      "idx:  970  Epoch:  115  loss:  222.56604  loss_sent:  1.8116665  loss_word:  213.5077  Time cost:  6.490049600601196\n",
      "Start and end (975, 980)\n",
      "idx:  975  Epoch:  115  loss:  287.93243  loss_sent:  1.4359477  loss_word:  280.7527  Time cost:  6.600868463516235\n",
      "Start and end (980, 985)\n",
      "idx:  980  Epoch:  115  loss:  234.24663  loss_sent:  1.8906031  loss_word:  224.79362  Time cost:  6.541654348373413\n",
      "Start and end (985, 990)\n",
      "idx:  985  Epoch:  115  loss:  224.69768  loss_sent:  1.8862646  loss_word:  215.26628  Time cost:  6.563800096511841\n",
      "Start and end (990, 995)\n",
      "idx:  990  Epoch:  115  loss:  311.58527  loss_sent:  1.592071  loss_word:  303.62494  Time cost:  6.487108945846558\n",
      "Start and end (995, 1000)\n",
      "idx:  995  Epoch:  115  loss:  244.47462  loss_sent:  2.1051068  loss_word:  233.94905  Time cost:  6.551359176635742\n",
      "Start and end (1000, 1005)\n",
      "idx:  1000  Epoch:  115  loss:  244.9067  loss_sent:  2.026137  loss_word:  234.77602  Time cost:  6.573143482208252\n",
      "Start and end (1005, 1010)\n",
      "idx:  1005  Epoch:  115  loss:  233.35379  loss_sent:  1.4426376  loss_word:  226.1406  Time cost:  6.450022220611572\n",
      "Start and end (1010, 1015)\n",
      "idx:  1010  Epoch:  115  loss:  219.85173  loss_sent:  1.8285102  loss_word:  210.7092  Time cost:  6.594670057296753\n",
      "Start and end (1015, 1020)\n",
      "idx:  1015  Epoch:  115  loss:  237.92621  loss_sent:  3.0691872  loss_word:  222.58028  Time cost:  6.573557615280151\n",
      "Start and end (1020, 1025)\n",
      "idx:  1020  Epoch:  115  loss:  250.7652  loss_sent:  2.5543149  loss_word:  237.99364  Time cost:  6.506190776824951\n",
      "Start and end (1025, 1030)\n",
      "idx:  1025  Epoch:  115  loss:  242.28436  loss_sent:  3.5137634  loss_word:  224.71553  Time cost:  6.570125341415405\n",
      "Start and end (1030, 1035)\n",
      "idx:  1030  Epoch:  115  loss:  303.3496  loss_sent:  2.186806  loss_word:  292.41556  Time cost:  6.618351459503174\n",
      "Start and end (1035, 1040)\n",
      "idx:  1035  Epoch:  115  loss:  236.15671  loss_sent:  2.7586076  loss_word:  222.36366  Time cost:  6.596365690231323\n",
      "Start and end (1040, 1045)\n",
      "idx:  1040  Epoch:  115  loss:  216.33807  loss_sent:  1.917146  loss_word:  206.75232  Time cost:  6.496789932250977\n",
      "Start and end (1045, 1050)\n",
      "idx:  1045  Epoch:  115  loss:  223.73323  loss_sent:  2.0647943  loss_word:  213.40926  Time cost:  6.555987119674683\n",
      "Start and end (1050, 1055)\n",
      "idx:  1050  Epoch:  115  loss:  227.59721  loss_sent:  2.2701838  loss_word:  216.24628  Time cost:  6.573515892028809\n",
      "Start and end (1055, 1060)\n",
      "idx:  1055  Epoch:  115  loss:  230.83817  loss_sent:  1.9820096  loss_word:  220.9281  Time cost:  6.566196441650391\n",
      "Start and end (1060, 1065)\n",
      "idx:  1060  Epoch:  115  loss:  190.89636  loss_sent:  1.518522  loss_word:  183.30376  Time cost:  6.583954811096191\n",
      "Start and end (1065, 1070)\n",
      "idx:  1065  Epoch:  115  loss:  145.35324  loss_sent:  3.281666  loss_word:  128.94492  Time cost:  6.526967287063599\n",
      "Start and end (1070, 1075)\n",
      "idx:  1070  Epoch:  115  loss:  185.38788  loss_sent:  2.417575  loss_word:  173.3  Time cost:  6.565503120422363\n",
      "Start and end (1075, 1080)\n",
      "idx:  1075  Epoch:  115  loss:  345.78436  loss_sent:  1.7724373  loss_word:  336.92218  Time cost:  6.496175289154053\n",
      "Start and end (1080, 1085)\n",
      "idx:  1080  Epoch:  115  loss:  233.0213  loss_sent:  1.6594292  loss_word:  224.72414  Time cost:  6.468804836273193\n",
      "Start and end (1085, 1090)\n",
      "idx:  1085  Epoch:  115  loss:  257.50967  loss_sent:  1.9382633  loss_word:  247.81833  Time cost:  6.526208162307739\n",
      "Start and end (1090, 1095)\n",
      "idx:  1090  Epoch:  115  loss:  348.21893  loss_sent:  1.9317776  loss_word:  338.56006  Time cost:  6.6207826137542725\n",
      "Start and end (1095, 1100)\n",
      "idx:  1095  Epoch:  115  loss:  246.57164  loss_sent:  1.7081944  loss_word:  238.03067  Time cost:  6.54033350944519\n",
      "Start and end (1100, 1105)\n",
      "idx:  1100  Epoch:  115  loss:  260.58347  loss_sent:  2.1289055  loss_word:  249.9389  Time cost:  6.614232540130615\n",
      "Start and end (1105, 1110)\n",
      "idx:  1105  Epoch:  115  loss:  128.6457  loss_sent:  1.5155431  loss_word:  121.06797  Time cost:  6.581114768981934\n",
      "Start and end (1110, 1115)\n",
      "idx:  1110  Epoch:  115  loss:  276.8244  loss_sent:  2.313979  loss_word:  265.25452  Time cost:  6.5333030223846436\n",
      "Start and end (1115, 1120)\n",
      "idx:  1115  Epoch:  115  loss:  272.15503  loss_sent:  3.0454571  loss_word:  256.9277  Time cost:  6.5658605098724365\n",
      "Start and end (1120, 1125)\n",
      "idx:  1120  Epoch:  115  loss:  245.19592  loss_sent:  2.1153622  loss_word:  234.61911  Time cost:  6.585044622421265\n",
      "Start and end (1125, 1130)\n",
      "idx:  1125  Epoch:  115  loss:  241.80737  loss_sent:  2.3572369  loss_word:  230.02121  Time cost:  6.621006488800049\n",
      "Start and end (1130, 1135)\n",
      "idx:  1130  Epoch:  115  loss:  291.3487  loss_sent:  1.722187  loss_word:  282.73776  Time cost:  6.502613067626953\n",
      "Start and end (1135, 1140)\n",
      "idx:  1135  Epoch:  115  loss:  208.41652  loss_sent:  1.9360824  loss_word:  198.73611  Time cost:  6.470520257949829\n",
      "Start and end (1140, 1145)\n",
      "idx:  1140  Epoch:  115  loss:  223.40103  loss_sent:  1.6725137  loss_word:  215.03848  Time cost:  6.446225881576538\n",
      "Start and end (1145, 1150)\n",
      "idx:  1145  Epoch:  115  loss:  157.25629  loss_sent:  1.6966534  loss_word:  148.77304  Time cost:  6.51557731628418\n",
      "Start and end (1150, 1155)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1150  Epoch:  115  loss:  241.41997  loss_sent:  1.5853776  loss_word:  233.49309  Time cost:  6.558082818984985\n",
      "Start and end (1155, 1160)\n",
      "idx:  1155  Epoch:  115  loss:  335.17902  loss_sent:  1.76161  loss_word:  326.37097  Time cost:  6.557438135147095\n",
      "Start and end (1160, 1165)\n",
      "idx:  1160  Epoch:  115  loss:  247.49905  loss_sent:  1.9415225  loss_word:  237.79144  Time cost:  6.608539342880249\n",
      "Start and end (1165, 1170)\n",
      "idx:  1165  Epoch:  115  loss:  228.19238  loss_sent:  2.186231  loss_word:  217.26125  Time cost:  6.610917091369629\n",
      "Start and end (1170, 1175)\n",
      "idx:  1170  Epoch:  115  loss:  343.43307  loss_sent:  1.8384043  loss_word:  334.24106  Time cost:  6.609920978546143\n",
      "Start and end (1175, 1180)\n",
      "idx:  1175  Epoch:  115  loss:  150.5342  loss_sent:  1.4695367  loss_word:  143.1865  Time cost:  6.611990928649902\n",
      "Start and end (1180, 1185)\n",
      "idx:  1180  Epoch:  115  loss:  206.59203  loss_sent:  1.7988496  loss_word:  197.59776  Time cost:  6.481385707855225\n",
      "Start and end (1185, 1190)\n",
      "idx:  1185  Epoch:  115  loss:  284.9615  loss_sent:  1.4081476  loss_word:  277.9208  Time cost:  6.572110414505005\n",
      "Start and end (1190, 1195)\n",
      "idx:  1190  Epoch:  115  loss:  148.43398  loss_sent:  3.063238  loss_word:  133.11777  Time cost:  6.554372310638428\n",
      "Start and end (1195, 1200)\n",
      "idx:  1195  Epoch:  115  loss:  223.64185  loss_sent:  2.5447013  loss_word:  210.91835  Time cost:  6.588755369186401\n",
      "Start and end (1200, 1205)\n",
      "idx:  1200  Epoch:  115  loss:  264.2517  loss_sent:  1.3688487  loss_word:  257.40747  Time cost:  6.545276641845703\n",
      "Start and end (1205, 1210)\n",
      "idx:  1205  Epoch:  115  loss:  293.4451  loss_sent:  1.8394486  loss_word:  284.24786  Time cost:  6.495962619781494\n",
      "Start and end (1210, 1215)\n",
      "idx:  1210  Epoch:  115  loss:  211.24542  loss_sent:  1.8476461  loss_word:  202.00719  Time cost:  6.598596572875977\n",
      "Start and end (1215, 1220)\n",
      "idx:  1215  Epoch:  115  loss:  287.59915  loss_sent:  2.3616703  loss_word:  275.79077  Time cost:  6.716066360473633\n",
      "Start and end (1220, 1225)\n",
      "idx:  1220  Epoch:  115  loss:  146.82735  loss_sent:  2.080991  loss_word:  136.42236  Time cost:  6.8613786697387695\n",
      "Start and end (1225, 1230)\n",
      "idx:  1225  Epoch:  115  loss:  241.28433  loss_sent:  1.9215431  loss_word:  231.67664  Time cost:  6.849562644958496\n",
      "Start and end (1230, 1235)\n",
      "idx:  1230  Epoch:  115  loss:  236.26257  loss_sent:  2.5488856  loss_word:  223.51814  Time cost:  6.55456805229187\n",
      "Start and end (1235, 1240)\n",
      "idx:  1235  Epoch:  115  loss:  323.93234  loss_sent:  1.5670092  loss_word:  316.0973  Time cost:  6.729562282562256\n",
      "Start and end (1240, 1245)\n",
      "idx:  1240  Epoch:  115  loss:  261.15878  loss_sent:  1.7874832  loss_word:  252.22134  Time cost:  6.8062615394592285\n",
      "Start and end (1245, 1250)\n",
      "idx:  1245  Epoch:  115  loss:  272.3558  loss_sent:  1.7344372  loss_word:  263.68362  Time cost:  6.715118408203125\n",
      "Start and end (1250, 1255)\n",
      "idx:  1250  Epoch:  115  loss:  204.84085  loss_sent:  1.6981883  loss_word:  196.3499  Time cost:  6.698637008666992\n",
      "Start and end (1255, 1260)\n",
      "idx:  1255  Epoch:  115  loss:  252.70764  loss_sent:  1.6362226  loss_word:  244.5265  Time cost:  6.5257604122161865\n",
      "Start and end (1260, 1265)\n",
      "idx:  1260  Epoch:  115  loss:  198.95557  loss_sent:  2.3464646  loss_word:  187.22324  Time cost:  6.570891857147217\n",
      "Start and end (1265, 1270)\n",
      "idx:  1265  Epoch:  115  loss:  215.8318  loss_sent:  2.9931397  loss_word:  200.8661  Time cost:  6.459651708602905\n",
      "Start and end (1270, 1275)\n",
      "idx:  1270  Epoch:  115  loss:  189.78114  loss_sent:  1.6142476  loss_word:  181.70992  Time cost:  6.553607225418091\n",
      "Start and end (1275, 1280)\n",
      "idx:  1275  Epoch:  115  loss:  226.67198  loss_sent:  1.8672785  loss_word:  217.33556  Time cost:  6.506146669387817\n",
      "Start and end (1280, 1285)\n",
      "idx:  1280  Epoch:  115  loss:  237.03412  loss_sent:  2.4767728  loss_word:  224.65024  Time cost:  6.5208868980407715\n",
      "Start and end (1285, 1290)\n",
      "idx:  1285  Epoch:  115  loss:  247.75363  loss_sent:  1.9639008  loss_word:  237.93414  Time cost:  6.552831172943115\n",
      "Start and end (1290, 1295)\n",
      "idx:  1290  Epoch:  115  loss:  244.33641  loss_sent:  1.5768399  loss_word:  236.45221  Time cost:  6.5343687534332275\n",
      "Start and end (1295, 1300)\n",
      "idx:  1295  Epoch:  115  loss:  248.20575  loss_sent:  2.2242503  loss_word:  237.0845  Time cost:  6.467658996582031\n",
      "Start and end (1300, 1305)\n",
      "idx:  1300  Epoch:  115  loss:  287.6552  loss_sent:  2.0873063  loss_word:  277.21872  Time cost:  6.5214738845825195\n",
      "Start and end (1305, 1310)\n",
      "idx:  1305  Epoch:  115  loss:  274.00156  loss_sent:  1.5577148  loss_word:  266.21292  Time cost:  6.54013466835022\n",
      "Start and end (1310, 1315)\n",
      "idx:  1310  Epoch:  115  loss:  272.31314  loss_sent:  2.362004  loss_word:  260.50314  Time cost:  6.5794196128845215\n",
      "Start and end (1315, 1320)\n",
      "idx:  1315  Epoch:  115  loss:  138.81543  loss_sent:  2.3680146  loss_word:  126.97534  Time cost:  6.52534031867981\n",
      "Start and end (1320, 1325)\n",
      "idx:  1320  Epoch:  115  loss:  201.26968  loss_sent:  2.0535877  loss_word:  191.00175  Time cost:  6.558528661727905\n",
      "Start and end (1325, 1330)\n",
      "idx:  1325  Epoch:  115  loss:  267.70816  loss_sent:  1.6042211  loss_word:  259.68707  Time cost:  6.5298912525177\n",
      "Start and end (1330, 1335)\n",
      "idx:  1330  Epoch:  115  loss:  196.77321  loss_sent:  1.8088326  loss_word:  187.72905  Time cost:  6.453229188919067\n",
      "Start and end (1335, 1340)\n",
      "idx:  1335  Epoch:  115  loss:  231.53772  loss_sent:  1.7665842  loss_word:  222.7048  Time cost:  6.509268045425415\n",
      "Start and end (1340, 1345)\n",
      "idx:  1340  Epoch:  115  loss:  231.12193  loss_sent:  2.0456507  loss_word:  220.89368  Time cost:  6.439319133758545\n",
      "Start and end (1345, 1350)\n",
      "idx:  1345  Epoch:  115  loss:  177.01097  loss_sent:  2.0431468  loss_word:  166.79526  Time cost:  6.520097494125366\n",
      "Start and end (1350, 1355)\n",
      "idx:  1350  Epoch:  115  loss:  219.28287  loss_sent:  2.0687819  loss_word:  208.93896  Time cost:  6.530476331710815\n",
      "Start and end (1355, 1360)\n",
      "idx:  1355  Epoch:  115  loss:  214.94328  loss_sent:  2.5896022  loss_word:  201.99527  Time cost:  6.490750789642334\n",
      "Start and end (1360, 1365)\n",
      "idx:  1360  Epoch:  115  loss:  368.77786  loss_sent:  2.3252807  loss_word:  357.15143  Time cost:  6.433973073959351\n",
      "Start and end (1365, 1370)\n",
      "idx:  1365  Epoch:  115  loss:  208.21019  loss_sent:  1.5588715  loss_word:  200.41585  Time cost:  6.592164993286133\n",
      "Start and end (1370, 1375)\n",
      "idx:  1370  Epoch:  115  loss:  253.99692  loss_sent:  2.0021608  loss_word:  243.9861  Time cost:  6.518270492553711\n",
      "Start and end (1375, 1380)\n",
      "idx:  1375  Epoch:  115  loss:  174.32349  loss_sent:  1.8602386  loss_word:  165.02228  Time cost:  6.621734380722046\n",
      "Start and end (1380, 1385)\n",
      "idx:  1380  Epoch:  115  loss:  234.01947  loss_sent:  1.601752  loss_word:  226.01071  Time cost:  6.496202230453491\n",
      "Start and end (1385, 1390)\n",
      "idx:  1385  Epoch:  115  loss:  139.0661  loss_sent:  1.855891  loss_word:  129.78665  Time cost:  6.51272177696228\n",
      "Start and end (1390, 1395)\n",
      "idx:  1390  Epoch:  115  loss:  291.4511  loss_sent:  2.0302744  loss_word:  281.2997  Time cost:  6.539520502090454\n",
      "Start and end (1395, 1400)\n",
      "idx:  1395  Epoch:  115  loss:  215.71849  loss_sent:  2.216519  loss_word:  204.63588  Time cost:  6.521384000778198\n",
      "Start and end (1400, 1405)\n",
      "idx:  1400  Epoch:  115  loss:  229.25093  loss_sent:  1.7659303  loss_word:  220.42128  Time cost:  6.487649440765381\n",
      "Start and end (1405, 1410)\n",
      "idx:  1405  Epoch:  115  loss:  235.2797  loss_sent:  1.8137648  loss_word:  226.21086  Time cost:  6.454959154129028\n",
      "Start and end (1410, 1415)\n",
      "idx:  1410  Epoch:  115  loss:  221.70518  loss_sent:  2.1267762  loss_word:  211.0713  Time cost:  6.523097276687622\n",
      "Start and end (1415, 1420)\n",
      "idx:  1415  Epoch:  115  loss:  236.40524  loss_sent:  1.6344445  loss_word:  228.233  Time cost:  6.460398197174072\n",
      "Start and end (1420, 1425)\n",
      "idx:  1420  Epoch:  115  loss:  241.04436  loss_sent:  1.8615885  loss_word:  231.7364  Time cost:  6.58161735534668\n",
      "Start and end (1425, 1430)\n",
      "idx:  1425  Epoch:  115  loss:  241.72006  loss_sent:  1.7461687  loss_word:  232.98923  Time cost:  6.581026554107666\n",
      "Start and end (1430, 1435)\n",
      "idx:  1430  Epoch:  115  loss:  158.6116  loss_sent:  2.3080091  loss_word:  147.07156  Time cost:  6.523233890533447\n",
      "Start and end (1435, 1440)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1435  Epoch:  115  loss:  223.71596  loss_sent:  3.4284072  loss_word:  206.57393  Time cost:  6.543074607849121\n",
      "Start and end (1440, 1445)\n",
      "idx:  1440  Epoch:  115  loss:  223.51949  loss_sent:  1.7686872  loss_word:  214.67606  Time cost:  6.5240864753723145\n",
      "Start and end (1445, 1450)\n",
      "idx:  1445  Epoch:  115  loss:  251.88718  loss_sent:  1.6569717  loss_word:  243.6023  Time cost:  6.531020879745483\n",
      "Start and end (1450, 1455)\n",
      "idx:  1450  Epoch:  115  loss:  202.00087  loss_sent:  1.3510231  loss_word:  195.24576  Time cost:  6.475329875946045\n",
      "Start and end (1455, 1460)\n",
      "idx:  1455  Epoch:  115  loss:  209.91248  loss_sent:  1.7041551  loss_word:  201.3917  Time cost:  6.44689154624939\n",
      "Start and end (1460, 1465)\n",
      "idx:  1460  Epoch:  115  loss:  251.84239  loss_sent:  1.9209813  loss_word:  242.23749  Time cost:  6.525359392166138\n",
      "Start and end (1465, 1470)\n",
      "idx:  1465  Epoch:  115  loss:  284.86087  loss_sent:  1.7404497  loss_word:  276.15863  Time cost:  6.439506530761719\n",
      "Start and end (1470, 1475)\n",
      "idx:  1470  Epoch:  115  loss:  262.67603  loss_sent:  1.8674995  loss_word:  253.33852  Time cost:  6.530920743942261\n",
      "Start and end (1475, 1480)\n",
      "idx:  1475  Epoch:  115  loss:  276.03275  loss_sent:  1.4975191  loss_word:  268.54514  Time cost:  6.529829263687134\n",
      "Start and end (1480, 1485)\n",
      "idx:  1480  Epoch:  115  loss:  256.19623  loss_sent:  2.2344162  loss_word:  245.02415  Time cost:  6.582603693008423\n",
      "Start and end (1485, 1490)\n",
      "idx:  1485  Epoch:  115  loss:  292.26056  loss_sent:  1.6629729  loss_word:  283.94568  Time cost:  6.581607341766357\n",
      "Start and end (1490, 1495)\n",
      "idx:  1490  Epoch:  115  loss:  284.75018  loss_sent:  1.9512817  loss_word:  274.99374  Time cost:  6.511080026626587\n",
      "Start and end (1495, 1500)\n",
      "idx:  1495  Epoch:  115  loss:  185.07254  loss_sent:  1.6090246  loss_word:  177.02742  Time cost:  6.501619815826416\n",
      "Start and end (1500, 1505)\n",
      "idx:  1500  Epoch:  115  loss:  189.43259  loss_sent:  2.1860332  loss_word:  178.50246  Time cost:  6.5453855991363525\n",
      "Start and end (1505, 1510)\n",
      "idx:  1505  Epoch:  115  loss:  236.14647  loss_sent:  1.8885922  loss_word:  226.7035  Time cost:  6.564865350723267\n",
      "Start and end (1510, 1515)\n",
      "idx:  1510  Epoch:  115  loss:  248.8071  loss_sent:  1.843291  loss_word:  239.59064  Time cost:  6.3892786502838135\n",
      "Start and end (1515, 1520)\n",
      "idx:  1515  Epoch:  115  loss:  195.0854  loss_sent:  1.4785006  loss_word:  187.6929  Time cost:  6.41726541519165\n",
      "Start and end (1520, 1525)\n",
      "idx:  1520  Epoch:  115  loss:  277.8264  loss_sent:  2.0153532  loss_word:  267.7496  Time cost:  6.542381525039673\n",
      "Start and end (1525, 1530)\n",
      "idx:  1525  Epoch:  115  loss:  239.78136  loss_sent:  1.5100468  loss_word:  232.23112  Time cost:  6.4365339279174805\n",
      "Start and end (1530, 1535)\n",
      "idx:  1530  Epoch:  115  loss:  299.72775  loss_sent:  1.9755826  loss_word:  289.84985  Time cost:  6.465639591217041\n",
      "Start and end (1535, 1540)\n",
      "idx:  1535  Epoch:  115  loss:  217.00294  loss_sent:  1.788964  loss_word:  208.05812  Time cost:  6.556562662124634\n",
      "Start and end (1540, 1545)\n",
      "idx:  1540  Epoch:  115  loss:  164.61661  loss_sent:  2.0064871  loss_word:  154.5842  Time cost:  6.49594521522522\n",
      "Start and end (1545, 1550)\n",
      "idx:  1545  Epoch:  115  loss:  146.56807  loss_sent:  2.1235285  loss_word:  135.95044  Time cost:  6.479269027709961\n",
      "Start and end (1550, 1555)\n",
      "idx:  1550  Epoch:  115  loss:  365.12192  loss_sent:  2.8411574  loss_word:  350.91608  Time cost:  6.526962041854858\n",
      "Start and end (1555, 1560)\n",
      "idx:  1555  Epoch:  115  loss:  219.43834  loss_sent:  2.4911742  loss_word:  206.98247  Time cost:  6.621929168701172\n",
      "Start and end (1560, 1565)\n",
      "idx:  1560  Epoch:  115  loss:  224.97392  loss_sent:  2.561862  loss_word:  212.1646  Time cost:  6.4212400913238525\n",
      "Start and end (1565, 1570)\n",
      "idx:  1565  Epoch:  115  loss:  219.65222  loss_sent:  2.0073903  loss_word:  209.61526  Time cost:  6.5356738567352295\n",
      "Start and end (1570, 1575)\n",
      "idx:  1570  Epoch:  115  loss:  261.84583  loss_sent:  1.7560694  loss_word:  253.0655  Time cost:  6.518415212631226\n",
      "Start and end (1575, 1580)\n",
      "idx:  1575  Epoch:  115  loss:  213.23923  loss_sent:  3.6622474  loss_word:  194.92796  Time cost:  6.565244913101196\n",
      "Start and end (1580, 1585)\n",
      "idx:  1580  Epoch:  115  loss:  220.94691  loss_sent:  2.9689379  loss_word:  206.10222  Time cost:  6.582623243331909\n",
      "Start and end (1585, 1590)\n",
      "idx:  1585  Epoch:  115  loss:  194.87242  loss_sent:  1.8964044  loss_word:  185.3904  Time cost:  6.575523853302002\n",
      "Start and end (1590, 1595)\n",
      "idx:  1590  Epoch:  115  loss:  204.23343  loss_sent:  2.1184268  loss_word:  193.64128  Time cost:  6.526167154312134\n",
      "Start and end (1595, 1600)\n",
      "idx:  1595  Epoch:  115  loss:  163.332  loss_sent:  1.5613471  loss_word:  155.52527  Time cost:  6.405148506164551\n",
      "Start and end (1600, 1605)\n",
      "idx:  1600  Epoch:  115  loss:  254.12141  loss_sent:  2.237592  loss_word:  242.93344  Time cost:  6.442925214767456\n",
      "Start and end (1605, 1610)\n",
      "idx:  1605  Epoch:  115  loss:  298.8789  loss_sent:  1.7150656  loss_word:  290.30356  Time cost:  6.479973793029785\n",
      "Start and end (1610, 1615)\n",
      "idx:  1610  Epoch:  115  loss:  273.4272  loss_sent:  2.1497881  loss_word:  262.6782  Time cost:  6.495795726776123\n",
      "Start and end (1615, 1620)\n",
      "idx:  1615  Epoch:  115  loss:  243.29074  loss_sent:  2.3751795  loss_word:  231.41481  Time cost:  6.519948959350586\n",
      "Start and end (1620, 1625)\n",
      "idx:  1620  Epoch:  115  loss:  251.17332  loss_sent:  2.5868292  loss_word:  238.23918  Time cost:  6.544716119766235\n",
      "Start and end (1625, 1630)\n",
      "idx:  1625  Epoch:  115  loss:  281.31906  loss_sent:  1.9799087  loss_word:  271.41956  Time cost:  6.523552179336548\n",
      "Start and end (1630, 1635)\n",
      "idx:  1630  Epoch:  115  loss:  228.84305  loss_sent:  1.6601025  loss_word:  220.54251  Time cost:  6.533864259719849\n",
      "Start and end (1635, 1640)\n",
      "idx:  1635  Epoch:  115  loss:  338.45108  loss_sent:  2.4042678  loss_word:  326.42978  Time cost:  6.4638097286224365\n",
      "Start and end (1640, 1645)\n",
      "idx:  1640  Epoch:  115  loss:  271.4528  loss_sent:  1.534764  loss_word:  263.77902  Time cost:  6.56900954246521\n",
      "Start and end (1645, 1650)\n",
      "idx:  1645  Epoch:  115  loss:  233.55435  loss_sent:  2.6814344  loss_word:  220.14719  Time cost:  6.539283990859985\n",
      "Start and end (1650, 1655)\n",
      "idx:  1650  Epoch:  115  loss:  269.21072  loss_sent:  2.3930616  loss_word:  257.24542  Time cost:  6.45798659324646\n",
      "Start and end (1655, 1660)\n",
      "idx:  1655  Epoch:  115  loss:  189.93134  loss_sent:  3.293003  loss_word:  173.46631  Time cost:  6.465647459030151\n",
      "Start and end (1660, 1665)\n",
      "idx:  1660  Epoch:  115  loss:  229.21233  loss_sent:  2.0768576  loss_word:  218.82802  Time cost:  6.466371059417725\n",
      "Start and end (1665, 1670)\n",
      "idx:  1665  Epoch:  115  loss:  340.57434  loss_sent:  2.0497642  loss_word:  330.3255  Time cost:  6.531100511550903\n",
      "Start and end (1670, 1675)\n",
      "idx:  1670  Epoch:  115  loss:  218.47823  loss_sent:  1.5478271  loss_word:  210.7391  Time cost:  6.519826173782349\n",
      "Start and end (1675, 1680)\n",
      "idx:  1675  Epoch:  115  loss:  269.56  loss_sent:  1.9501237  loss_word:  259.8094  Time cost:  6.573817253112793\n",
      "Start and end (1680, 1685)\n",
      "idx:  1680  Epoch:  115  loss:  228.35312  loss_sent:  2.0040133  loss_word:  218.33307  Time cost:  6.529094696044922\n",
      "Start and end (1685, 1690)\n",
      "idx:  1685  Epoch:  115  loss:  267.97952  loss_sent:  2.4366713  loss_word:  255.7961  Time cost:  6.53377890586853\n",
      "Start and end (1690, 1695)\n",
      "idx:  1690  Epoch:  115  loss:  217.30243  loss_sent:  1.8735862  loss_word:  207.9345  Time cost:  6.44021463394165\n",
      "Start and end (1695, 1700)\n",
      "idx:  1695  Epoch:  115  loss:  293.05673  loss_sent:  2.5302665  loss_word:  280.40543  Time cost:  6.55928111076355\n",
      "Start and end (1700, 1705)\n",
      "idx:  1700  Epoch:  115  loss:  243.65314  loss_sent:  1.961386  loss_word:  233.84622  Time cost:  6.514214992523193\n",
      "Start and end (1705, 1710)\n",
      "idx:  1705  Epoch:  115  loss:  277.58453  loss_sent:  1.8653729  loss_word:  268.25766  Time cost:  6.541204214096069\n",
      "Start and end (1710, 1715)\n",
      "idx:  1710  Epoch:  115  loss:  297.73746  loss_sent:  1.9145228  loss_word:  288.1649  Time cost:  6.526838302612305\n",
      "Start and end (1715, 1720)\n",
      "idx:  1715  Epoch:  115  loss:  197.47191  loss_sent:  2.051477  loss_word:  187.21454  Time cost:  6.539011240005493\n",
      "Start and end (1720, 1725)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1720  Epoch:  115  loss:  235.7444  loss_sent:  2.37408  loss_word:  223.87398  Time cost:  6.526697397232056\n",
      "Start and end (1725, 1730)\n",
      "idx:  1725  Epoch:  115  loss:  296.78937  loss_sent:  2.088348  loss_word:  286.34766  Time cost:  6.57950496673584\n",
      "Start and end (1730, 1735)\n",
      "idx:  1730  Epoch:  115  loss:  252.71794  loss_sent:  1.6555738  loss_word:  244.44008  Time cost:  6.545632839202881\n",
      "Start and end (1735, 1740)\n",
      "idx:  1735  Epoch:  115  loss:  241.51262  loss_sent:  2.200041  loss_word:  230.51239  Time cost:  6.518685579299927\n",
      "Start and end (1740, 1745)\n",
      "idx:  1740  Epoch:  115  loss:  202.64272  loss_sent:  2.0387526  loss_word:  192.44894  Time cost:  6.5222368240356445\n",
      "Start and end (1745, 1750)\n",
      "idx:  1745  Epoch:  115  loss:  264.62445  loss_sent:  1.9806218  loss_word:  254.72136  Time cost:  6.586141109466553\n",
      "Start and end (1750, 1755)\n",
      "idx:  1750  Epoch:  115  loss:  238.78598  loss_sent:  1.5175191  loss_word:  231.19838  Time cost:  6.536270618438721\n",
      "Start and end (1755, 1760)\n",
      "idx:  1755  Epoch:  115  loss:  225.55278  loss_sent:  1.7064916  loss_word:  217.02034  Time cost:  6.569398880004883\n",
      "Start and end (1760, 1765)\n",
      "idx:  1760  Epoch:  115  loss:  268.32782  loss_sent:  1.7768695  loss_word:  259.44345  Time cost:  6.528681039810181\n",
      "Start and end (1765, 1770)\n",
      "idx:  1765  Epoch:  115  loss:  239.72382  loss_sent:  2.0331872  loss_word:  229.55789  Time cost:  6.558336019515991\n",
      "Start and end (1770, 1775)\n",
      "idx:  1770  Epoch:  115  loss:  225.49371  loss_sent:  2.4529667  loss_word:  213.22884  Time cost:  6.429803133010864\n",
      "Start and end (1775, 1780)\n",
      "idx:  1775  Epoch:  115  loss:  157.15825  loss_sent:  1.4375511  loss_word:  149.97049  Time cost:  6.531298637390137\n",
      "Start and end (1780, 1785)\n",
      "idx:  1780  Epoch:  115  loss:  190.49591  loss_sent:  1.8418561  loss_word:  181.28664  Time cost:  6.4934210777282715\n",
      "Start and end (1785, 1790)\n",
      "idx:  1785  Epoch:  115  loss:  237.91245  loss_sent:  1.450647  loss_word:  230.6592  Time cost:  6.585960865020752\n",
      "Start and end (1790, 1795)\n",
      "idx:  1790  Epoch:  115  loss:  256.14487  loss_sent:  2.1502829  loss_word:  245.39346  Time cost:  6.4630632400512695\n",
      "Start and end (1795, 1800)\n",
      "idx:  1795  Epoch:  115  loss:  203.61227  loss_sent:  1.6452984  loss_word:  195.38577  Time cost:  6.577221870422363\n",
      "Start and end (1800, 1805)\n",
      "idx:  1800  Epoch:  115  loss:  178.90479  loss_sent:  2.1029031  loss_word:  168.39026  Time cost:  6.500622034072876\n",
      "Start and end (1805, 1810)\n",
      "idx:  1805  Epoch:  115  loss:  168.8515  loss_sent:  1.5131297  loss_word:  161.28587  Time cost:  6.505731105804443\n",
      "Start and end (1810, 1815)\n",
      "idx:  1810  Epoch:  115  loss:  232.7968  loss_sent:  2.073738  loss_word:  222.4281  Time cost:  6.4883317947387695\n",
      "Start and end (1815, 1820)\n",
      "idx:  1815  Epoch:  115  loss:  237.94977  loss_sent:  2.9840682  loss_word:  223.02942  Time cost:  6.433526039123535\n",
      "Start and end (1820, 1825)\n",
      "idx:  1820  Epoch:  115  loss:  323.60474  loss_sent:  1.7831118  loss_word:  314.68918  Time cost:  6.5105884075164795\n",
      "Start and end (1825, 1830)\n",
      "idx:  1825  Epoch:  115  loss:  262.7429  loss_sent:  1.7607902  loss_word:  253.93892  Time cost:  6.505284547805786\n",
      "Start and end (1830, 1835)\n",
      "idx:  1830  Epoch:  115  loss:  228.49326  loss_sent:  2.2608998  loss_word:  217.18875  Time cost:  6.547008037567139\n",
      "Start and end (1835, 1840)\n",
      "idx:  1835  Epoch:  115  loss:  240.73932  loss_sent:  1.6347332  loss_word:  232.56566  Time cost:  6.504162073135376\n",
      "Start and end (1840, 1845)\n",
      "idx:  1840  Epoch:  115  loss:  225.27284  loss_sent:  1.8287276  loss_word:  216.12921  Time cost:  6.504319190979004\n",
      "Start and end (1845, 1850)\n",
      "idx:  1845  Epoch:  115  loss:  264.38205  loss_sent:  1.6176419  loss_word:  256.29385  Time cost:  6.519448518753052\n",
      "Start and end (1850, 1855)\n",
      "idx:  1850  Epoch:  115  loss:  188.56721  loss_sent:  1.8149673  loss_word:  179.49239  Time cost:  6.54068922996521\n",
      "Start and end (1855, 1860)\n",
      "idx:  1855  Epoch:  115  loss:  242.584  loss_sent:  1.5262474  loss_word:  234.95276  Time cost:  6.57442831993103\n",
      "Start and end (1860, 1865)\n",
      "idx:  1860  Epoch:  115  loss:  189.76434  loss_sent:  1.7163498  loss_word:  181.1826  Time cost:  6.5941948890686035\n",
      "Start and end (1865, 1870)\n",
      "idx:  1865  Epoch:  115  loss:  198.6443  loss_sent:  1.8187628  loss_word:  189.55049  Time cost:  6.510613679885864\n",
      "Start and end (1870, 1875)\n",
      "idx:  1870  Epoch:  115  loss:  298.23032  loss_sent:  1.9726688  loss_word:  288.36697  Time cost:  6.584215879440308\n",
      "Start and end (1875, 1880)\n",
      "idx:  1875  Epoch:  115  loss:  212.77963  loss_sent:  1.6373744  loss_word:  204.59277  Time cost:  6.523805141448975\n",
      "Start and end (1880, 1885)\n",
      "idx:  1880  Epoch:  115  loss:  180.5815  loss_sent:  1.8239151  loss_word:  171.4619  Time cost:  6.416460037231445\n",
      "Start and end (1885, 1890)\n",
      "idx:  1885  Epoch:  115  loss:  207.34161  loss_sent:  1.7560222  loss_word:  198.5615  Time cost:  6.446021318435669\n",
      "Start and end (1890, 1895)\n",
      "idx:  1890  Epoch:  115  loss:  267.65604  loss_sent:  1.9165558  loss_word:  258.07324  Time cost:  6.583370923995972\n",
      "Start and end (1895, 1900)\n",
      "idx:  1895  Epoch:  115  loss:  172.36703  loss_sent:  1.4732615  loss_word:  165.00073  Time cost:  6.510265588760376\n",
      "Start and end (1900, 1905)\n",
      "idx:  1900  Epoch:  115  loss:  197.20294  loss_sent:  1.543956  loss_word:  189.48317  Time cost:  6.562507390975952\n",
      "Start and end (1905, 1910)\n",
      "idx:  1905  Epoch:  115  loss:  224.67915  loss_sent:  1.776962  loss_word:  215.79436  Time cost:  6.421429872512817\n",
      "Start and end (1910, 1915)\n",
      "idx:  1910  Epoch:  115  loss:  275.55508  loss_sent:  2.6044006  loss_word:  262.53308  Time cost:  6.511494398117065\n",
      "Start and end (1915, 1920)\n",
      "idx:  1915  Epoch:  115  loss:  305.4099  loss_sent:  1.8767781  loss_word:  296.02603  Time cost:  6.591240167617798\n",
      "Start and end (1920, 1925)\n",
      "idx:  1920  Epoch:  115  loss:  199.28445  loss_sent:  1.8154336  loss_word:  190.20728  Time cost:  6.472646236419678\n",
      "Start and end (1925, 1930)\n",
      "idx:  1925  Epoch:  115  loss:  183.0667  loss_sent:  1.5402414  loss_word:  175.36548  Time cost:  6.50495457649231\n",
      "Start and end (1930, 1935)\n",
      "idx:  1930  Epoch:  115  loss:  261.83557  loss_sent:  2.9305346  loss_word:  247.18289  Time cost:  6.56607985496521\n",
      "Start and end (1935, 1940)\n",
      "idx:  1935  Epoch:  115  loss:  258.44626  loss_sent:  1.9175642  loss_word:  248.85843  Time cost:  6.579526901245117\n",
      "Start and end (1940, 1945)\n",
      "idx:  1940  Epoch:  115  loss:  275.6533  loss_sent:  2.2063096  loss_word:  264.62173  Time cost:  6.484130382537842\n",
      "Start and end (1945, 1950)\n",
      "idx:  1945  Epoch:  115  loss:  203.50137  loss_sent:  2.6290243  loss_word:  190.35626  Time cost:  6.463500261306763\n",
      "Start and end (1950, 1955)\n",
      "idx:  1950  Epoch:  115  loss:  271.57175  loss_sent:  1.5277959  loss_word:  263.9328  Time cost:  6.526364803314209\n",
      "Start and end (1955, 1960)\n",
      "idx:  1955  Epoch:  115  loss:  314.2446  loss_sent:  1.4291086  loss_word:  307.09903  Time cost:  6.574841737747192\n",
      "Start and end (1960, 1965)\n",
      "idx:  1960  Epoch:  115  loss:  254.26384  loss_sent:  1.5655813  loss_word:  246.43593  Time cost:  6.437137126922607\n",
      "Start and end (1965, 1970)\n",
      "idx:  1965  Epoch:  115  loss:  319.92664  loss_sent:  1.7604736  loss_word:  311.12427  Time cost:  6.542396545410156\n",
      "Start and end (1970, 1975)\n",
      "idx:  1970  Epoch:  115  loss:  179.36179  loss_sent:  2.0480952  loss_word:  169.12129  Time cost:  6.49462366104126\n",
      "Start and end (1975, 1980)\n",
      "idx:  1975  Epoch:  115  loss:  269.51636  loss_sent:  1.9554667  loss_word:  259.739  Time cost:  6.479446887969971\n",
      "Start and end (1980, 1985)\n",
      "idx:  1980  Epoch:  115  loss:  251.26671  loss_sent:  3.0930376  loss_word:  235.80153  Time cost:  6.547316074371338\n",
      "Start and end (1985, 1990)\n",
      "idx:  1985  Epoch:  115  loss:  246.20277  loss_sent:  1.5865123  loss_word:  238.27022  Time cost:  6.535287141799927\n",
      "Start and end (1990, 1995)\n",
      "idx:  1990  Epoch:  115  loss:  339.505  loss_sent:  1.8312954  loss_word:  330.3485  Time cost:  6.490917682647705\n",
      "Start and end (1995, 2000)\n",
      "idx:  1995  Epoch:  115  loss:  239.98152  loss_sent:  2.097611  loss_word:  229.49348  Time cost:  6.512807369232178\n",
      "Start and end (2000, 2005)\n",
      "idx:  2000  Epoch:  115  loss:  285.4537  loss_sent:  1.9655814  loss_word:  275.62582  Time cost:  6.544353246688843\n",
      "Start and end (2005, 2010)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2005  Epoch:  115  loss:  218.5558  loss_sent:  2.0246706  loss_word:  208.43242  Time cost:  6.446069955825806\n",
      "Start and end (2010, 2015)\n",
      "idx:  2010  Epoch:  115  loss:  251.82864  loss_sent:  1.9265916  loss_word:  242.19571  Time cost:  6.560038328170776\n",
      "Start and end (2015, 2020)\n",
      "idx:  2015  Epoch:  115  loss:  214.91997  loss_sent:  1.568485  loss_word:  207.07753  Time cost:  6.5617899894714355\n",
      "Start and end (2020, 2025)\n",
      "idx:  2020  Epoch:  115  loss:  220.8105  loss_sent:  1.84816  loss_word:  211.56972  Time cost:  6.567040920257568\n",
      "Start and end (2025, 2030)\n",
      "idx:  2025  Epoch:  115  loss:  301.48276  loss_sent:  1.3959672  loss_word:  294.50287  Time cost:  6.5800440311431885\n",
      "Start and end (2030, 2035)\n",
      "idx:  2030  Epoch:  115  loss:  201.01833  loss_sent:  2.3664014  loss_word:  189.1863  Time cost:  6.528951168060303\n",
      "Start and end (2035, 2040)\n",
      "idx:  2035  Epoch:  115  loss:  242.8575  loss_sent:  1.751348  loss_word:  234.10077  Time cost:  6.444778919219971\n",
      "Start and end (2040, 2045)\n",
      "idx:  2040  Epoch:  115  loss:  257.42484  loss_sent:  1.5794401  loss_word:  249.5276  Time cost:  6.5523176193237305\n",
      "Start and end (2045, 2050)\n",
      "idx:  2045  Epoch:  115  loss:  271.30222  loss_sent:  2.0187452  loss_word:  261.20847  Time cost:  6.5385048389434814\n",
      "Start and end (2050, 2055)\n",
      "idx:  2050  Epoch:  115  loss:  207.01706  loss_sent:  1.9198462  loss_word:  197.41782  Time cost:  6.531591892242432\n",
      "Start and end (2055, 2060)\n",
      "idx:  2055  Epoch:  115  loss:  235.63249  loss_sent:  1.5104793  loss_word:  228.08006  Time cost:  6.410721302032471\n",
      "Start and end (2060, 2065)\n",
      "idx:  2060  Epoch:  115  loss:  218.60593  loss_sent:  2.0493574  loss_word:  208.35913  Time cost:  6.562927722930908\n",
      "Start and end (2065, 2070)\n",
      "idx:  2065  Epoch:  115  loss:  213.23923  loss_sent:  1.8026295  loss_word:  204.22609  Time cost:  6.50187087059021\n",
      "Start and end (2070, 2075)\n",
      "idx:  2070  Epoch:  115  loss:  225.66339  loss_sent:  1.875057  loss_word:  216.28812  Time cost:  6.521726608276367\n",
      "Start and end (2075, 2080)\n",
      "idx:  2075  Epoch:  115  loss:  280.65363  loss_sent:  1.8829662  loss_word:  271.2388  Time cost:  6.554128885269165\n",
      "Start and end (2080, 2085)\n",
      "idx:  2080  Epoch:  115  loss:  220.47083  loss_sent:  1.9507178  loss_word:  210.71727  Time cost:  6.45975136756897\n",
      "Start and end (2085, 2090)\n",
      "idx:  2085  Epoch:  115  loss:  188.96164  loss_sent:  1.6491163  loss_word:  180.71605  Time cost:  6.512714624404907\n",
      "Start and end (2090, 2095)\n",
      "idx:  2090  Epoch:  115  loss:  262.82446  loss_sent:  1.9706973  loss_word:  252.97095  Time cost:  6.581871271133423\n",
      "Start and end (2095, 2100)\n",
      "idx:  2095  Epoch:  115  loss:  207.70299  loss_sent:  1.5644717  loss_word:  199.88063  Time cost:  6.534542083740234\n",
      "Start and end (2100, 2105)\n",
      "idx:  2100  Epoch:  115  loss:  178.11324  loss_sent:  2.6219656  loss_word:  165.0034  Time cost:  6.519725322723389\n",
      "Start and end (2105, 2110)\n",
      "idx:  2105  Epoch:  115  loss:  290.92807  loss_sent:  2.450072  loss_word:  278.67767  Time cost:  6.528656482696533\n",
      "Start and end (2110, 2115)\n",
      "idx:  2110  Epoch:  115  loss:  309.66104  loss_sent:  1.9307688  loss_word:  300.0072  Time cost:  6.531890630722046\n",
      "Start and end (2115, 2120)\n",
      "idx:  2115  Epoch:  115  loss:  250.19089  loss_sent:  2.0242143  loss_word:  240.06982  Time cost:  6.48199200630188\n",
      "Start and end (2120, 2125)\n",
      "idx:  2120  Epoch:  115  loss:  270.92487  loss_sent:  1.6740187  loss_word:  262.5548  Time cost:  6.528852462768555\n",
      "Start and end (2125, 2130)\n",
      "idx:  2125  Epoch:  115  loss:  241.91286  loss_sent:  2.9845679  loss_word:  226.99005  Time cost:  6.582388877868652\n",
      "Start and end (2130, 2135)\n",
      "idx:  2130  Epoch:  115  loss:  128.28944  loss_sent:  1.7745891  loss_word:  119.41651  Time cost:  6.491698265075684\n",
      "Start and end (2135, 2140)\n",
      "idx:  2135  Epoch:  115  loss:  164.4817  loss_sent:  1.5862999  loss_word:  156.5502  Time cost:  6.546443700790405\n",
      "Start and end (2140, 2145)\n",
      "idx:  2140  Epoch:  115  loss:  254.79187  loss_sent:  2.1405292  loss_word:  244.0892  Time cost:  6.52614688873291\n",
      "Start and end (2145, 2150)\n",
      "idx:  2145  Epoch:  115  loss:  307.86023  loss_sent:  1.8203235  loss_word:  298.75864  Time cost:  6.539881229400635\n",
      "Start and end (2150, 2155)\n",
      "idx:  2150  Epoch:  115  loss:  249.91414  loss_sent:  1.9435146  loss_word:  240.1966  Time cost:  6.471591949462891\n",
      "Start and end (2155, 2160)\n",
      "idx:  2155  Epoch:  115  loss:  257.8408  loss_sent:  1.7060399  loss_word:  249.31055  Time cost:  6.3863818645477295\n",
      "Start and end (2160, 2165)\n",
      "idx:  2160  Epoch:  115  loss:  170.8595  loss_sent:  2.3347492  loss_word:  159.18576  Time cost:  6.4646313190460205\n",
      "Start and end (2165, 2170)\n",
      "idx:  2165  Epoch:  115  loss:  269.89838  loss_sent:  2.3015287  loss_word:  258.39075  Time cost:  6.516014099121094\n",
      "Start and end (2170, 2175)\n",
      "idx:  2170  Epoch:  115  loss:  240.01868  loss_sent:  2.1454635  loss_word:  229.29135  Time cost:  6.441006898880005\n",
      "Start and end (2175, 2180)\n",
      "idx:  2175  Epoch:  115  loss:  242.30287  loss_sent:  1.7528651  loss_word:  233.53856  Time cost:  6.677589178085327\n",
      "Start and end (2180, 2185)\n",
      "idx:  2180  Epoch:  115  loss:  257.27423  loss_sent:  2.2044334  loss_word:  246.25209  Time cost:  6.8015711307525635\n",
      "Start and end (2185, 2190)\n",
      "idx:  2185  Epoch:  115  loss:  270.11154  loss_sent:  1.53041  loss_word:  262.4595  Time cost:  6.6614439487457275\n",
      "Start and end (2190, 2195)\n",
      "idx:  2190  Epoch:  115  loss:  300.54498  loss_sent:  1.5239182  loss_word:  292.9254  Time cost:  6.742672443389893\n",
      "Start and end (2195, 2200)\n",
      "idx:  2195  Epoch:  115  loss:  245.96004  loss_sent:  1.5867196  loss_word:  238.02641  Time cost:  6.628371715545654\n",
      "Start and end (2200, 2205)\n",
      "idx:  2200  Epoch:  115  loss:  186.18475  loss_sent:  2.1353245  loss_word:  175.50815  Time cost:  6.506417751312256\n",
      "Start and end (2205, 2210)\n",
      "idx:  2205  Epoch:  115  loss:  220.21591  loss_sent:  1.3952987  loss_word:  213.23941  Time cost:  6.668653249740601\n",
      "Start and end (2210, 2215)\n",
      "idx:  2210  Epoch:  115  loss:  249.86053  loss_sent:  2.1519165  loss_word:  239.10095  Time cost:  7.160218000411987\n",
      "Start and end (2215, 2220)\n",
      "idx:  2215  Epoch:  115  loss:  253.96295  loss_sent:  2.1427367  loss_word:  243.24927  Time cost:  6.8588995933532715\n",
      "Start and end (2220, 2225)\n",
      "idx:  2220  Epoch:  115  loss:  239.94412  loss_sent:  1.9596491  loss_word:  230.14587  Time cost:  6.586652994155884\n",
      "Start and end (2225, 2230)\n",
      "idx:  2225  Epoch:  115  loss:  143.71104  loss_sent:  1.322008  loss_word:  137.10103  Time cost:  6.610874176025391\n",
      "Start and end (2230, 2235)\n",
      "idx:  2230  Epoch:  115  loss:  187.62906  loss_sent:  1.424039  loss_word:  180.50885  Time cost:  6.618481159210205\n",
      "Start and end (2235, 2240)\n",
      "idx:  2235  Epoch:  115  loss:  258.4454  loss_sent:  2.2905595  loss_word:  246.99261  Time cost:  7.059225082397461\n",
      "Start and end (2240, 2245)\n",
      "idx:  2240  Epoch:  115  loss:  174.03421  loss_sent:  1.4858513  loss_word:  166.60497  Time cost:  6.732313871383667\n",
      "Start and end (2245, 2250)\n",
      "idx:  2245  Epoch:  115  loss:  214.61101  loss_sent:  2.6006813  loss_word:  201.60757  Time cost:  6.747045278549194\n",
      "Start and end (2250, 2255)\n",
      "idx:  2250  Epoch:  115  loss:  259.89468  loss_sent:  2.1285539  loss_word:  249.25189  Time cost:  6.635167121887207\n",
      "Start and end (2255, 2260)\n",
      "idx:  2255  Epoch:  115  loss:  180.75766  loss_sent:  2.3379982  loss_word:  169.06766  Time cost:  6.4590747356414795\n",
      "Start and end (2260, 2265)\n",
      "idx:  2260  Epoch:  115  loss:  244.27753  loss_sent:  2.6561904  loss_word:  230.99657  Time cost:  6.5901288986206055\n",
      "Start and end (2265, 2270)\n",
      "idx:  2265  Epoch:  115  loss:  170.7945  loss_sent:  1.7137644  loss_word:  162.22568  Time cost:  6.575327634811401\n",
      "Start and end (2270, 2275)\n",
      "idx:  2270  Epoch:  115  loss:  209.87878  loss_sent:  1.6238042  loss_word:  201.75977  Time cost:  6.93695592880249\n",
      "Start and end (2275, 2280)\n",
      "idx:  2275  Epoch:  115  loss:  208.84474  loss_sent:  1.8417723  loss_word:  199.63585  Time cost:  6.617794990539551\n",
      "Start and end (2280, 2285)\n",
      "idx:  2280  Epoch:  115  loss:  269.0658  loss_sent:  1.9074751  loss_word:  259.52844  Time cost:  6.856746196746826\n",
      "Start and end (2285, 2290)\n",
      "idx:  2285  Epoch:  115  loss:  188.29135  loss_sent:  1.4695103  loss_word:  180.94379  Time cost:  7.0038697719573975\n",
      "Start and end (2290, 2295)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2290  Epoch:  115  loss:  286.65564  loss_sent:  2.1584187  loss_word:  275.86353  Time cost:  7.011322975158691\n",
      "Start and end (2295, 2300)\n",
      "idx:  2295  Epoch:  115  loss:  214.82559  loss_sent:  1.9687979  loss_word:  204.98161  Time cost:  6.668293476104736\n",
      "Start and end (2300, 2305)\n",
      "idx:  2300  Epoch:  115  loss:  229.82474  loss_sent:  2.5597181  loss_word:  217.02614  Time cost:  6.835778474807739\n",
      "Start and end (2305, 2310)\n",
      "idx:  2305  Epoch:  115  loss:  278.28897  loss_sent:  1.5750645  loss_word:  270.41367  Time cost:  6.6488142013549805\n",
      "Start and end (2310, 2315)\n",
      "idx:  2310  Epoch:  115  loss:  257.2102  loss_sent:  2.1851792  loss_word:  246.28429  Time cost:  6.523748159408569\n",
      "Start and end (2315, 2320)\n",
      "idx:  2315  Epoch:  115  loss:  284.06726  loss_sent:  1.8730042  loss_word:  274.7022  Time cost:  6.595515966415405\n",
      "Start and end (2320, 2325)\n",
      "idx:  2320  Epoch:  115  loss:  244.36913  loss_sent:  1.9575689  loss_word:  234.58128  Time cost:  6.668626070022583\n",
      "Start and end (2325, 2330)\n",
      "idx:  2325  Epoch:  115  loss:  184.54189  loss_sent:  1.8490263  loss_word:  175.29675  Time cost:  7.103030681610107\n",
      "Start and end (2330, 2335)\n",
      "idx:  2330  Epoch:  115  loss:  166.40102  loss_sent:  1.6501431  loss_word:  158.15031  Time cost:  7.460940599441528\n",
      "Start and end (2335, 2340)\n",
      "idx:  2335  Epoch:  115  loss:  336.2323  loss_sent:  2.333724  loss_word:  324.56366  Time cost:  7.087215423583984\n",
      "Start and end (2340, 2345)\n",
      "idx:  2340  Epoch:  115  loss:  152.8935  loss_sent:  2.351968  loss_word:  141.13367  Time cost:  6.786519765853882\n",
      "Start and end (2345, 2350)\n",
      "idx:  2345  Epoch:  115  loss:  199.81665  loss_sent:  1.8230146  loss_word:  190.70158  Time cost:  6.989425897598267\n",
      "Start and end (2350, 2355)\n",
      "idx:  2350  Epoch:  115  loss:  257.7493  loss_sent:  2.0222344  loss_word:  247.63812  Time cost:  7.056737422943115\n",
      "Start and end (2355, 2360)\n",
      "idx:  2355  Epoch:  115  loss:  280.2369  loss_sent:  2.1301088  loss_word:  269.58643  Time cost:  7.161539793014526\n",
      "Start and end (2360, 2365)\n",
      "idx:  2360  Epoch:  115  loss:  210.75194  loss_sent:  1.9191773  loss_word:  201.15605  Time cost:  7.267984628677368\n",
      "Start and end (2365, 2370)\n",
      "idx:  2365  Epoch:  115  loss:  244.56647  loss_sent:  1.4410806  loss_word:  237.36105  Time cost:  6.9369261264801025\n",
      "Start and end (2370, 2375)\n",
      "idx:  2370  Epoch:  115  loss:  276.3362  loss_sent:  1.9778172  loss_word:  266.4471  Time cost:  7.326913356781006\n",
      "Start and end (2375, 2380)\n",
      "idx:  2375  Epoch:  115  loss:  241.95197  loss_sent:  1.725112  loss_word:  233.32642  Time cost:  7.120200872421265\n",
      "Start and end (2380, 2385)\n",
      "idx:  2380  Epoch:  115  loss:  251.42456  loss_sent:  2.0699964  loss_word:  241.0746  Time cost:  7.058667421340942\n",
      "Start and end (2385, 2390)\n",
      "idx:  2385  Epoch:  115  loss:  174.55203  loss_sent:  2.439528  loss_word:  162.35439  Time cost:  7.169817924499512\n",
      "Start and end (2390, 2395)\n",
      "idx:  2390  Epoch:  115  loss:  231.98448  loss_sent:  2.3951626  loss_word:  220.00868  Time cost:  7.1212780475616455\n",
      "Start and end (2395, 2400)\n",
      "idx:  2395  Epoch:  115  loss:  233.34201  loss_sent:  1.4423242  loss_word:  226.13037  Time cost:  7.2069337368011475\n",
      "Start and end (2400, 2405)\n",
      "idx:  2400  Epoch:  115  loss:  242.05612  loss_sent:  1.5982575  loss_word:  234.06482  Time cost:  7.144119501113892\n",
      "Start and end (2405, 2410)\n",
      "idx:  2405  Epoch:  115  loss:  191.30206  loss_sent:  1.3362191  loss_word:  184.62096  Time cost:  6.800078392028809\n",
      "Start and end (2410, 2415)\n",
      "idx:  2410  Epoch:  115  loss:  177.53094  loss_sent:  2.5931342  loss_word:  164.56523  Time cost:  7.01800012588501\n",
      "Start and end (2415, 2420)\n",
      "idx:  2415  Epoch:  115  loss:  230.1808  loss_sent:  2.5544174  loss_word:  217.40869  Time cost:  6.771498441696167\n",
      "Start and end (2420, 2425)\n",
      "idx:  2420  Epoch:  115  loss:  242.9431  loss_sent:  1.401936  loss_word:  235.93341  Time cost:  6.712386131286621\n",
      "Start and end (2425, 2430)\n",
      "idx:  2425  Epoch:  115  loss:  281.0254  loss_sent:  1.6107018  loss_word:  272.97195  Time cost:  6.5787131786346436\n",
      "Start and end (2430, 2435)\n",
      "idx:  2430  Epoch:  115  loss:  249.8392  loss_sent:  2.5396419  loss_word:  237.14099  Time cost:  6.826478958129883\n",
      "Start and end (2435, 2440)\n",
      "idx:  2435  Epoch:  115  loss:  248.28433  loss_sent:  2.3527093  loss_word:  236.52078  Time cost:  6.796571731567383\n",
      "Start and end (2440, 2445)\n",
      "idx:  2440  Epoch:  115  loss:  220.82799  loss_sent:  1.7749324  loss_word:  211.95332  Time cost:  6.865709543228149\n",
      "Start and end (2445, 2450)\n",
      "idx:  2445  Epoch:  115  loss:  288.60217  loss_sent:  1.8046526  loss_word:  279.57895  Time cost:  7.0003981590271\n",
      "Start and end (2450, 2455)\n",
      "idx:  2450  Epoch:  115  loss:  257.048  loss_sent:  2.4233732  loss_word:  244.93117  Time cost:  6.800175666809082\n",
      "Start and end (2455, 2460)\n",
      "idx:  2455  Epoch:  115  loss:  300.8381  loss_sent:  1.5493135  loss_word:  293.09152  Time cost:  6.771198034286499\n",
      "Start and end (2460, 2465)\n",
      "idx:  2460  Epoch:  115  loss:  172.07689  loss_sent:  2.331681  loss_word:  160.41849  Time cost:  6.910980463027954\n",
      "Start and end (2465, 2470)\n",
      "idx:  2465  Epoch:  115  loss:  277.66412  loss_sent:  1.5452096  loss_word:  269.93805  Time cost:  6.951048851013184\n",
      "Start and end (2470, 2475)\n",
      "idx:  2470  Epoch:  115  loss:  248.80734  loss_sent:  2.4431794  loss_word:  236.59143  Time cost:  6.679713487625122\n",
      "Start and end (2475, 2480)\n",
      "idx:  2475  Epoch:  115  loss:  260.40503  loss_sent:  1.643305  loss_word:  252.18854  Time cost:  6.710158586502075\n",
      "Start and end (2480, 2485)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0724 11:58:59.870782 140197742454528 saver.py:1134] *******************************************************\n",
      "W0724 11:58:59.871258 140197742454528 saver.py:1135] TensorFlow's V1 checkpoint format has been deprecated.\n",
      "W0724 11:58:59.871570 140197742454528 saver.py:1136] Consider switching to the more efficient V2 format:\n",
      "W0724 11:58:59.871879 140197742454528 saver.py:1137]    `tf.train.Saver(write_version=tf.train.SaverDef.V2)`\n",
      "W0724 11:58:59.872207 140197742454528 saver.py:1138] now on by default.\n",
      "W0724 11:58:59.872504 140197742454528 saver.py:1139] *******************************************************\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2480  Epoch:  115  loss:  274.12943  loss_sent:  3.4450078  loss_word:  256.9043  Time cost:  6.557159900665283\n",
      "Successfully Written to temporary\n",
      "Epoch  115  is done. Saving the model ...\n",
      "Start and end (0, 5)\n",
      "idx:  0  Epoch:  116  loss:  214.57721  loss_sent:  1.4643154  loss_word:  207.25565  Time cost:  6.5497119426727295\n",
      "Start and end (5, 10)\n",
      "idx:  5  Epoch:  116  loss:  226.42267  loss_sent:  1.5222309  loss_word:  218.81151  Time cost:  6.793993949890137\n",
      "Start and end (10, 15)\n",
      "idx:  10  Epoch:  116  loss:  236.79851  loss_sent:  1.5173012  loss_word:  229.21199  Time cost:  6.813993453979492\n",
      "Start and end (15, 20)\n",
      "idx:  15  Epoch:  116  loss:  248.33698  loss_sent:  1.5474281  loss_word:  240.59984  Time cost:  6.807343244552612\n",
      "Start and end (20, 25)\n",
      "idx:  20  Epoch:  116  loss:  276.98434  loss_sent:  1.3671223  loss_word:  270.1487  Time cost:  6.736278772354126\n",
      "Start and end (25, 30)\n",
      "idx:  25  Epoch:  116  loss:  165.45421  loss_sent:  2.9490829  loss_word:  150.70883  Time cost:  7.016906023025513\n",
      "Start and end (30, 35)\n",
      "idx:  30  Epoch:  116  loss:  272.16605  loss_sent:  1.7914306  loss_word:  263.2088  Time cost:  6.904466867446899\n",
      "Start and end (35, 40)\n",
      "idx:  35  Epoch:  116  loss:  224.42833  loss_sent:  1.4297652  loss_word:  217.2795  Time cost:  6.9619786739349365\n",
      "Start and end (40, 45)\n",
      "idx:  40  Epoch:  116  loss:  173.7355  loss_sent:  1.2301853  loss_word:  167.58458  Time cost:  7.187245845794678\n",
      "Start and end (45, 50)\n",
      "idx:  45  Epoch:  116  loss:  222.20163  loss_sent:  1.7135435  loss_word:  213.63391  Time cost:  6.960906982421875\n",
      "Start and end (50, 55)\n",
      "idx:  50  Epoch:  116  loss:  190.44672  loss_sent:  2.2284966  loss_word:  179.30424  Time cost:  7.107164144515991\n",
      "Start and end (55, 60)\n",
      "idx:  55  Epoch:  116  loss:  209.75258  loss_sent:  2.8155522  loss_word:  195.67482  Time cost:  7.1226890087127686\n",
      "Start and end (60, 65)\n",
      "idx:  60  Epoch:  116  loss:  261.317  loss_sent:  1.3238914  loss_word:  254.69753  Time cost:  6.644665718078613\n",
      "Start and end (65, 70)\n",
      "idx:  65  Epoch:  116  loss:  252.65166  loss_sent:  1.8055198  loss_word:  243.62407  Time cost:  6.7714409828186035\n",
      "Start and end (70, 75)\n",
      "idx:  70  Epoch:  116  loss:  235.27306  loss_sent:  2.41974  loss_word:  223.17436  Time cost:  6.685769557952881\n",
      "Start and end (75, 80)\n",
      "idx:  75  Epoch:  116  loss:  218.00119  loss_sent:  1.2365044  loss_word:  211.81868  Time cost:  6.688208103179932\n",
      "Start and end (80, 85)\n",
      "idx:  80  Epoch:  116  loss:  233.74731  loss_sent:  1.5840054  loss_word:  225.8273  Time cost:  7.047340154647827\n",
      "Start and end (85, 90)\n",
      "idx:  85  Epoch:  116  loss:  225.96458  loss_sent:  1.5269159  loss_word:  218.33  Time cost:  7.146918296813965\n",
      "Start and end (90, 95)\n",
      "idx:  90  Epoch:  116  loss:  218.83987  loss_sent:  1.5625677  loss_word:  211.02705  Time cost:  6.941406488418579\n",
      "Start and end (95, 100)\n",
      "idx:  95  Epoch:  116  loss:  260.5954  loss_sent:  1.9916301  loss_word:  250.63725  Time cost:  6.656237363815308\n",
      "Start and end (100, 105)\n",
      "idx:  100  Epoch:  116  loss:  242.39078  loss_sent:  1.3167756  loss_word:  235.80688  Time cost:  6.79493522644043\n",
      "Start and end (105, 110)\n",
      "idx:  105  Epoch:  116  loss:  209.76997  loss_sent:  1.6411402  loss_word:  201.56427  Time cost:  6.846845865249634\n",
      "Start and end (110, 115)\n",
      "idx:  110  Epoch:  116  loss:  276.80795  loss_sent:  1.3371824  loss_word:  270.12204  Time cost:  7.069379806518555\n",
      "Start and end (115, 120)\n",
      "idx:  115  Epoch:  116  loss:  263.32346  loss_sent:  1.6658062  loss_word:  254.9944  Time cost:  7.119920253753662\n",
      "Start and end (120, 125)\n",
      "idx:  120  Epoch:  116  loss:  307.9237  loss_sent:  2.0333195  loss_word:  297.7571  Time cost:  7.069389581680298\n",
      "Start and end (125, 130)\n",
      "idx:  125  Epoch:  116  loss:  251.23839  loss_sent:  1.9216343  loss_word:  241.63022  Time cost:  6.601571083068848\n",
      "Start and end (130, 135)\n",
      "idx:  130  Epoch:  116  loss:  256.6221  loss_sent:  2.4099355  loss_word:  244.57242  Time cost:  6.747812986373901\n",
      "Start and end (135, 140)\n",
      "idx:  135  Epoch:  116  loss:  278.2471  loss_sent:  1.5954816  loss_word:  270.2697  Time cost:  6.551982402801514\n",
      "Start and end (140, 145)\n",
      "idx:  140  Epoch:  116  loss:  234.65378  loss_sent:  1.2372947  loss_word:  228.46732  Time cost:  6.519570589065552\n",
      "Start and end (145, 150)\n",
      "idx:  145  Epoch:  116  loss:  145.20274  loss_sent:  1.3667876  loss_word:  138.36884  Time cost:  6.616430282592773\n",
      "Start and end (150, 155)\n",
      "idx:  150  Epoch:  116  loss:  242.30518  loss_sent:  2.1059241  loss_word:  231.77556  Time cost:  6.855592489242554\n",
      "Start and end (155, 160)\n",
      "idx:  155  Epoch:  116  loss:  201.17914  loss_sent:  2.0610027  loss_word:  190.87411  Time cost:  6.8197503089904785\n",
      "Start and end (160, 165)\n",
      "idx:  160  Epoch:  116  loss:  309.53793  loss_sent:  2.0672956  loss_word:  299.20148  Time cost:  7.684532880783081\n",
      "Start and end (165, 170)\n",
      "idx:  165  Epoch:  116  loss:  195.12505  loss_sent:  1.2194458  loss_word:  189.02782  Time cost:  6.91046667098999\n",
      "Start and end (170, 175)\n",
      "idx:  170  Epoch:  116  loss:  400.2516  loss_sent:  1.6314312  loss_word:  392.09448  Time cost:  6.923160791397095\n",
      "Start and end (175, 180)\n",
      "idx:  175  Epoch:  116  loss:  256.636  loss_sent:  2.034966  loss_word:  246.46123  Time cost:  6.864031791687012\n",
      "Start and end (180, 185)\n",
      "idx:  180  Epoch:  116  loss:  274.05017  loss_sent:  1.2517529  loss_word:  267.7914  Time cost:  6.956631660461426\n",
      "Start and end (185, 190)\n",
      "idx:  185  Epoch:  116  loss:  178.92986  loss_sent:  1.794312  loss_word:  169.95831  Time cost:  6.649618864059448\n",
      "Start and end (190, 195)\n",
      "idx:  190  Epoch:  116  loss:  223.4562  loss_sent:  1.7323172  loss_word:  214.79462  Time cost:  6.8790504932403564\n",
      "Start and end (195, 200)\n",
      "idx:  195  Epoch:  116  loss:  180.76945  loss_sent:  2.05928  loss_word:  170.47307  Time cost:  7.001638889312744\n",
      "Start and end (200, 205)\n",
      "idx:  200  Epoch:  116  loss:  219.43312  loss_sent:  1.3107932  loss_word:  212.87915  Time cost:  6.770519256591797\n",
      "Start and end (205, 210)\n",
      "idx:  205  Epoch:  116  loss:  291.52087  loss_sent:  1.2530779  loss_word:  285.2555  Time cost:  6.782202959060669\n",
      "Start and end (210, 215)\n",
      "idx:  210  Epoch:  116  loss:  271.99506  loss_sent:  1.276468  loss_word:  265.61273  Time cost:  6.817255973815918\n",
      "Start and end (215, 220)\n",
      "idx:  215  Epoch:  116  loss:  149.88289  loss_sent:  1.4485729  loss_word:  142.64003  Time cost:  6.8281333446502686\n",
      "Start and end (220, 225)\n",
      "idx:  220  Epoch:  116  loss:  235.00641  loss_sent:  2.0627692  loss_word:  224.69257  Time cost:  6.689146518707275\n",
      "Start and end (225, 230)\n",
      "idx:  225  Epoch:  116  loss:  230.1598  loss_sent:  1.7096808  loss_word:  221.6114  Time cost:  6.780617952346802\n",
      "Start and end (230, 235)\n",
      "idx:  230  Epoch:  116  loss:  274.70398  loss_sent:  1.6308441  loss_word:  266.54974  Time cost:  7.062026023864746\n",
      "Start and end (235, 240)\n",
      "idx:  235  Epoch:  116  loss:  251.49422  loss_sent:  1.1258264  loss_word:  245.86508  Time cost:  7.059441804885864\n",
      "Start and end (240, 245)\n",
      "idx:  240  Epoch:  116  loss:  243.64268  loss_sent:  2.203407  loss_word:  232.62563  Time cost:  7.008103847503662\n",
      "Start and end (245, 250)\n",
      "idx:  245  Epoch:  116  loss:  231.57797  loss_sent:  1.6465409  loss_word:  223.34528  Time cost:  6.774128198623657\n",
      "Start and end (250, 255)\n",
      "idx:  250  Epoch:  116  loss:  212.7947  loss_sent:  1.9886848  loss_word:  202.85129  Time cost:  7.037268161773682\n",
      "Start and end (255, 260)\n",
      "idx:  255  Epoch:  116  loss:  222.60341  loss_sent:  1.9204383  loss_word:  213.0012  Time cost:  6.925984859466553\n",
      "Start and end (260, 265)\n",
      "idx:  260  Epoch:  116  loss:  145.72517  loss_sent:  1.8944598  loss_word:  136.25287  Time cost:  6.964893579483032\n",
      "Start and end (265, 270)\n",
      "idx:  265  Epoch:  116  loss:  291.49716  loss_sent:  1.4023099  loss_word:  284.4856  Time cost:  6.774552583694458\n",
      "Start and end (270, 275)\n",
      "idx:  270  Epoch:  116  loss:  203.25093  loss_sent:  1.0956988  loss_word:  197.77243  Time cost:  6.615016222000122\n",
      "Start and end (275, 280)\n",
      "idx:  275  Epoch:  116  loss:  232.37886  loss_sent:  1.5564791  loss_word:  224.59647  Time cost:  6.669080018997192\n",
      "Start and end (280, 285)\n",
      "idx:  280  Epoch:  116  loss:  177.35017  loss_sent:  1.2140722  loss_word:  171.27982  Time cost:  6.958525657653809\n",
      "Start and end (285, 290)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  285  Epoch:  116  loss:  256.75812  loss_sent:  1.4917855  loss_word:  249.29918  Time cost:  6.836450815200806\n",
      "Start and end (290, 295)\n",
      "idx:  290  Epoch:  116  loss:  235.51767  loss_sent:  1.3905865  loss_word:  228.56473  Time cost:  6.8856635093688965\n",
      "Start and end (295, 300)\n",
      "idx:  295  Epoch:  116  loss:  296.29495  loss_sent:  1.5092947  loss_word:  288.7485  Time cost:  6.780900716781616\n",
      "Start and end (300, 305)\n",
      "idx:  300  Epoch:  116  loss:  285.37378  loss_sent:  1.2355509  loss_word:  279.196  Time cost:  6.6342785358428955\n",
      "Start and end (305, 310)\n",
      "idx:  305  Epoch:  116  loss:  310.14587  loss_sent:  1.8777735  loss_word:  300.757  Time cost:  6.624628305435181\n",
      "Start and end (310, 315)\n",
      "idx:  310  Epoch:  116  loss:  204.18013  loss_sent:  1.4008405  loss_word:  197.17592  Time cost:  6.641436815261841\n",
      "Start and end (315, 320)\n",
      "idx:  315  Epoch:  116  loss:  274.6104  loss_sent:  1.4802868  loss_word:  267.20898  Time cost:  6.650635480880737\n",
      "Start and end (320, 325)\n",
      "idx:  320  Epoch:  116  loss:  248.46124  loss_sent:  1.8556643  loss_word:  239.18292  Time cost:  6.676030397415161\n",
      "Start and end (325, 330)\n",
      "idx:  325  Epoch:  116  loss:  248.27779  loss_sent:  1.620673  loss_word:  240.17444  Time cost:  6.478214263916016\n",
      "Start and end (330, 335)\n",
      "idx:  330  Epoch:  116  loss:  222.96043  loss_sent:  1.4848397  loss_word:  215.53622  Time cost:  6.565547704696655\n",
      "Start and end (335, 340)\n",
      "idx:  335  Epoch:  116  loss:  218.88873  loss_sent:  1.5912849  loss_word:  210.9323  Time cost:  6.613651752471924\n",
      "Start and end (340, 345)\n",
      "idx:  340  Epoch:  116  loss:  278.00674  loss_sent:  1.2897588  loss_word:  271.55795  Time cost:  6.521976947784424\n",
      "Start and end (345, 350)\n",
      "idx:  345  Epoch:  116  loss:  238.09514  loss_sent:  1.4876891  loss_word:  230.65668  Time cost:  6.636993885040283\n",
      "Start and end (350, 355)\n",
      "idx:  350  Epoch:  116  loss:  206.4481  loss_sent:  1.5594112  loss_word:  198.65105  Time cost:  6.657983303070068\n",
      "Start and end (355, 360)\n",
      "idx:  355  Epoch:  116  loss:  211.06447  loss_sent:  1.9881485  loss_word:  201.12372  Time cost:  6.66694188117981\n",
      "Start and end (360, 365)\n",
      "idx:  360  Epoch:  116  loss:  280.64694  loss_sent:  1.6546748  loss_word:  272.3736  Time cost:  6.782546043395996\n",
      "Start and end (365, 370)\n",
      "idx:  365  Epoch:  116  loss:  161.89668  loss_sent:  1.2526519  loss_word:  155.63345  Time cost:  6.744914293289185\n",
      "Start and end (370, 375)\n",
      "idx:  370  Epoch:  116  loss:  146.65204  loss_sent:  2.0761108  loss_word:  136.27148  Time cost:  6.739752292633057\n",
      "Start and end (375, 380)\n",
      "idx:  375  Epoch:  116  loss:  283.55823  loss_sent:  1.7774408  loss_word:  274.67105  Time cost:  6.9861650466918945\n",
      "Start and end (380, 385)\n",
      "idx:  380  Epoch:  116  loss:  208.37398  loss_sent:  1.7514147  loss_word:  199.6169  Time cost:  6.993632078170776\n",
      "Start and end (385, 390)\n",
      "idx:  385  Epoch:  116  loss:  246.25409  loss_sent:  1.6736376  loss_word:  237.88591  Time cost:  6.974940538406372\n",
      "Start and end (390, 395)\n",
      "idx:  390  Epoch:  116  loss:  188.8299  loss_sent:  1.2747757  loss_word:  182.45601  Time cost:  7.3002002239227295\n",
      "Start and end (395, 400)\n",
      "idx:  395  Epoch:  116  loss:  162.70105  loss_sent:  1.6646934  loss_word:  154.37761  Time cost:  6.969140291213989\n",
      "Start and end (400, 405)\n",
      "idx:  400  Epoch:  116  loss:  320.05762  loss_sent:  1.5806185  loss_word:  312.15454  Time cost:  6.979966163635254\n",
      "Start and end (405, 410)\n",
      "idx:  405  Epoch:  116  loss:  223.10463  loss_sent:  1.2338626  loss_word:  216.9353  Time cost:  6.5765814781188965\n",
      "Start and end (410, 415)\n",
      "idx:  410  Epoch:  116  loss:  298.23822  loss_sent:  1.4783795  loss_word:  290.8463  Time cost:  7.272502422332764\n",
      "Start and end (415, 420)\n",
      "idx:  415  Epoch:  116  loss:  216.0846  loss_sent:  1.3868698  loss_word:  209.15024  Time cost:  6.965927839279175\n",
      "Start and end (420, 425)\n",
      "idx:  420  Epoch:  116  loss:  242.33896  loss_sent:  1.5801146  loss_word:  234.43839  Time cost:  8.080721139907837\n",
      "Start and end (425, 430)\n",
      "idx:  425  Epoch:  116  loss:  280.12335  loss_sent:  1.9261837  loss_word:  270.49246  Time cost:  6.814347982406616\n",
      "Start and end (430, 435)\n",
      "idx:  430  Epoch:  116  loss:  248.91048  loss_sent:  1.5890014  loss_word:  240.96547  Time cost:  6.727077007293701\n",
      "Start and end (435, 440)\n",
      "idx:  435  Epoch:  116  loss:  192.88823  loss_sent:  1.7190756  loss_word:  184.29286  Time cost:  6.7873125076293945\n",
      "Start and end (440, 445)\n",
      "idx:  440  Epoch:  116  loss:  217.67004  loss_sent:  1.1795915  loss_word:  211.7721  Time cost:  6.571475267410278\n",
      "Start and end (445, 450)\n",
      "idx:  445  Epoch:  116  loss:  195.59375  loss_sent:  1.9932915  loss_word:  185.62729  Time cost:  6.772737503051758\n",
      "Start and end (450, 455)\n",
      "idx:  450  Epoch:  116  loss:  212.80495  loss_sent:  1.2553807  loss_word:  206.52805  Time cost:  6.694159746170044\n",
      "Start and end (455, 460)\n",
      "idx:  455  Epoch:  116  loss:  229.54187  loss_sent:  1.4611261  loss_word:  222.23624  Time cost:  6.769005537033081\n",
      "Start and end (460, 465)\n",
      "idx:  460  Epoch:  116  loss:  217.46628  loss_sent:  1.5767758  loss_word:  209.5824  Time cost:  6.811160326004028\n",
      "Start and end (465, 470)\n",
      "idx:  465  Epoch:  116  loss:  255.85536  loss_sent:  1.194711  loss_word:  249.88182  Time cost:  6.673142910003662\n",
      "Start and end (470, 475)\n",
      "idx:  470  Epoch:  116  loss:  209.23909  loss_sent:  2.18755  loss_word:  198.30135  Time cost:  6.718069076538086\n",
      "Start and end (475, 480)\n",
      "idx:  475  Epoch:  116  loss:  184.36032  loss_sent:  1.3736819  loss_word:  177.49194  Time cost:  6.762485027313232\n",
      "Start and end (480, 485)\n",
      "idx:  480  Epoch:  116  loss:  231.7754  loss_sent:  1.7448332  loss_word:  223.05124  Time cost:  6.668807744979858\n",
      "Start and end (485, 490)\n",
      "idx:  485  Epoch:  116  loss:  220.37833  loss_sent:  1.6774917  loss_word:  211.99089  Time cost:  6.830305576324463\n",
      "Start and end (490, 495)\n",
      "idx:  490  Epoch:  116  loss:  236.14246  loss_sent:  1.3007476  loss_word:  229.63867  Time cost:  6.711546182632446\n",
      "Start and end (495, 500)\n",
      "idx:  495  Epoch:  116  loss:  191.68114  loss_sent:  1.5313135  loss_word:  184.02455  Time cost:  6.918544054031372\n",
      "Start and end (500, 505)\n",
      "idx:  500  Epoch:  116  loss:  201.81662  loss_sent:  1.4027951  loss_word:  194.80266  Time cost:  6.615357875823975\n",
      "Start and end (505, 510)\n",
      "idx:  505  Epoch:  116  loss:  171.65706  loss_sent:  1.3953152  loss_word:  164.68048  Time cost:  6.778941869735718\n",
      "Start and end (510, 515)\n",
      "idx:  510  Epoch:  116  loss:  185.38036  loss_sent:  1.3866508  loss_word:  178.44713  Time cost:  6.802431106567383\n",
      "Start and end (515, 520)\n",
      "idx:  515  Epoch:  116  loss:  235.53128  loss_sent:  1.0156711  loss_word:  230.45291  Time cost:  6.833782911300659\n",
      "Start and end (520, 525)\n",
      "idx:  520  Epoch:  116  loss:  182.30302  loss_sent:  1.4556574  loss_word:  175.02473  Time cost:  6.798122882843018\n",
      "Start and end (525, 530)\n",
      "idx:  525  Epoch:  116  loss:  244.85759  loss_sent:  1.4034951  loss_word:  237.84012  Time cost:  6.934109926223755\n",
      "Start and end (530, 535)\n",
      "idx:  530  Epoch:  116  loss:  254.28331  loss_sent:  1.5637896  loss_word:  246.46436  Time cost:  6.905130863189697\n",
      "Start and end (535, 540)\n",
      "idx:  535  Epoch:  116  loss:  169.12227  loss_sent:  1.1933215  loss_word:  163.15569  Time cost:  6.789161205291748\n",
      "Start and end (540, 545)\n",
      "idx:  540  Epoch:  116  loss:  294.491  loss_sent:  1.4708284  loss_word:  287.13684  Time cost:  6.7382402420043945\n",
      "Start and end (545, 550)\n",
      "idx:  545  Epoch:  116  loss:  188.45909  loss_sent:  1.3977292  loss_word:  181.47044  Time cost:  6.839019298553467\n",
      "Start and end (550, 555)\n",
      "idx:  550  Epoch:  116  loss:  165.61314  loss_sent:  1.5183402  loss_word:  158.02145  Time cost:  6.819293737411499\n",
      "Start and end (555, 560)\n",
      "idx:  555  Epoch:  116  loss:  162.49208  loss_sent:  1.1880602  loss_word:  156.55179  Time cost:  6.8670334815979\n",
      "Start and end (560, 565)\n",
      "idx:  560  Epoch:  116  loss:  229.79901  loss_sent:  2.0820076  loss_word:  219.38898  Time cost:  6.803935527801514\n",
      "Start and end (565, 570)\n",
      "idx:  565  Epoch:  116  loss:  143.2571  loss_sent:  1.4261757  loss_word:  136.12622  Time cost:  6.816638708114624\n",
      "Start and end (570, 575)\n",
      "idx:  570  Epoch:  116  loss:  204.03392  loss_sent:  1.3650569  loss_word:  197.20862  Time cost:  6.749948501586914\n",
      "Start and end (575, 580)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  575  Epoch:  116  loss:  257.10095  loss_sent:  1.2532676  loss_word:  250.83458  Time cost:  6.732599496841431\n",
      "Start and end (580, 585)\n",
      "idx:  580  Epoch:  116  loss:  306.0624  loss_sent:  1.7369485  loss_word:  297.37766  Time cost:  6.686900854110718\n",
      "Start and end (585, 590)\n",
      "idx:  585  Epoch:  116  loss:  261.5146  loss_sent:  1.2504791  loss_word:  255.26222  Time cost:  6.701721429824829\n",
      "Start and end (590, 595)\n",
      "idx:  590  Epoch:  116  loss:  213.55055  loss_sent:  1.3869649  loss_word:  206.61574  Time cost:  6.690624237060547\n",
      "Start and end (595, 600)\n",
      "idx:  595  Epoch:  116  loss:  160.78584  loss_sent:  1.4998825  loss_word:  153.28645  Time cost:  6.725364923477173\n",
      "Start and end (600, 605)\n",
      "idx:  600  Epoch:  116  loss:  218.67072  loss_sent:  1.2048901  loss_word:  212.64627  Time cost:  7.337717294692993\n",
      "Start and end (605, 610)\n",
      "idx:  605  Epoch:  116  loss:  247.7607  loss_sent:  1.4544078  loss_word:  240.48866  Time cost:  6.927732229232788\n",
      "Start and end (610, 615)\n",
      "idx:  610  Epoch:  116  loss:  226.41176  loss_sent:  1.1131123  loss_word:  220.84619  Time cost:  6.781819581985474\n",
      "Start and end (615, 620)\n",
      "idx:  615  Epoch:  116  loss:  236.91055  loss_sent:  2.6686695  loss_word:  223.5672  Time cost:  6.680739641189575\n",
      "Start and end (620, 625)\n",
      "idx:  620  Epoch:  116  loss:  205.64546  loss_sent:  1.1687891  loss_word:  199.80151  Time cost:  6.575701951980591\n",
      "Start and end (625, 630)\n",
      "idx:  625  Epoch:  116  loss:  178.36974  loss_sent:  1.7569504  loss_word:  169.58496  Time cost:  6.534673690795898\n",
      "Start and end (630, 635)\n",
      "idx:  630  Epoch:  116  loss:  254.17734  loss_sent:  1.448035  loss_word:  246.93715  Time cost:  6.5699005126953125\n",
      "Start and end (635, 640)\n",
      "idx:  635  Epoch:  116  loss:  303.99545  loss_sent:  1.612584  loss_word:  295.93253  Time cost:  6.549483776092529\n",
      "Start and end (640, 645)\n",
      "idx:  640  Epoch:  116  loss:  221.18015  loss_sent:  1.8129535  loss_word:  212.11536  Time cost:  6.493494272232056\n",
      "Start and end (645, 650)\n",
      "idx:  645  Epoch:  116  loss:  174.46797  loss_sent:  1.3350089  loss_word:  167.79292  Time cost:  6.479721307754517\n",
      "Start and end (650, 655)\n",
      "idx:  650  Epoch:  116  loss:  205.60309  loss_sent:  1.4691157  loss_word:  198.25752  Time cost:  6.933324337005615\n",
      "Start and end (655, 660)\n",
      "idx:  655  Epoch:  116  loss:  177.77638  loss_sent:  1.1471641  loss_word:  172.04056  Time cost:  6.708876609802246\n",
      "Start and end (660, 665)\n",
      "idx:  660  Epoch:  116  loss:  256.96667  loss_sent:  1.427418  loss_word:  249.8296  Time cost:  6.708704471588135\n",
      "Start and end (665, 670)\n",
      "idx:  665  Epoch:  116  loss:  226.72629  loss_sent:  1.5865794  loss_word:  218.7934  Time cost:  6.500312805175781\n",
      "Start and end (670, 675)\n",
      "idx:  670  Epoch:  116  loss:  227.65204  loss_sent:  1.8032387  loss_word:  218.63583  Time cost:  6.59365701675415\n",
      "Start and end (675, 680)\n",
      "idx:  675  Epoch:  116  loss:  211.30223  loss_sent:  1.6327341  loss_word:  203.13857  Time cost:  6.842148065567017\n",
      "Start and end (680, 685)\n",
      "idx:  680  Epoch:  116  loss:  203.33902  loss_sent:  1.6846646  loss_word:  194.91568  Time cost:  7.284081935882568\n",
      "Start and end (685, 690)\n",
      "idx:  685  Epoch:  116  loss:  316.54385  loss_sent:  1.3738766  loss_word:  309.6745  Time cost:  6.817506313323975\n",
      "Start and end (690, 695)\n",
      "idx:  690  Epoch:  116  loss:  216.66508  loss_sent:  1.9708278  loss_word:  206.81094  Time cost:  6.698826313018799\n",
      "Start and end (695, 700)\n",
      "idx:  695  Epoch:  116  loss:  299.59204  loss_sent:  1.9763201  loss_word:  289.71045  Time cost:  6.779696702957153\n",
      "Start and end (700, 705)\n",
      "idx:  700  Epoch:  116  loss:  240.1555  loss_sent:  1.3311629  loss_word:  233.4997  Time cost:  6.810937166213989\n",
      "Start and end (705, 710)\n",
      "idx:  705  Epoch:  116  loss:  364.97372  loss_sent:  1.2475129  loss_word:  358.73618  Time cost:  7.009565830230713\n",
      "Start and end (710, 715)\n",
      "idx:  710  Epoch:  116  loss:  227.42291  loss_sent:  1.3415349  loss_word:  220.71524  Time cost:  6.775005102157593\n",
      "Start and end (715, 720)\n",
      "idx:  715  Epoch:  116  loss:  262.21008  loss_sent:  1.5570672  loss_word:  254.42479  Time cost:  6.8376264572143555\n",
      "Start and end (720, 725)\n",
      "idx:  720  Epoch:  116  loss:  217.23752  loss_sent:  1.3676543  loss_word:  210.39925  Time cost:  6.955682754516602\n",
      "Start and end (725, 730)\n",
      "idx:  725  Epoch:  116  loss:  245.9952  loss_sent:  2.2117224  loss_word:  234.93658  Time cost:  6.691842317581177\n",
      "Start and end (730, 735)\n",
      "idx:  730  Epoch:  116  loss:  317.7568  loss_sent:  1.5125617  loss_word:  310.19397  Time cost:  6.681088924407959\n",
      "Start and end (735, 740)\n",
      "idx:  735  Epoch:  116  loss:  199.53198  loss_sent:  1.0957828  loss_word:  194.05305  Time cost:  6.601060152053833\n",
      "Start and end (740, 745)\n",
      "idx:  740  Epoch:  116  loss:  192.1723  loss_sent:  1.3178865  loss_word:  185.58287  Time cost:  6.632692575454712\n",
      "Start and end (745, 750)\n",
      "idx:  745  Epoch:  116  loss:  200.70459  loss_sent:  2.1885743  loss_word:  189.76175  Time cost:  6.583362340927124\n",
      "Start and end (750, 755)\n",
      "idx:  750  Epoch:  116  loss:  221.39586  loss_sent:  1.5247074  loss_word:  213.7723  Time cost:  6.618322134017944\n",
      "Start and end (755, 760)\n",
      "idx:  755  Epoch:  116  loss:  167.17787  loss_sent:  1.2013992  loss_word:  161.17091  Time cost:  6.665602922439575\n",
      "Start and end (760, 765)\n",
      "idx:  760  Epoch:  116  loss:  314.95856  loss_sent:  1.3766967  loss_word:  308.075  Time cost:  6.51938533782959\n",
      "Start and end (765, 770)\n",
      "idx:  765  Epoch:  116  loss:  188.58817  loss_sent:  1.441542  loss_word:  181.38045  Time cost:  6.587023496627808\n",
      "Start and end (770, 775)\n",
      "idx:  770  Epoch:  116  loss:  160.49007  loss_sent:  1.4537694  loss_word:  153.22124  Time cost:  6.610689163208008\n",
      "Start and end (775, 780)\n",
      "idx:  775  Epoch:  116  loss:  229.55508  loss_sent:  1.52001  loss_word:  221.95503  Time cost:  6.6010119915008545\n",
      "Start and end (780, 785)\n",
      "idx:  780  Epoch:  116  loss:  232.19489  loss_sent:  1.5547873  loss_word:  224.42093  Time cost:  6.584455966949463\n",
      "Start and end (785, 790)\n",
      "idx:  785  Epoch:  116  loss:  241.26529  loss_sent:  1.3383603  loss_word:  234.57349  Time cost:  6.526170015335083\n",
      "Start and end (790, 795)\n",
      "idx:  790  Epoch:  116  loss:  209.4279  loss_sent:  1.3487027  loss_word:  202.68439  Time cost:  6.642438650131226\n",
      "Start and end (795, 800)\n",
      "idx:  795  Epoch:  116  loss:  192.3126  loss_sent:  1.5786068  loss_word:  184.4196  Time cost:  6.797171592712402\n",
      "Start and end (800, 805)\n",
      "idx:  800  Epoch:  116  loss:  153.91168  loss_sent:  1.723965  loss_word:  145.29185  Time cost:  6.672710180282593\n",
      "Start and end (805, 810)\n",
      "idx:  805  Epoch:  116  loss:  205.30168  loss_sent:  1.367509  loss_word:  198.46414  Time cost:  6.5102739334106445\n",
      "Start and end (810, 815)\n",
      "idx:  810  Epoch:  116  loss:  241.4336  loss_sent:  1.3241119  loss_word:  234.81302  Time cost:  6.5601866245269775\n",
      "Start and end (815, 820)\n",
      "idx:  815  Epoch:  116  loss:  237.41405  loss_sent:  1.189131  loss_word:  231.46837  Time cost:  6.650867223739624\n",
      "Start and end (820, 825)\n",
      "idx:  820  Epoch:  116  loss:  213.21011  loss_sent:  1.878284  loss_word:  203.81866  Time cost:  6.572501182556152\n",
      "Start and end (825, 830)\n",
      "idx:  825  Epoch:  116  loss:  326.56232  loss_sent:  1.7735683  loss_word:  317.69446  Time cost:  6.521097898483276\n",
      "Start and end (830, 835)\n",
      "idx:  830  Epoch:  116  loss:  131.88765  loss_sent:  1.1906533  loss_word:  125.93438  Time cost:  6.6631739139556885\n",
      "Start and end (835, 840)\n",
      "idx:  835  Epoch:  116  loss:  188.16911  loss_sent:  1.4484311  loss_word:  180.92697  Time cost:  6.557085275650024\n",
      "Start and end (840, 845)\n",
      "idx:  840  Epoch:  116  loss:  259.25198  loss_sent:  1.6315671  loss_word:  251.09415  Time cost:  6.705300331115723\n",
      "Start and end (845, 850)\n",
      "idx:  845  Epoch:  116  loss:  302.33423  loss_sent:  1.6152523  loss_word:  294.25797  Time cost:  6.467591285705566\n",
      "Start and end (850, 855)\n",
      "idx:  850  Epoch:  116  loss:  222.70251  loss_sent:  1.7331487  loss_word:  214.03677  Time cost:  6.6191184520721436\n",
      "Start and end (855, 860)\n",
      "idx:  855  Epoch:  116  loss:  219.1744  loss_sent:  1.454735  loss_word:  211.90073  Time cost:  6.547698020935059\n",
      "Start and end (860, 865)\n",
      "idx:  860  Epoch:  116  loss:  185.62613  loss_sent:  1.6724045  loss_word:  177.26408  Time cost:  6.613363027572632\n",
      "Start and end (865, 870)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  865  Epoch:  116  loss:  238.18073  loss_sent:  1.6185447  loss_word:  230.08801  Time cost:  6.631990194320679\n",
      "Start and end (870, 875)\n",
      "idx:  870  Epoch:  116  loss:  248.19334  loss_sent:  1.9669595  loss_word:  238.35857  Time cost:  6.60932993888855\n",
      "Start and end (875, 880)\n",
      "idx:  875  Epoch:  116  loss:  188.81473  loss_sent:  1.04707  loss_word:  183.57938  Time cost:  6.625004291534424\n",
      "Start and end (880, 885)\n",
      "idx:  880  Epoch:  116  loss:  186.78175  loss_sent:  1.8117179  loss_word:  177.72316  Time cost:  6.567403078079224\n",
      "Start and end (885, 890)\n",
      "idx:  885  Epoch:  116  loss:  263.43686  loss_sent:  2.1137187  loss_word:  252.86833  Time cost:  6.707356214523315\n",
      "Start and end (890, 895)\n",
      "idx:  890  Epoch:  116  loss:  250.62712  loss_sent:  1.2053692  loss_word:  244.60023  Time cost:  6.8424506187438965\n",
      "Start and end (895, 900)\n",
      "idx:  895  Epoch:  116  loss:  191.50798  loss_sent:  1.3474648  loss_word:  184.77068  Time cost:  6.56720495223999\n",
      "Start and end (900, 905)\n",
      "idx:  900  Epoch:  116  loss:  212.18633  loss_sent:  1.2558929  loss_word:  205.90686  Time cost:  6.711552143096924\n",
      "Start and end (905, 910)\n",
      "idx:  905  Epoch:  116  loss:  206.43442  loss_sent:  1.3724214  loss_word:  199.57233  Time cost:  6.6102516651153564\n",
      "Start and end (910, 915)\n",
      "idx:  910  Epoch:  116  loss:  220.35146  loss_sent:  1.8844032  loss_word:  210.92944  Time cost:  6.58156943321228\n",
      "Start and end (915, 920)\n",
      "idx:  915  Epoch:  116  loss:  320.7283  loss_sent:  1.1653373  loss_word:  314.9016  Time cost:  6.596658945083618\n",
      "Start and end (920, 925)\n",
      "idx:  920  Epoch:  116  loss:  319.62292  loss_sent:  1.7119792  loss_word:  311.06305  Time cost:  6.605177879333496\n",
      "Start and end (925, 930)\n",
      "idx:  925  Epoch:  116  loss:  162.27219  loss_sent:  1.3327272  loss_word:  155.60854  Time cost:  6.601773262023926\n",
      "Start and end (930, 935)\n",
      "idx:  930  Epoch:  116  loss:  323.64664  loss_sent:  1.9310927  loss_word:  313.99112  Time cost:  6.586780309677124\n",
      "Start and end (935, 940)\n",
      "idx:  935  Epoch:  116  loss:  243.57968  loss_sent:  1.2886546  loss_word:  237.13644  Time cost:  6.644167423248291\n",
      "Start and end (940, 945)\n",
      "idx:  940  Epoch:  116  loss:  244.11125  loss_sent:  1.7854499  loss_word:  235.184  Time cost:  6.635694980621338\n",
      "Start and end (945, 950)\n",
      "idx:  945  Epoch:  116  loss:  157.65082  loss_sent:  1.3987029  loss_word:  150.6573  Time cost:  6.573305368423462\n",
      "Start and end (950, 955)\n",
      "idx:  950  Epoch:  116  loss:  206.93584  loss_sent:  1.260469  loss_word:  200.6335  Time cost:  6.583267688751221\n",
      "Start and end (955, 960)\n",
      "idx:  955  Epoch:  116  loss:  198.47456  loss_sent:  2.2828803  loss_word:  187.06017  Time cost:  6.651716709136963\n",
      "Start and end (960, 965)\n",
      "idx:  960  Epoch:  116  loss:  229.69606  loss_sent:  1.3667985  loss_word:  222.86208  Time cost:  6.498596906661987\n",
      "Start and end (965, 970)\n",
      "idx:  965  Epoch:  116  loss:  159.51738  loss_sent:  1.6302432  loss_word:  151.36615  Time cost:  6.551079988479614\n",
      "Start and end (970, 975)\n",
      "idx:  970  Epoch:  116  loss:  252.92253  loss_sent:  1.5757425  loss_word:  245.04381  Time cost:  6.730637311935425\n",
      "Start and end (975, 980)\n",
      "idx:  975  Epoch:  116  loss:  284.73187  loss_sent:  1.4675124  loss_word:  277.3943  Time cost:  6.77489161491394\n",
      "Start and end (980, 985)\n",
      "idx:  980  Epoch:  116  loss:  258.75323  loss_sent:  1.6911461  loss_word:  250.29749  Time cost:  6.671435594558716\n",
      "Start and end (985, 990)\n",
      "idx:  985  Epoch:  116  loss:  196.82677  loss_sent:  1.1263086  loss_word:  191.19524  Time cost:  6.527776479721069\n",
      "Start and end (990, 995)\n",
      "idx:  990  Epoch:  116  loss:  257.074  loss_sent:  1.3108361  loss_word:  250.51984  Time cost:  6.5165393352508545\n",
      "Start and end (995, 1000)\n",
      "idx:  995  Epoch:  116  loss:  197.3455  loss_sent:  1.8816979  loss_word:  187.93701  Time cost:  6.644513130187988\n",
      "Start and end (1000, 1005)\n",
      "idx:  1000  Epoch:  116  loss:  183.46123  loss_sent:  1.9004908  loss_word:  173.95877  Time cost:  6.5977771282196045\n",
      "Start and end (1005, 1010)\n",
      "idx:  1005  Epoch:  116  loss:  234.44379  loss_sent:  1.4447851  loss_word:  227.21985  Time cost:  6.661003828048706\n",
      "Start and end (1010, 1015)\n",
      "idx:  1010  Epoch:  116  loss:  184.28056  loss_sent:  1.5709713  loss_word:  176.4257  Time cost:  6.528634071350098\n",
      "Start and end (1015, 1020)\n",
      "idx:  1015  Epoch:  116  loss:  186.88985  loss_sent:  1.3611201  loss_word:  180.08424  Time cost:  6.701786041259766\n",
      "Start and end (1020, 1025)\n",
      "idx:  1020  Epoch:  116  loss:  177.87425  loss_sent:  1.736513  loss_word:  169.1917  Time cost:  6.47139835357666\n",
      "Start and end (1025, 1030)\n",
      "idx:  1025  Epoch:  116  loss:  244.87688  loss_sent:  1.3707218  loss_word:  238.02325  Time cost:  6.781311988830566\n",
      "Start and end (1030, 1035)\n",
      "idx:  1030  Epoch:  116  loss:  261.3581  loss_sent:  1.8542634  loss_word:  252.08678  Time cost:  6.588073015213013\n",
      "Start and end (1035, 1040)\n",
      "idx:  1035  Epoch:  116  loss:  222.65977  loss_sent:  1.0897729  loss_word:  217.21089  Time cost:  6.491015672683716\n",
      "Start and end (1040, 1045)\n",
      "idx:  1040  Epoch:  116  loss:  232.27844  loss_sent:  1.1052369  loss_word:  226.75224  Time cost:  6.575242042541504\n",
      "Start and end (1045, 1050)\n",
      "idx:  1045  Epoch:  116  loss:  283.7985  loss_sent:  1.942541  loss_word:  274.0858  Time cost:  6.504274606704712\n",
      "Start and end (1050, 1055)\n",
      "idx:  1050  Epoch:  116  loss:  237.08182  loss_sent:  2.0629845  loss_word:  226.76689  Time cost:  6.639456033706665\n",
      "Start and end (1055, 1060)\n",
      "idx:  1055  Epoch:  116  loss:  155.48033  loss_sent:  1.5568745  loss_word:  147.69598  Time cost:  6.574857234954834\n",
      "Start and end (1060, 1065)\n",
      "idx:  1060  Epoch:  116  loss:  246.69177  loss_sent:  1.2437172  loss_word:  240.47319  Time cost:  6.6124279499053955\n",
      "Start and end (1065, 1070)\n",
      "idx:  1065  Epoch:  116  loss:  242.58977  loss_sent:  1.3277795  loss_word:  235.95085  Time cost:  6.55367112159729\n",
      "Start and end (1070, 1075)\n",
      "idx:  1070  Epoch:  116  loss:  153.62152  loss_sent:  1.6173778  loss_word:  145.53462  Time cost:  6.573421955108643\n",
      "Start and end (1075, 1080)\n",
      "idx:  1075  Epoch:  116  loss:  209.81143  loss_sent:  1.6477407  loss_word:  201.57272  Time cost:  6.612092971801758\n",
      "Start and end (1080, 1085)\n",
      "idx:  1080  Epoch:  116  loss:  165.22798  loss_sent:  1.2972244  loss_word:  158.74187  Time cost:  6.65557336807251\n",
      "Start and end (1085, 1090)\n",
      "idx:  1085  Epoch:  116  loss:  210.03651  loss_sent:  1.6032761  loss_word:  202.02014  Time cost:  6.502428293228149\n",
      "Start and end (1090, 1095)\n",
      "idx:  1090  Epoch:  116  loss:  146.02367  loss_sent:  1.7351048  loss_word:  137.34814  Time cost:  6.642767667770386\n",
      "Start and end (1095, 1100)\n",
      "idx:  1095  Epoch:  116  loss:  148.64935  loss_sent:  1.5904771  loss_word:  140.69699  Time cost:  6.68747353553772\n",
      "Start and end (1100, 1105)\n",
      "idx:  1100  Epoch:  116  loss:  332.8068  loss_sent:  1.8474475  loss_word:  323.56955  Time cost:  6.609562873840332\n",
      "Start and end (1105, 1110)\n",
      "idx:  1105  Epoch:  116  loss:  256.602  loss_sent:  1.6341707  loss_word:  248.43114  Time cost:  6.7120680809021\n",
      "Start and end (1110, 1115)\n",
      "idx:  1110  Epoch:  116  loss:  266.6887  loss_sent:  1.854046  loss_word:  257.4185  Time cost:  6.833364486694336\n",
      "Start and end (1115, 1120)\n",
      "idx:  1115  Epoch:  116  loss:  179.81674  loss_sent:  1.6016821  loss_word:  171.80835  Time cost:  6.7765278816223145\n",
      "Start and end (1120, 1125)\n",
      "idx:  1120  Epoch:  116  loss:  234.88126  loss_sent:  1.7196946  loss_word:  226.2828  Time cost:  7.095032453536987\n",
      "Start and end (1125, 1130)\n",
      "idx:  1125  Epoch:  116  loss:  235.68275  loss_sent:  1.4049565  loss_word:  228.65796  Time cost:  6.731786489486694\n",
      "Start and end (1130, 1135)\n",
      "idx:  1130  Epoch:  116  loss:  232.48238  loss_sent:  1.4373746  loss_word:  225.29549  Time cost:  6.522664546966553\n",
      "Start and end (1135, 1140)\n",
      "idx:  1135  Epoch:  116  loss:  294.3756  loss_sent:  1.186852  loss_word:  288.44135  Time cost:  6.736085891723633\n",
      "Start and end (1140, 1145)\n",
      "idx:  1140  Epoch:  116  loss:  288.22275  loss_sent:  1.4598185  loss_word:  280.9236  Time cost:  6.984959840774536\n",
      "Start and end (1145, 1150)\n",
      "idx:  1145  Epoch:  116  loss:  183.6753  loss_sent:  1.110112  loss_word:  178.12473  Time cost:  6.817992687225342\n",
      "Start and end (1150, 1155)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1150  Epoch:  116  loss:  170.81062  loss_sent:  1.1400726  loss_word:  165.11026  Time cost:  6.846503257751465\n",
      "Start and end (1155, 1160)\n",
      "idx:  1155  Epoch:  116  loss:  195.16835  loss_sent:  1.4004796  loss_word:  188.16594  Time cost:  6.690985441207886\n",
      "Start and end (1160, 1165)\n",
      "idx:  1160  Epoch:  116  loss:  205.12071  loss_sent:  1.6420488  loss_word:  196.91048  Time cost:  6.809703826904297\n",
      "Start and end (1165, 1170)\n",
      "idx:  1165  Epoch:  116  loss:  184.79576  loss_sent:  1.7277169  loss_word:  176.15717  Time cost:  6.754482984542847\n",
      "Start and end (1170, 1175)\n",
      "idx:  1170  Epoch:  116  loss:  224.18658  loss_sent:  1.823055  loss_word:  215.0713  Time cost:  6.888874769210815\n",
      "Start and end (1175, 1180)\n",
      "idx:  1175  Epoch:  116  loss:  175.02159  loss_sent:  1.4687452  loss_word:  167.67786  Time cost:  6.602054119110107\n",
      "Start and end (1180, 1185)\n",
      "idx:  1180  Epoch:  116  loss:  203.1011  loss_sent:  1.573884  loss_word:  195.23167  Time cost:  6.970076322555542\n",
      "Start and end (1185, 1190)\n",
      "idx:  1185  Epoch:  116  loss:  209.229  loss_sent:  1.6891171  loss_word:  200.78342  Time cost:  6.679449796676636\n",
      "Start and end (1190, 1195)\n",
      "idx:  1190  Epoch:  116  loss:  231.48027  loss_sent:  1.9514594  loss_word:  221.72298  Time cost:  6.632785081863403\n",
      "Start and end (1195, 1200)\n",
      "idx:  1195  Epoch:  116  loss:  194.11879  loss_sent:  1.9044821  loss_word:  184.59639  Time cost:  6.9502904415130615\n",
      "Start and end (1200, 1205)\n",
      "idx:  1200  Epoch:  116  loss:  214.08449  loss_sent:  1.8493361  loss_word:  204.83781  Time cost:  6.663024663925171\n",
      "Start and end (1205, 1210)\n",
      "idx:  1205  Epoch:  116  loss:  288.12177  loss_sent:  1.603269  loss_word:  280.10544  Time cost:  6.649949073791504\n",
      "Start and end (1210, 1215)\n",
      "idx:  1210  Epoch:  116  loss:  237.59816  loss_sent:  1.7403529  loss_word:  228.89638  Time cost:  6.706387758255005\n",
      "Start and end (1215, 1220)\n",
      "idx:  1215  Epoch:  116  loss:  205.57248  loss_sent:  1.6769769  loss_word:  197.18759  Time cost:  6.678299903869629\n",
      "Start and end (1220, 1225)\n",
      "idx:  1220  Epoch:  116  loss:  354.06155  loss_sent:  1.0941479  loss_word:  348.5908  Time cost:  6.669140815734863\n",
      "Start and end (1225, 1230)\n",
      "idx:  1225  Epoch:  116  loss:  206.25175  loss_sent:  2.3398385  loss_word:  194.55255  Time cost:  6.918013334274292\n",
      "Start and end (1230, 1235)\n",
      "idx:  1230  Epoch:  116  loss:  266.47247  loss_sent:  1.8209667  loss_word:  257.36765  Time cost:  6.8273420333862305\n",
      "Start and end (1235, 1240)\n",
      "idx:  1235  Epoch:  116  loss:  285.8284  loss_sent:  2.066227  loss_word:  275.49728  Time cost:  6.688719034194946\n",
      "Start and end (1240, 1245)\n",
      "idx:  1240  Epoch:  116  loss:  168.29793  loss_sent:  1.5713938  loss_word:  160.44093  Time cost:  6.571681499481201\n",
      "Start and end (1245, 1250)\n",
      "idx:  1245  Epoch:  116  loss:  212.1377  loss_sent:  1.5332426  loss_word:  204.4715  Time cost:  6.7017598152160645\n",
      "Start and end (1250, 1255)\n",
      "idx:  1250  Epoch:  116  loss:  240.75801  loss_sent:  1.2044499  loss_word:  234.73576  Time cost:  6.749185800552368\n",
      "Start and end (1255, 1260)\n",
      "idx:  1255  Epoch:  116  loss:  296.7456  loss_sent:  1.1527047  loss_word:  290.9821  Time cost:  6.654234409332275\n",
      "Start and end (1260, 1265)\n",
      "idx:  1260  Epoch:  116  loss:  149.1771  loss_sent:  1.261964  loss_word:  142.86728  Time cost:  6.488769769668579\n",
      "Start and end (1265, 1270)\n",
      "idx:  1265  Epoch:  116  loss:  240.75647  loss_sent:  1.3922558  loss_word:  233.79517  Time cost:  6.503582954406738\n",
      "Start and end (1270, 1275)\n",
      "idx:  1270  Epoch:  116  loss:  230.83333  loss_sent:  1.6058447  loss_word:  222.80408  Time cost:  6.687570095062256\n",
      "Start and end (1275, 1280)\n",
      "idx:  1275  Epoch:  116  loss:  209.66026  loss_sent:  1.2586156  loss_word:  203.3672  Time cost:  6.5240278244018555\n",
      "Start and end (1280, 1285)\n",
      "idx:  1280  Epoch:  116  loss:  249.50688  loss_sent:  1.7735513  loss_word:  240.63911  Time cost:  6.518672704696655\n",
      "Start and end (1285, 1290)\n",
      "idx:  1285  Epoch:  116  loss:  289.38434  loss_sent:  0.92674756  loss_word:  284.75055  Time cost:  6.583901643753052\n",
      "Start and end (1290, 1295)\n",
      "idx:  1290  Epoch:  116  loss:  287.22675  loss_sent:  1.4070053  loss_word:  280.19174  Time cost:  6.894373893737793\n",
      "Start and end (1295, 1300)\n",
      "idx:  1295  Epoch:  116  loss:  228.29793  loss_sent:  1.113025  loss_word:  222.73279  Time cost:  6.677449941635132\n",
      "Start and end (1300, 1305)\n",
      "idx:  1300  Epoch:  116  loss:  288.6661  loss_sent:  1.5693282  loss_word:  280.81946  Time cost:  6.577805042266846\n",
      "Start and end (1305, 1310)\n",
      "idx:  1305  Epoch:  116  loss:  253.02568  loss_sent:  1.2870489  loss_word:  246.59042  Time cost:  6.482613801956177\n",
      "Start and end (1310, 1315)\n",
      "idx:  1310  Epoch:  116  loss:  304.0353  loss_sent:  1.4528573  loss_word:  296.77103  Time cost:  6.581581354141235\n",
      "Start and end (1315, 1320)\n",
      "idx:  1315  Epoch:  116  loss:  200.15729  loss_sent:  1.0975491  loss_word:  194.66954  Time cost:  6.67552924156189\n",
      "Start and end (1320, 1325)\n",
      "idx:  1320  Epoch:  116  loss:  251.0636  loss_sent:  1.6961625  loss_word:  242.5828  Time cost:  6.647836685180664\n",
      "Start and end (1325, 1330)\n",
      "idx:  1325  Epoch:  116  loss:  240.43661  loss_sent:  1.3706266  loss_word:  233.58348  Time cost:  6.609379529953003\n",
      "Start and end (1330, 1335)\n",
      "idx:  1330  Epoch:  116  loss:  213.1951  loss_sent:  1.4810754  loss_word:  205.78975  Time cost:  6.611427068710327\n",
      "Start and end (1335, 1340)\n",
      "idx:  1335  Epoch:  116  loss:  203.35063  loss_sent:  1.6001146  loss_word:  195.35007  Time cost:  6.671243906021118\n",
      "Start and end (1340, 1345)\n",
      "idx:  1340  Epoch:  116  loss:  323.39578  loss_sent:  1.9112177  loss_word:  313.83966  Time cost:  6.56920862197876\n",
      "Start and end (1345, 1350)\n",
      "idx:  1345  Epoch:  116  loss:  216.22743  loss_sent:  1.5468252  loss_word:  208.49333  Time cost:  6.655975341796875\n",
      "Start and end (1350, 1355)\n",
      "idx:  1350  Epoch:  116  loss:  167.67558  loss_sent:  1.656852  loss_word:  159.39133  Time cost:  6.5287580490112305\n",
      "Start and end (1355, 1360)\n",
      "idx:  1355  Epoch:  116  loss:  187.36215  loss_sent:  1.0599891  loss_word:  182.06223  Time cost:  6.576386451721191\n",
      "Start and end (1360, 1365)\n",
      "idx:  1360  Epoch:  116  loss:  165.17111  loss_sent:  1.6499083  loss_word:  156.92159  Time cost:  6.571340322494507\n",
      "Start and end (1365, 1370)\n",
      "idx:  1365  Epoch:  116  loss:  241.75713  loss_sent:  2.1144588  loss_word:  231.18483  Time cost:  6.7148826122283936\n",
      "Start and end (1370, 1375)\n",
      "idx:  1370  Epoch:  116  loss:  347.00415  loss_sent:  1.7367083  loss_word:  338.32062  Time cost:  6.575153112411499\n",
      "Start and end (1375, 1380)\n",
      "idx:  1375  Epoch:  116  loss:  235.18727  loss_sent:  1.4013493  loss_word:  228.18053  Time cost:  6.633340120315552\n",
      "Start and end (1380, 1385)\n",
      "idx:  1380  Epoch:  116  loss:  241.27277  loss_sent:  2.1728885  loss_word:  230.40833  Time cost:  6.729464054107666\n",
      "Start and end (1385, 1390)\n",
      "idx:  1385  Epoch:  116  loss:  184.25273  loss_sent:  1.4508684  loss_word:  176.9984  Time cost:  6.636036396026611\n",
      "Start and end (1390, 1395)\n",
      "idx:  1390  Epoch:  116  loss:  226.3922  loss_sent:  1.3028954  loss_word:  219.87767  Time cost:  6.642367362976074\n",
      "Start and end (1395, 1400)\n",
      "idx:  1395  Epoch:  116  loss:  183.19742  loss_sent:  1.4681442  loss_word:  175.8567  Time cost:  6.514646291732788\n",
      "Start and end (1400, 1405)\n",
      "idx:  1400  Epoch:  116  loss:  230.47192  loss_sent:  1.3510736  loss_word:  223.71655  Time cost:  6.481451988220215\n",
      "Start and end (1405, 1410)\n",
      "idx:  1405  Epoch:  116  loss:  188.6724  loss_sent:  1.6429589  loss_word:  180.45763  Time cost:  6.589199781417847\n",
      "Start and end (1410, 1415)\n",
      "idx:  1410  Epoch:  116  loss:  147.93932  loss_sent:  1.3882234  loss_word:  140.99815  Time cost:  6.596113920211792\n",
      "Start and end (1415, 1420)\n",
      "idx:  1415  Epoch:  116  loss:  182.68008  loss_sent:  1.4656595  loss_word:  175.35175  Time cost:  6.597460746765137\n",
      "Start and end (1420, 1425)\n",
      "idx:  1420  Epoch:  116  loss:  128.39565  loss_sent:  1.4859418  loss_word:  120.965935  Time cost:  6.511759519577026\n",
      "Start and end (1425, 1430)\n",
      "idx:  1425  Epoch:  116  loss:  240.83348  loss_sent:  1.7292881  loss_word:  232.18703  Time cost:  6.67645525932312\n",
      "Start and end (1430, 1435)\n",
      "idx:  1430  Epoch:  116  loss:  221.68239  loss_sent:  1.6641393  loss_word:  213.36166  Time cost:  6.548446893692017\n",
      "Start and end (1435, 1440)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1435  Epoch:  116  loss:  247.74803  loss_sent:  1.1351839  loss_word:  242.07213  Time cost:  6.591799020767212\n",
      "Start and end (1440, 1445)\n",
      "idx:  1440  Epoch:  116  loss:  347.86484  loss_sent:  1.97542  loss_word:  337.98776  Time cost:  6.598008155822754\n",
      "Start and end (1445, 1450)\n",
      "idx:  1445  Epoch:  116  loss:  256.5285  loss_sent:  2.0402637  loss_word:  246.3272  Time cost:  6.5604774951934814\n",
      "Start and end (1450, 1455)\n",
      "idx:  1450  Epoch:  116  loss:  218.16039  loss_sent:  2.1161335  loss_word:  207.57974  Time cost:  6.600542068481445\n",
      "Start and end (1455, 1460)\n",
      "idx:  1455  Epoch:  116  loss:  309.77814  loss_sent:  1.6951352  loss_word:  301.30237  Time cost:  6.607466697692871\n",
      "Start and end (1460, 1465)\n",
      "idx:  1460  Epoch:  116  loss:  271.83838  loss_sent:  1.6561826  loss_word:  263.55743  Time cost:  6.58764386177063\n",
      "Start and end (1465, 1470)\n",
      "idx:  1465  Epoch:  116  loss:  229.43634  loss_sent:  1.6139278  loss_word:  221.36671  Time cost:  6.672659397125244\n",
      "Start and end (1470, 1475)\n",
      "idx:  1470  Epoch:  116  loss:  195.58093  loss_sent:  2.188192  loss_word:  184.63997  Time cost:  6.721330165863037\n",
      "Start and end (1475, 1480)\n",
      "idx:  1475  Epoch:  116  loss:  227.43344  loss_sent:  1.5289925  loss_word:  219.78848  Time cost:  6.613846778869629\n",
      "Start and end (1480, 1485)\n",
      "idx:  1480  Epoch:  116  loss:  231.49538  loss_sent:  1.7995478  loss_word:  222.49765  Time cost:  6.561617851257324\n",
      "Start and end (1485, 1490)\n",
      "idx:  1485  Epoch:  116  loss:  223.93239  loss_sent:  1.6003636  loss_word:  215.93056  Time cost:  6.543254852294922\n",
      "Start and end (1490, 1495)\n",
      "idx:  1490  Epoch:  116  loss:  271.56088  loss_sent:  1.2437482  loss_word:  265.34213  Time cost:  6.697572469711304\n",
      "Start and end (1495, 1500)\n",
      "idx:  1495  Epoch:  116  loss:  171.5702  loss_sent:  1.5705163  loss_word:  163.71762  Time cost:  6.493174314498901\n",
      "Start and end (1500, 1505)\n",
      "idx:  1500  Epoch:  116  loss:  173.5026  loss_sent:  1.2957306  loss_word:  167.02394  Time cost:  6.587276458740234\n",
      "Start and end (1505, 1510)\n",
      "idx:  1505  Epoch:  116  loss:  188.48796  loss_sent:  1.1043054  loss_word:  182.96643  Time cost:  6.6029722690582275\n",
      "Start and end (1510, 1515)\n",
      "idx:  1510  Epoch:  116  loss:  236.97981  loss_sent:  1.4935577  loss_word:  229.51204  Time cost:  6.607158899307251\n",
      "Start and end (1515, 1520)\n",
      "idx:  1515  Epoch:  116  loss:  183.08492  loss_sent:  1.7458937  loss_word:  174.35545  Time cost:  6.733335733413696\n",
      "Start and end (1520, 1525)\n",
      "idx:  1520  Epoch:  116  loss:  211.57922  loss_sent:  1.5114356  loss_word:  204.02203  Time cost:  6.572608947753906\n",
      "Start and end (1525, 1530)\n",
      "idx:  1525  Epoch:  116  loss:  205.21255  loss_sent:  1.3393054  loss_word:  198.51604  Time cost:  6.494997978210449\n",
      "Start and end (1530, 1535)\n",
      "idx:  1530  Epoch:  116  loss:  236.68105  loss_sent:  1.5143102  loss_word:  229.1095  Time cost:  6.51763391494751\n",
      "Start and end (1535, 1540)\n",
      "idx:  1535  Epoch:  116  loss:  184.22531  loss_sent:  1.8231211  loss_word:  175.10971  Time cost:  6.546542167663574\n",
      "Start and end (1540, 1545)\n",
      "idx:  1540  Epoch:  116  loss:  248.17003  loss_sent:  1.3443063  loss_word:  241.4485  Time cost:  6.532248020172119\n",
      "Start and end (1545, 1550)\n",
      "idx:  1545  Epoch:  116  loss:  273.41562  loss_sent:  1.322035  loss_word:  266.80545  Time cost:  6.590209484100342\n",
      "Start and end (1550, 1555)\n",
      "idx:  1550  Epoch:  116  loss:  240.44516  loss_sent:  1.7695093  loss_word:  231.5976  Time cost:  6.6970086097717285\n",
      "Start and end (1555, 1560)\n",
      "idx:  1555  Epoch:  116  loss:  202.97203  loss_sent:  1.2088454  loss_word:  196.9278  Time cost:  6.535136699676514\n",
      "Start and end (1560, 1565)\n",
      "idx:  1560  Epoch:  116  loss:  190.97887  loss_sent:  0.9790306  loss_word:  186.08372  Time cost:  6.508336544036865\n",
      "Start and end (1565, 1570)\n",
      "idx:  1565  Epoch:  116  loss:  190.68112  loss_sent:  1.7789023  loss_word:  181.7866  Time cost:  6.56043815612793\n",
      "Start and end (1570, 1575)\n",
      "idx:  1570  Epoch:  116  loss:  212.00948  loss_sent:  1.8021293  loss_word:  202.99884  Time cost:  6.472527742385864\n",
      "Start and end (1575, 1580)\n",
      "idx:  1575  Epoch:  116  loss:  195.71223  loss_sent:  1.8516316  loss_word:  186.45406  Time cost:  6.6289825439453125\n",
      "Start and end (1580, 1585)\n",
      "idx:  1580  Epoch:  116  loss:  260.5882  loss_sent:  1.5102222  loss_word:  253.03711  Time cost:  6.733196258544922\n",
      "Start and end (1585, 1590)\n",
      "idx:  1585  Epoch:  116  loss:  226.74957  loss_sent:  1.65369  loss_word:  218.48112  Time cost:  6.637142181396484\n",
      "Start and end (1590, 1595)\n",
      "idx:  1590  Epoch:  116  loss:  212.94972  loss_sent:  1.2114446  loss_word:  206.8925  Time cost:  6.591414213180542\n",
      "Start and end (1595, 1600)\n",
      "idx:  1595  Epoch:  116  loss:  233.74957  loss_sent:  1.5885007  loss_word:  225.80705  Time cost:  6.485024929046631\n",
      "Start and end (1600, 1605)\n",
      "idx:  1600  Epoch:  116  loss:  183.33023  loss_sent:  1.5172244  loss_word:  175.74411  Time cost:  6.621765375137329\n",
      "Start and end (1605, 1610)\n",
      "idx:  1605  Epoch:  116  loss:  272.8862  loss_sent:  1.6299593  loss_word:  264.73642  Time cost:  6.734843969345093\n",
      "Start and end (1610, 1615)\n",
      "idx:  1610  Epoch:  116  loss:  191.61465  loss_sent:  1.5893383  loss_word:  183.66795  Time cost:  6.481053352355957\n",
      "Start and end (1615, 1620)\n",
      "idx:  1615  Epoch:  116  loss:  259.4838  loss_sent:  1.9017222  loss_word:  249.9752  Time cost:  6.599679708480835\n",
      "Start and end (1620, 1625)\n",
      "idx:  1620  Epoch:  116  loss:  278.8729  loss_sent:  1.6821878  loss_word:  270.46198  Time cost:  6.626800537109375\n",
      "Start and end (1625, 1630)\n",
      "idx:  1625  Epoch:  116  loss:  175.62598  loss_sent:  1.2533382  loss_word:  169.35928  Time cost:  6.593828439712524\n",
      "Start and end (1630, 1635)\n",
      "idx:  1630  Epoch:  116  loss:  259.73083  loss_sent:  1.361876  loss_word:  252.92142  Time cost:  6.597116470336914\n",
      "Start and end (1635, 1640)\n",
      "idx:  1635  Epoch:  116  loss:  214.26988  loss_sent:  1.5077494  loss_word:  206.73112  Time cost:  6.667825698852539\n",
      "Start and end (1640, 1645)\n",
      "idx:  1640  Epoch:  116  loss:  186.42322  loss_sent:  0.93184686  loss_word:  181.76399  Time cost:  6.526250123977661\n",
      "Start and end (1645, 1650)\n",
      "idx:  1645  Epoch:  116  loss:  245.93198  loss_sent:  1.3593904  loss_word:  239.13503  Time cost:  6.776534557342529\n",
      "Start and end (1650, 1655)\n",
      "idx:  1650  Epoch:  116  loss:  346.09778  loss_sent:  1.7986934  loss_word:  337.10434  Time cost:  6.633001804351807\n",
      "Start and end (1655, 1660)\n",
      "idx:  1655  Epoch:  116  loss:  280.38037  loss_sent:  1.5500014  loss_word:  272.63037  Time cost:  6.7390217781066895\n",
      "Start and end (1660, 1665)\n",
      "idx:  1660  Epoch:  116  loss:  280.209  loss_sent:  1.495889  loss_word:  272.7296  Time cost:  6.614731788635254\n",
      "Start and end (1665, 1670)\n",
      "idx:  1665  Epoch:  116  loss:  322.97528  loss_sent:  1.4415777  loss_word:  315.76743  Time cost:  6.640909910202026\n",
      "Start and end (1670, 1675)\n",
      "idx:  1670  Epoch:  116  loss:  268.67487  loss_sent:  1.8076115  loss_word:  259.63684  Time cost:  6.607561826705933\n",
      "Start and end (1675, 1680)\n",
      "idx:  1675  Epoch:  116  loss:  172.56929  loss_sent:  1.5579509  loss_word:  164.7795  Time cost:  6.481310606002808\n",
      "Start and end (1680, 1685)\n",
      "idx:  1680  Epoch:  116  loss:  218.27896  loss_sent:  1.9117925  loss_word:  208.71999  Time cost:  6.653663873672485\n",
      "Start and end (1685, 1690)\n",
      "idx:  1685  Epoch:  116  loss:  258.56757  loss_sent:  1.1391102  loss_word:  252.87196  Time cost:  6.730045557022095\n",
      "Start and end (1690, 1695)\n",
      "idx:  1690  Epoch:  116  loss:  244.05072  loss_sent:  1.3454878  loss_word:  237.32329  Time cost:  6.717966318130493\n",
      "Start and end (1695, 1700)\n",
      "idx:  1695  Epoch:  116  loss:  286.75055  loss_sent:  1.8330653  loss_word:  277.5852  Time cost:  6.509933233261108\n",
      "Start and end (1700, 1705)\n",
      "idx:  1700  Epoch:  116  loss:  267.43103  loss_sent:  1.1388501  loss_word:  261.7368  Time cost:  6.662348031997681\n",
      "Start and end (1705, 1710)\n",
      "idx:  1705  Epoch:  116  loss:  309.9191  loss_sent:  1.4390949  loss_word:  302.72363  Time cost:  6.661677598953247\n",
      "Start and end (1710, 1715)\n",
      "idx:  1710  Epoch:  116  loss:  240.6788  loss_sent:  1.3358488  loss_word:  233.99957  Time cost:  6.57244610786438\n",
      "Start and end (1715, 1720)\n",
      "idx:  1715  Epoch:  116  loss:  291.3761  loss_sent:  1.1882939  loss_word:  285.4346  Time cost:  6.710064888000488\n",
      "Start and end (1720, 1725)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  1720  Epoch:  116  loss:  260.91483  loss_sent:  1.2647294  loss_word:  254.59122  Time cost:  6.5714874267578125\n",
      "Start and end (1725, 1730)\n",
      "idx:  1725  Epoch:  116  loss:  153.15689  loss_sent:  1.4084318  loss_word:  146.11472  Time cost:  6.603158473968506\n",
      "Start and end (1730, 1735)\n",
      "idx:  1730  Epoch:  116  loss:  303.88687  loss_sent:  1.9322457  loss_word:  294.22562  Time cost:  6.533473491668701\n",
      "Start and end (1735, 1740)\n",
      "idx:  1735  Epoch:  116  loss:  197.74818  loss_sent:  2.015944  loss_word:  187.66849  Time cost:  6.69771146774292\n",
      "Start and end (1740, 1745)\n",
      "idx:  1740  Epoch:  116  loss:  261.70065  loss_sent:  1.5242091  loss_word:  254.07964  Time cost:  6.621786832809448\n",
      "Start and end (1745, 1750)\n",
      "idx:  1745  Epoch:  116  loss:  176.10977  loss_sent:  1.3488815  loss_word:  169.36536  Time cost:  6.638092041015625\n",
      "Start and end (1750, 1755)\n",
      "idx:  1750  Epoch:  116  loss:  290.11472  loss_sent:  1.1757708  loss_word:  284.23584  Time cost:  6.485036134719849\n",
      "Start and end (1755, 1760)\n",
      "idx:  1755  Epoch:  116  loss:  278.66638  loss_sent:  2.3793578  loss_word:  266.7696  Time cost:  6.682978868484497\n",
      "Start and end (1760, 1765)\n",
      "idx:  1760  Epoch:  116  loss:  291.71024  loss_sent:  1.5128791  loss_word:  284.1458  Time cost:  6.639707326889038\n",
      "Start and end (1765, 1770)\n",
      "idx:  1765  Epoch:  116  loss:  220.5764  loss_sent:  1.3530428  loss_word:  213.81117  Time cost:  6.643928050994873\n",
      "Start and end (1770, 1775)\n",
      "idx:  1770  Epoch:  116  loss:  361.43582  loss_sent:  1.8079797  loss_word:  352.3959  Time cost:  6.605355978012085\n",
      "Start and end (1775, 1780)\n",
      "idx:  1775  Epoch:  116  loss:  193.73776  loss_sent:  1.9734708  loss_word:  183.87045  Time cost:  6.535089015960693\n",
      "Start and end (1780, 1785)\n",
      "idx:  1780  Epoch:  116  loss:  229.97884  loss_sent:  1.4333544  loss_word:  222.81209  Time cost:  6.63401198387146\n",
      "Start and end (1785, 1790)\n",
      "idx:  1785  Epoch:  116  loss:  284.31696  loss_sent:  2.502513  loss_word:  271.80435  Time cost:  6.626134872436523\n",
      "Start and end (1790, 1795)\n",
      "idx:  1790  Epoch:  116  loss:  110.86149  loss_sent:  1.9042333  loss_word:  101.34032  Time cost:  6.678132057189941\n",
      "Start and end (1795, 1800)\n",
      "idx:  1795  Epoch:  116  loss:  173.34473  loss_sent:  1.4243098  loss_word:  166.22319  Time cost:  6.608352422714233\n",
      "Start and end (1800, 1805)\n",
      "idx:  1800  Epoch:  116  loss:  225.66237  loss_sent:  1.9963683  loss_word:  215.68053  Time cost:  6.639783620834351\n",
      "Start and end (1805, 1810)\n",
      "idx:  1805  Epoch:  116  loss:  218.96655  loss_sent:  1.5942395  loss_word:  210.99536  Time cost:  6.567826509475708\n",
      "Start and end (1810, 1815)\n",
      "idx:  1810  Epoch:  116  loss:  152.51166  loss_sent:  1.193178  loss_word:  146.54575  Time cost:  6.702535629272461\n",
      "Start and end (1815, 1820)\n",
      "idx:  1815  Epoch:  116  loss:  263.37268  loss_sent:  1.4056938  loss_word:  256.34415  Time cost:  6.6242735385894775\n",
      "Start and end (1820, 1825)\n",
      "idx:  1820  Epoch:  116  loss:  260.38315  loss_sent:  1.6051368  loss_word:  252.3575  Time cost:  6.638595104217529\n",
      "Start and end (1825, 1830)\n",
      "idx:  1825  Epoch:  116  loss:  203.20428  loss_sent:  1.7544851  loss_word:  194.43185  Time cost:  6.626309871673584\n",
      "Start and end (1830, 1835)\n",
      "idx:  1830  Epoch:  116  loss:  195.38872  loss_sent:  1.2388798  loss_word:  189.19432  Time cost:  6.619909286499023\n",
      "Start and end (1835, 1840)\n",
      "idx:  1835  Epoch:  116  loss:  152.5857  loss_sent:  1.0598611  loss_word:  147.28636  Time cost:  6.668148517608643\n",
      "Start and end (1840, 1845)\n",
      "idx:  1840  Epoch:  116  loss:  284.04742  loss_sent:  1.1735976  loss_word:  278.17944  Time cost:  6.666426420211792\n",
      "Start and end (1845, 1850)\n",
      "idx:  1845  Epoch:  116  loss:  229.52582  loss_sent:  1.706526  loss_word:  220.99318  Time cost:  6.686080694198608\n",
      "Start and end (1850, 1855)\n",
      "idx:  1850  Epoch:  116  loss:  235.83835  loss_sent:  1.7385738  loss_word:  227.14545  Time cost:  6.712735652923584\n",
      "Start and end (1855, 1860)\n",
      "idx:  1855  Epoch:  116  loss:  243.68138  loss_sent:  1.6685213  loss_word:  235.33878  Time cost:  6.667213439941406\n",
      "Start and end (1860, 1865)\n",
      "idx:  1860  Epoch:  116  loss:  268.71234  loss_sent:  1.5884678  loss_word:  260.76996  Time cost:  6.579905033111572\n",
      "Start and end (1865, 1870)\n",
      "idx:  1865  Epoch:  116  loss:  145.0435  loss_sent:  1.7474151  loss_word:  136.30643  Time cost:  6.63593316078186\n",
      "Start and end (1870, 1875)\n",
      "idx:  1870  Epoch:  116  loss:  251.74475  loss_sent:  1.2920108  loss_word:  245.28471  Time cost:  6.64894700050354\n",
      "Start and end (1875, 1880)\n",
      "idx:  1875  Epoch:  116  loss:  256.5772  loss_sent:  1.8249573  loss_word:  247.45244  Time cost:  6.534345388412476\n",
      "Start and end (1880, 1885)\n",
      "idx:  1880  Epoch:  116  loss:  231.72154  loss_sent:  1.1404213  loss_word:  226.01944  Time cost:  6.660382032394409\n",
      "Start and end (1885, 1890)\n",
      "idx:  1885  Epoch:  116  loss:  124.05249  loss_sent:  1.6854693  loss_word:  115.62514  Time cost:  6.520721197128296\n",
      "Start and end (1890, 1895)\n",
      "idx:  1890  Epoch:  116  loss:  208.93793  loss_sent:  1.5945814  loss_word:  200.96503  Time cost:  6.680073022842407\n",
      "Start and end (1895, 1900)\n",
      "idx:  1895  Epoch:  116  loss:  317.63293  loss_sent:  1.6996773  loss_word:  309.13452  Time cost:  6.620701313018799\n",
      "Start and end (1900, 1905)\n",
      "idx:  1900  Epoch:  116  loss:  167.1576  loss_sent:  2.1637187  loss_word:  156.33902  Time cost:  6.712318420410156\n",
      "Start and end (1905, 1910)\n",
      "idx:  1905  Epoch:  116  loss:  241.27339  loss_sent:  1.0953517  loss_word:  235.79665  Time cost:  6.700058698654175\n",
      "Start and end (1910, 1915)\n",
      "idx:  1910  Epoch:  116  loss:  222.34532  loss_sent:  1.5354642  loss_word:  214.66801  Time cost:  6.7255027294158936\n",
      "Start and end (1915, 1920)\n",
      "idx:  1915  Epoch:  116  loss:  198.13206  loss_sent:  1.3767546  loss_word:  191.24829  Time cost:  6.65900444984436\n",
      "Start and end (1920, 1925)\n",
      "idx:  1920  Epoch:  116  loss:  188.37495  loss_sent:  1.415381  loss_word:  181.29805  Time cost:  6.603758811950684\n",
      "Start and end (1925, 1930)\n",
      "idx:  1925  Epoch:  116  loss:  255.66765  loss_sent:  1.2769006  loss_word:  249.28314  Time cost:  6.589915037155151\n",
      "Start and end (1930, 1935)\n",
      "idx:  1930  Epoch:  116  loss:  227.6741  loss_sent:  1.3667111  loss_word:  220.84056  Time cost:  6.623523473739624\n",
      "Start and end (1935, 1940)\n",
      "idx:  1935  Epoch:  116  loss:  257.73807  loss_sent:  1.1087271  loss_word:  252.1944  Time cost:  6.692340135574341\n",
      "Start and end (1940, 1945)\n",
      "idx:  1940  Epoch:  116  loss:  159.8751  loss_sent:  1.35213  loss_word:  153.11446  Time cost:  6.629709243774414\n",
      "Start and end (1945, 1950)\n",
      "idx:  1945  Epoch:  116  loss:  231.55951  loss_sent:  1.358478  loss_word:  224.76712  Time cost:  6.6484904289245605\n",
      "Start and end (1950, 1955)\n",
      "idx:  1950  Epoch:  116  loss:  180.23076  loss_sent:  0.95863897  loss_word:  175.43755  Time cost:  6.683498382568359\n",
      "Start and end (1955, 1960)\n",
      "idx:  1955  Epoch:  116  loss:  218.69473  loss_sent:  2.275346  loss_word:  207.31802  Time cost:  6.724438190460205\n",
      "Start and end (1960, 1965)\n",
      "idx:  1960  Epoch:  116  loss:  256.35614  loss_sent:  1.6683617  loss_word:  248.01437  Time cost:  6.61261773109436\n",
      "Start and end (1965, 1970)\n",
      "idx:  1965  Epoch:  116  loss:  231.89952  loss_sent:  1.427936  loss_word:  224.75984  Time cost:  6.534518718719482\n",
      "Start and end (1970, 1975)\n",
      "idx:  1970  Epoch:  116  loss:  147.57674  loss_sent:  1.662562  loss_word:  139.26393  Time cost:  6.704292058944702\n",
      "Start and end (1975, 1980)\n",
      "idx:  1975  Epoch:  116  loss:  153.84024  loss_sent:  1.2394809  loss_word:  147.64282  Time cost:  6.69421124458313\n",
      "Start and end (1980, 1985)\n",
      "idx:  1980  Epoch:  116  loss:  198.80417  loss_sent:  2.0664473  loss_word:  188.47192  Time cost:  6.684082269668579\n",
      "Start and end (1985, 1990)\n",
      "idx:  1985  Epoch:  116  loss:  166.33464  loss_sent:  1.6399919  loss_word:  158.13469  Time cost:  6.511962652206421\n",
      "Start and end (1990, 1995)\n",
      "idx:  1990  Epoch:  116  loss:  252.10239  loss_sent:  1.267559  loss_word:  245.76457  Time cost:  6.605895042419434\n",
      "Start and end (1995, 2000)\n",
      "idx:  1995  Epoch:  116  loss:  223.39084  loss_sent:  1.524726  loss_word:  215.76721  Time cost:  6.678630828857422\n",
      "Start and end (2000, 2005)\n",
      "idx:  2000  Epoch:  116  loss:  256.85193  loss_sent:  2.0410442  loss_word:  246.64667  Time cost:  6.642580986022949\n",
      "Start and end (2005, 2010)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2005  Epoch:  116  loss:  174.81937  loss_sent:  1.1781232  loss_word:  168.92874  Time cost:  6.6687092781066895\n",
      "Start and end (2010, 2015)\n",
      "idx:  2010  Epoch:  116  loss:  156.54439  loss_sent:  1.7322464  loss_word:  147.88313  Time cost:  6.684261798858643\n",
      "Start and end (2015, 2020)\n",
      "idx:  2015  Epoch:  116  loss:  311.5389  loss_sent:  2.0236783  loss_word:  301.4205  Time cost:  6.641899824142456\n",
      "Start and end (2020, 2025)\n",
      "idx:  2020  Epoch:  116  loss:  252.16397  loss_sent:  2.2485445  loss_word:  240.92126  Time cost:  6.538182973861694\n",
      "Start and end (2025, 2030)\n",
      "idx:  2025  Epoch:  116  loss:  219.28587  loss_sent:  1.4650816  loss_word:  211.96045  Time cost:  6.5297887325286865\n",
      "Start and end (2030, 2035)\n",
      "idx:  2030  Epoch:  116  loss:  148.0013  loss_sent:  1.6819333  loss_word:  139.59163  Time cost:  6.582320928573608\n",
      "Start and end (2035, 2040)\n",
      "idx:  2035  Epoch:  116  loss:  219.84068  loss_sent:  1.2117935  loss_word:  213.78174  Time cost:  6.700220823287964\n",
      "Start and end (2040, 2045)\n",
      "idx:  2040  Epoch:  116  loss:  180.3335  loss_sent:  1.5666969  loss_word:  172.5  Time cost:  6.76785945892334\n",
      "Start and end (2045, 2050)\n",
      "idx:  2045  Epoch:  116  loss:  264.6403  loss_sent:  1.348153  loss_word:  257.8995  Time cost:  6.571847438812256\n",
      "Start and end (2050, 2055)\n",
      "idx:  2050  Epoch:  116  loss:  159.35059  loss_sent:  1.4005476  loss_word:  152.34785  Time cost:  6.577768802642822\n",
      "Start and end (2055, 2060)\n",
      "idx:  2055  Epoch:  116  loss:  214.06065  loss_sent:  1.1274309  loss_word:  208.42348  Time cost:  6.616706371307373\n",
      "Start and end (2060, 2065)\n",
      "idx:  2060  Epoch:  116  loss:  243.67648  loss_sent:  2.600266  loss_word:  230.67513  Time cost:  6.58280611038208\n",
      "Start and end (2065, 2070)\n",
      "idx:  2065  Epoch:  116  loss:  215.15833  loss_sent:  1.2577845  loss_word:  208.86938  Time cost:  6.6324357986450195\n",
      "Start and end (2070, 2075)\n",
      "idx:  2070  Epoch:  116  loss:  222.9957  loss_sent:  1.5480545  loss_word:  215.25543  Time cost:  6.657024145126343\n",
      "Start and end (2075, 2080)\n",
      "idx:  2075  Epoch:  116  loss:  216.59915  loss_sent:  1.4619956  loss_word:  209.28918  Time cost:  6.648588418960571\n",
      "Start and end (2080, 2085)\n",
      "idx:  2080  Epoch:  116  loss:  168.96547  loss_sent:  1.1141124  loss_word:  163.39491  Time cost:  6.544125080108643\n",
      "Start and end (2085, 2090)\n",
      "idx:  2085  Epoch:  116  loss:  236.51144  loss_sent:  1.3755875  loss_word:  229.6335  Time cost:  6.718194246292114\n",
      "Start and end (2090, 2095)\n",
      "idx:  2090  Epoch:  116  loss:  245.58289  loss_sent:  1.2305928  loss_word:  239.4299  Time cost:  6.698804616928101\n",
      "Start and end (2095, 2100)\n",
      "idx:  2095  Epoch:  116  loss:  259.73947  loss_sent:  1.7263925  loss_word:  251.10748  Time cost:  6.64731240272522\n",
      "Start and end (2100, 2105)\n",
      "idx:  2100  Epoch:  116  loss:  252.77988  loss_sent:  2.205874  loss_word:  241.75052  Time cost:  6.58025860786438\n",
      "Start and end (2105, 2110)\n",
      "idx:  2105  Epoch:  116  loss:  226.20732  loss_sent:  1.343827  loss_word:  219.48817  Time cost:  6.669782638549805\n",
      "Start and end (2110, 2115)\n",
      "idx:  2110  Epoch:  116  loss:  240.95749  loss_sent:  2.0253837  loss_word:  230.83058  Time cost:  6.510783433914185\n",
      "Start and end (2115, 2120)\n",
      "idx:  2115  Epoch:  116  loss:  282.94025  loss_sent:  1.4881344  loss_word:  275.4996  Time cost:  6.659217834472656\n",
      "Start and end (2120, 2125)\n",
      "idx:  2120  Epoch:  116  loss:  249.82993  loss_sent:  1.9501683  loss_word:  240.07906  Time cost:  6.645135164260864\n",
      "Start and end (2125, 2130)\n",
      "idx:  2125  Epoch:  116  loss:  343.42072  loss_sent:  1.562316  loss_word:  335.60916  Time cost:  6.639257907867432\n",
      "Start and end (2130, 2135)\n",
      "idx:  2130  Epoch:  116  loss:  228.82877  loss_sent:  1.5939088  loss_word:  220.8592  Time cost:  6.599827766418457\n",
      "Start and end (2135, 2140)\n",
      "idx:  2135  Epoch:  116  loss:  147.00363  loss_sent:  1.9249651  loss_word:  137.3788  Time cost:  6.565226316452026\n",
      "Start and end (2140, 2145)\n",
      "idx:  2140  Epoch:  116  loss:  227.98035  loss_sent:  1.69326  loss_word:  219.51404  Time cost:  6.652045965194702\n",
      "Start and end (2145, 2150)\n",
      "idx:  2145  Epoch:  116  loss:  127.8727  loss_sent:  1.4909891  loss_word:  120.41777  Time cost:  6.575541019439697\n",
      "Start and end (2150, 2155)\n",
      "idx:  2150  Epoch:  116  loss:  234.25682  loss_sent:  1.4638993  loss_word:  226.93732  Time cost:  6.75498366355896\n",
      "Start and end (2155, 2160)\n",
      "idx:  2155  Epoch:  116  loss:  207.46909  loss_sent:  1.0816453  loss_word:  202.06087  Time cost:  6.54003119468689\n",
      "Start and end (2160, 2165)\n",
      "idx:  2160  Epoch:  116  loss:  205.24738  loss_sent:  1.9999511  loss_word:  195.24763  Time cost:  6.704921245574951\n",
      "Start and end (2165, 2170)\n",
      "idx:  2165  Epoch:  116  loss:  239.53482  loss_sent:  1.2396944  loss_word:  233.33632  Time cost:  6.6922948360443115\n",
      "Start and end (2170, 2175)\n",
      "idx:  2170  Epoch:  116  loss:  201.7758  loss_sent:  1.4854805  loss_word:  194.34837  Time cost:  6.568530797958374\n",
      "Start and end (2175, 2180)\n",
      "idx:  2175  Epoch:  116  loss:  163.56279  loss_sent:  0.94088596  loss_word:  158.85837  Time cost:  6.509057521820068\n",
      "Start and end (2180, 2185)\n",
      "idx:  2180  Epoch:  116  loss:  207.65576  loss_sent:  1.1873689  loss_word:  201.71893  Time cost:  6.674218416213989\n",
      "Start and end (2185, 2190)\n",
      "idx:  2185  Epoch:  116  loss:  186.725  loss_sent:  1.341098  loss_word:  180.01952  Time cost:  6.633817195892334\n",
      "Start and end (2190, 2195)\n",
      "idx:  2190  Epoch:  116  loss:  168.87794  loss_sent:  1.9876001  loss_word:  158.93997  Time cost:  6.627317667007446\n",
      "Start and end (2195, 2200)\n",
      "idx:  2195  Epoch:  116  loss:  237.13135  loss_sent:  2.0174477  loss_word:  227.0441  Time cost:  6.465669870376587\n",
      "Start and end (2200, 2205)\n",
      "idx:  2200  Epoch:  116  loss:  156.95537  loss_sent:  1.4204977  loss_word:  149.85289  Time cost:  6.49333119392395\n",
      "Start and end (2205, 2210)\n",
      "idx:  2205  Epoch:  116  loss:  206.68486  loss_sent:  1.4117464  loss_word:  199.62613  Time cost:  6.535257577896118\n",
      "Start and end (2210, 2215)\n",
      "idx:  2210  Epoch:  116  loss:  293.36472  loss_sent:  1.6971028  loss_word:  284.87918  Time cost:  6.523311614990234\n",
      "Start and end (2215, 2220)\n",
      "idx:  2215  Epoch:  116  loss:  191.87975  loss_sent:  1.7438011  loss_word:  183.16075  Time cost:  6.558443069458008\n",
      "Start and end (2220, 2225)\n",
      "idx:  2220  Epoch:  116  loss:  264.90195  loss_sent:  1.232312  loss_word:  258.7404  Time cost:  6.667824745178223\n",
      "Start and end (2225, 2230)\n",
      "idx:  2225  Epoch:  116  loss:  183.94418  loss_sent:  1.3773609  loss_word:  177.05739  Time cost:  6.651071071624756\n",
      "Start and end (2230, 2235)\n",
      "idx:  2230  Epoch:  116  loss:  212.33153  loss_sent:  1.9285067  loss_word:  202.689  Time cost:  6.660029649734497\n",
      "Start and end (2235, 2240)\n",
      "idx:  2235  Epoch:  116  loss:  178.65419  loss_sent:  1.504544  loss_word:  171.13145  Time cost:  6.625656366348267\n",
      "Start and end (2240, 2245)\n",
      "idx:  2240  Epoch:  116  loss:  242.0198  loss_sent:  1.4373335  loss_word:  234.83316  Time cost:  6.74592399597168\n",
      "Start and end (2245, 2250)\n",
      "idx:  2245  Epoch:  116  loss:  213.16374  loss_sent:  1.5615126  loss_word:  205.35617  Time cost:  6.620302200317383\n",
      "Start and end (2250, 2255)\n",
      "idx:  2250  Epoch:  116  loss:  333.78958  loss_sent:  1.0775671  loss_word:  328.40176  Time cost:  6.723209619522095\n",
      "Start and end (2255, 2260)\n",
      "idx:  2255  Epoch:  116  loss:  227.14917  loss_sent:  2.0801353  loss_word:  216.74846  Time cost:  6.502751111984253\n",
      "Start and end (2260, 2265)\n",
      "idx:  2260  Epoch:  116  loss:  175.2641  loss_sent:  1.195834  loss_word:  169.28496  Time cost:  6.653665542602539\n",
      "Start and end (2265, 2270)\n",
      "idx:  2265  Epoch:  116  loss:  164.31311  loss_sent:  2.7786574  loss_word:  150.41983  Time cost:  6.685009956359863\n",
      "Start and end (2270, 2275)\n",
      "idx:  2270  Epoch:  116  loss:  202.81465  loss_sent:  1.2366484  loss_word:  196.6314  Time cost:  6.706578493118286\n",
      "Start and end (2275, 2280)\n",
      "idx:  2275  Epoch:  116  loss:  216.99625  loss_sent:  1.4345627  loss_word:  209.82343  Time cost:  6.795418739318848\n",
      "Start and end (2280, 2285)\n",
      "idx:  2280  Epoch:  116  loss:  184.43478  loss_sent:  1.9492177  loss_word:  174.68867  Time cost:  6.626237392425537\n",
      "Start and end (2285, 2290)\n",
      "idx:  2285  Epoch:  116  loss:  309.55872  loss_sent:  1.7993562  loss_word:  300.56195  Time cost:  6.658436298370361\n",
      "Start and end (2290, 2295)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  2290  Epoch:  116  loss:  272.47754  loss_sent:  1.2879571  loss_word:  266.03778  Time cost:  6.537327527999878\n",
      "Start and end (2295, 2300)\n",
      "idx:  2295  Epoch:  116  loss:  181.50351  loss_sent:  1.5086969  loss_word:  173.96002  Time cost:  6.617474555969238\n",
      "Start and end (2300, 2305)\n",
      "idx:  2300  Epoch:  116  loss:  178.2933  loss_sent:  2.01723  loss_word:  168.20715  Time cost:  6.513428449630737\n",
      "Start and end (2305, 2310)\n",
      "idx:  2305  Epoch:  116  loss:  196.03473  loss_sent:  1.5574759  loss_word:  188.24736  Time cost:  6.675695419311523\n",
      "Start and end (2310, 2315)\n",
      "idx:  2310  Epoch:  116  loss:  226.78693  loss_sent:  1.2810957  loss_word:  220.38147  Time cost:  6.700114727020264\n",
      "Start and end (2315, 2320)\n",
      "idx:  2315  Epoch:  116  loss:  197.16026  loss_sent:  1.3686907  loss_word:  190.3168  Time cost:  6.713260650634766\n",
      "Start and end (2320, 2325)\n",
      "idx:  2320  Epoch:  116  loss:  155.27032  loss_sent:  1.4687889  loss_word:  147.92638  Time cost:  6.697458982467651\n",
      "Start and end (2325, 2330)\n",
      "idx:  2325  Epoch:  116  loss:  196.85312  loss_sent:  2.444201  loss_word:  184.6321  Time cost:  6.569079875946045\n",
      "Start and end (2330, 2335)\n",
      "idx:  2330  Epoch:  116  loss:  246.45825  loss_sent:  2.3258145  loss_word:  234.8292  Time cost:  6.699617385864258\n",
      "Start and end (2335, 2340)\n",
      "idx:  2335  Epoch:  116  loss:  224.67378  loss_sent:  2.217378  loss_word:  213.58691  Time cost:  6.536456346511841\n",
      "Start and end (2340, 2345)\n",
      "idx:  2340  Epoch:  116  loss:  226.9819  loss_sent:  1.1362734  loss_word:  221.30052  Time cost:  6.579853773117065\n",
      "Start and end (2345, 2350)\n",
      "idx:  2345  Epoch:  116  loss:  218.45503  loss_sent:  1.2944703  loss_word:  211.9827  Time cost:  6.623218297958374\n",
      "Start and end (2350, 2355)\n",
      "idx:  2350  Epoch:  116  loss:  244.31738  loss_sent:  1.3474237  loss_word:  237.58026  Time cost:  6.546076059341431\n",
      "Start and end (2355, 2360)\n",
      "idx:  2355  Epoch:  116  loss:  210.46162  loss_sent:  1.3826439  loss_word:  203.54845  Time cost:  6.644642353057861\n",
      "Start and end (2360, 2365)\n",
      "idx:  2360  Epoch:  116  loss:  189.50124  loss_sent:  1.0266305  loss_word:  184.36807  Time cost:  6.619368314743042\n",
      "Start and end (2365, 2370)\n",
      "idx:  2365  Epoch:  116  loss:  249.79741  loss_sent:  1.6801424  loss_word:  241.39671  Time cost:  6.632345199584961\n",
      "Start and end (2370, 2375)\n",
      "idx:  2370  Epoch:  116  loss:  242.69092  loss_sent:  1.4399289  loss_word:  235.49127  Time cost:  6.684096097946167\n",
      "Start and end (2375, 2380)\n",
      "idx:  2375  Epoch:  116  loss:  162.19017  loss_sent:  1.8831716  loss_word:  152.77432  Time cost:  6.671422243118286\n",
      "Start and end (2380, 2385)\n",
      "idx:  2380  Epoch:  116  loss:  168.49048  loss_sent:  2.0762563  loss_word:  158.10918  Time cost:  6.602005243301392\n",
      "Start and end (2385, 2390)\n",
      "idx:  2385  Epoch:  116  loss:  233.74501  loss_sent:  1.3745048  loss_word:  226.8725  Time cost:  6.514819145202637\n",
      "Start and end (2390, 2395)\n",
      "idx:  2390  Epoch:  116  loss:  264.9643  loss_sent:  1.336751  loss_word:  258.28055  Time cost:  6.68375825881958\n",
      "Start and end (2395, 2400)\n",
      "idx:  2395  Epoch:  116  loss:  191.02669  loss_sent:  1.762431  loss_word:  182.21454  Time cost:  6.709858655929565\n",
      "Start and end (2400, 2405)\n",
      "idx:  2400  Epoch:  116  loss:  204.03548  loss_sent:  1.2760088  loss_word:  197.65544  Time cost:  6.732301473617554\n",
      "Start and end (2405, 2410)\n",
      "idx:  2405  Epoch:  116  loss:  261.4256  loss_sent:  1.4940367  loss_word:  253.95544  Time cost:  6.703309059143066\n",
      "Start and end (2410, 2415)\n",
      "idx:  2410  Epoch:  116  loss:  274.87848  loss_sent:  1.6890146  loss_word:  266.43335  Time cost:  6.5915422439575195\n",
      "Start and end (2415, 2420)\n",
      "idx:  2415  Epoch:  116  loss:  172.69308  loss_sent:  1.4997702  loss_word:  165.19421  Time cost:  6.554014205932617\n",
      "Start and end (2420, 2425)\n",
      "idx:  2420  Epoch:  116  loss:  186.93575  loss_sent:  2.2489865  loss_word:  175.6908  Time cost:  6.570105791091919\n",
      "Start and end (2425, 2430)\n",
      "idx:  2425  Epoch:  116  loss:  219.7636  loss_sent:  2.4326458  loss_word:  207.60036  Time cost:  6.63210654258728\n",
      "Start and end (2430, 2435)\n",
      "idx:  2430  Epoch:  116  loss:  153.58922  loss_sent:  1.8379766  loss_word:  144.39934  Time cost:  6.55327296257019\n",
      "Start and end (2435, 2440)\n",
      "idx:  2435  Epoch:  116  loss:  159.68765  loss_sent:  1.7690506  loss_word:  150.84242  Time cost:  6.523995637893677\n",
      "Start and end (2440, 2445)\n",
      "idx:  2440  Epoch:  116  loss:  159.9579  loss_sent:  1.0644431  loss_word:  154.63568  Time cost:  6.648654937744141\n",
      "Start and end (2445, 2450)\n",
      "idx:  2445  Epoch:  116  loss:  206.68413  loss_sent:  1.4127227  loss_word:  199.62051  Time cost:  6.736655950546265\n",
      "Start and end (2450, 2455)\n",
      "idx:  2450  Epoch:  116  loss:  236.17082  loss_sent:  1.8105198  loss_word:  227.11821  Time cost:  6.635170936584473\n",
      "Start and end (2455, 2460)\n",
      "idx:  2455  Epoch:  116  loss:  170.88626  loss_sent:  2.1970503  loss_word:  159.90097  Time cost:  6.671826124191284\n",
      "Start and end (2460, 2465)\n",
      "idx:  2460  Epoch:  116  loss:  242.40189  loss_sent:  2.3998299  loss_word:  230.40274  Time cost:  6.533017635345459\n",
      "Start and end (2465, 2470)\n",
      "idx:  2465  Epoch:  116  loss:  157.60107  loss_sent:  1.8171715  loss_word:  148.51523  Time cost:  6.553972482681274\n",
      "Start and end (2470, 2475)\n",
      "idx:  2470  Epoch:  116  loss:  215.78224  loss_sent:  2.7814243  loss_word:  201.87515  Time cost:  6.64054274559021\n",
      "Start and end (2475, 2480)\n",
      "idx:  2475  Epoch:  116  loss:  243.88332  loss_sent:  1.3024541  loss_word:  237.37105  Time cost:  6.544477224349976\n",
      "Start and end (2480, 2485)\n",
      "idx:  2480  Epoch:  116  loss:  290.34595  loss_sent:  1.530214  loss_word:  282.69492  Time cost:  6.660400152206421\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0724 12:54:25.701179 140197742454528 saver.py:1134] *******************************************************\n",
      "W0724 12:54:25.701931 140197742454528 saver.py:1135] TensorFlow's V1 checkpoint format has been deprecated.\n",
      "W0724 12:54:25.703800 140197742454528 saver.py:1136] Consider switching to the more efficient V2 format:\n",
      "W0724 12:54:25.704243 140197742454528 saver.py:1137]    `tf.train.Saver(write_version=tf.train.SaverDef.V2)`\n",
      "W0724 12:54:25.705804 140197742454528 saver.py:1138] now on by default.\n",
      "W0724 12:54:25.706351 140197742454528 saver.py:1139] *******************************************************\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully Written to temporary\n",
      "Epoch  116  is done. Saving the model ...\n",
      "Start and end (0, 5)\n",
      "idx:  0  Epoch:  117  loss:  306.0431  loss_sent:  2.2035294  loss_word:  295.02545  Time cost:  6.699774265289307\n",
      "Start and end (5, 10)\n",
      "idx:  5  Epoch:  117  loss:  195.7753  loss_sent:  2.1817558  loss_word:  184.86652  Time cost:  6.577568769454956\n",
      "Start and end (10, 15)\n",
      "idx:  10  Epoch:  117  loss:  254.49966  loss_sent:  1.750996  loss_word:  245.74469  Time cost:  6.66348123550415\n",
      "Start and end (15, 20)\n",
      "idx:  15  Epoch:  117  loss:  252.05351  loss_sent:  1.5523162  loss_word:  244.29192  Time cost:  6.678164482116699\n",
      "Start and end (20, 25)\n",
      "idx:  20  Epoch:  117  loss:  270.15604  loss_sent:  1.5204961  loss_word:  262.55353  Time cost:  6.657594442367554\n",
      "Start and end (25, 30)\n",
      "idx:  25  Epoch:  117  loss:  266.9403  loss_sent:  1.2484947  loss_word:  260.69788  Time cost:  6.559634447097778\n",
      "Start and end (30, 35)\n",
      "idx:  30  Epoch:  117  loss:  144.37234  loss_sent:  0.9423062  loss_word:  139.66081  Time cost:  6.60602879524231\n",
      "Start and end (35, 40)\n",
      "idx:  35  Epoch:  117  loss:  147.25262  loss_sent:  1.0895252  loss_word:  141.805  Time cost:  6.579108238220215\n",
      "Start and end (40, 45)\n",
      "idx:  40  Epoch:  117  loss:  233.95464  loss_sent:  1.8622398  loss_word:  224.64346  Time cost:  6.588969945907593\n",
      "Start and end (45, 50)\n",
      "idx:  45  Epoch:  117  loss:  141.80273  loss_sent:  1.4279113  loss_word:  134.6632  Time cost:  6.526709318161011\n",
      "Start and end (50, 55)\n",
      "idx:  50  Epoch:  117  loss:  191.79176  loss_sent:  1.3333424  loss_word:  185.12506  Time cost:  6.53370475769043\n",
      "Start and end (55, 60)\n",
      "idx:  55  Epoch:  117  loss:  226.92809  loss_sent:  1.888045  loss_word:  217.48787  Time cost:  6.6913769245147705\n",
      "Start and end (60, 65)\n",
      "idx:  60  Epoch:  117  loss:  239.89012  loss_sent:  1.2668312  loss_word:  233.55597  Time cost:  6.801069021224976\n",
      "Start and end (65, 70)\n",
      "idx:  65  Epoch:  117  loss:  255.62051  loss_sent:  1.5135365  loss_word:  248.05283  Time cost:  6.710395097732544\n",
      "Start and end (70, 75)\n",
      "idx:  70  Epoch:  117  loss:  179.07231  loss_sent:  1.508821  loss_word:  171.5282  Time cost:  6.527684450149536\n",
      "Start and end (75, 80)\n",
      "idx:  75  Epoch:  117  loss:  185.21843  loss_sent:  1.291372  loss_word:  178.76155  Time cost:  6.605578660964966\n",
      "Start and end (80, 85)\n",
      "idx:  80  Epoch:  117  loss:  214.79572  loss_sent:  2.258412  loss_word:  203.50366  Time cost:  6.620896339416504\n",
      "Start and end (85, 90)\n",
      "idx:  85  Epoch:  117  loss:  168.30359  loss_sent:  1.4101585  loss_word:  161.25279  Time cost:  6.547307252883911\n",
      "Start and end (90, 95)\n",
      "idx:  90  Epoch:  117  loss:  147.03513  loss_sent:  1.5761786  loss_word:  139.15424  Time cost:  6.657144546508789\n",
      "Start and end (95, 100)\n",
      "idx:  95  Epoch:  117  loss:  240.87814  loss_sent:  1.3309162  loss_word:  234.22357  Time cost:  6.518487930297852\n",
      "Start and end (100, 105)\n",
      "idx:  100  Epoch:  117  loss:  208.73972  loss_sent:  0.98177105  loss_word:  203.83087  Time cost:  6.60415506362915\n",
      "Start and end (105, 110)\n",
      "idx:  105  Epoch:  117  loss:  242.94943  loss_sent:  1.6355209  loss_word:  234.77184  Time cost:  6.658457517623901\n",
      "Start and end (110, 115)\n",
      "idx:  110  Epoch:  117  loss:  204.41978  loss_sent:  2.507652  loss_word:  191.88156  Time cost:  6.622431516647339\n",
      "Start and end (115, 120)\n",
      "idx:  115  Epoch:  117  loss:  166.36154  loss_sent:  0.80351114  loss_word:  162.34396  Time cost:  6.592669248580933\n",
      "Start and end (120, 125)\n",
      "idx:  120  Epoch:  117  loss:  247.30809  loss_sent:  2.8040757  loss_word:  233.2877  Time cost:  6.570029258728027\n",
      "Start and end (125, 130)\n",
      "idx:  125  Epoch:  117  loss:  207.71611  loss_sent:  1.1519386  loss_word:  201.95642  Time cost:  6.64224100112915\n",
      "Start and end (130, 135)\n",
      "idx:  130  Epoch:  117  loss:  242.73782  loss_sent:  1.3745873  loss_word:  235.86487  Time cost:  6.64862847328186\n",
      "Start and end (135, 140)\n",
      "idx:  135  Epoch:  117  loss:  175.44289  loss_sent:  1.4202361  loss_word:  168.34169  Time cost:  6.497362375259399\n",
      "Start and end (140, 145)\n",
      "idx:  140  Epoch:  117  loss:  214.95943  loss_sent:  1.2995539  loss_word:  208.46169  Time cost:  6.719506025314331\n",
      "Start and end (145, 150)\n",
      "idx:  145  Epoch:  117  loss:  195.17091  loss_sent:  1.2479674  loss_word:  188.9311  Time cost:  6.584052085876465\n",
      "Start and end (150, 155)\n",
      "idx:  150  Epoch:  117  loss:  210.27777  loss_sent:  0.9679733  loss_word:  205.43793  Time cost:  6.540065288543701\n",
      "Start and end (155, 160)\n",
      "idx:  155  Epoch:  117  loss:  163.01166  loss_sent:  1.0473049  loss_word:  157.77512  Time cost:  6.490424156188965\n",
      "Start and end (160, 165)\n",
      "idx:  160  Epoch:  117  loss:  327.2765  loss_sent:  1.0595481  loss_word:  321.97876  Time cost:  6.69803524017334\n",
      "Start and end (165, 170)\n",
      "idx:  165  Epoch:  117  loss:  224.40437  loss_sent:  1.797913  loss_word:  215.41481  Time cost:  6.559496164321899\n",
      "Start and end (170, 175)\n",
      "idx:  170  Epoch:  117  loss:  160.75246  loss_sent:  1.4947553  loss_word:  153.2787  Time cost:  6.67525315284729\n",
      "Start and end (175, 180)\n",
      "idx:  175  Epoch:  117  loss:  179.41805  loss_sent:  1.2797812  loss_word:  173.01915  Time cost:  6.6938395500183105\n",
      "Start and end (180, 185)\n",
      "idx:  180  Epoch:  117  loss:  187.63585  loss_sent:  1.3744633  loss_word:  180.76352  Time cost:  6.7018513679504395\n",
      "Start and end (185, 190)\n",
      "idx:  185  Epoch:  117  loss:  229.26888  loss_sent:  1.8938782  loss_word:  219.79945  Time cost:  6.6668596267700195\n",
      "Start and end (190, 195)\n",
      "idx:  190  Epoch:  117  loss:  192.80261  loss_sent:  1.2328817  loss_word:  186.63823  Time cost:  6.583254098892212\n",
      "Start and end (195, 200)\n",
      "idx:  195  Epoch:  117  loss:  247.08916  loss_sent:  1.2789855  loss_word:  240.69423  Time cost:  6.523444414138794\n",
      "Start and end (200, 205)\n",
      "idx:  200  Epoch:  117  loss:  234.75166  loss_sent:  1.7849859  loss_word:  225.82669  Time cost:  6.709418535232544\n",
      "Start and end (205, 210)\n",
      "idx:  205  Epoch:  117  loss:  174.01888  loss_sent:  1.4794556  loss_word:  166.6216  Time cost:  6.708991289138794\n",
      "Start and end (210, 215)\n",
      "idx:  210  Epoch:  117  loss:  169.11295  loss_sent:  1.3349736  loss_word:  162.43806  Time cost:  6.679375648498535\n",
      "Start and end (215, 220)\n",
      "idx:  215  Epoch:  117  loss:  282.5726  loss_sent:  1.2391254  loss_word:  276.37695  Time cost:  6.489034414291382\n",
      "Start and end (220, 225)\n",
      "idx:  220  Epoch:  117  loss:  192.79257  loss_sent:  1.3967353  loss_word:  185.8089  Time cost:  6.69689679145813\n",
      "Start and end (225, 230)\n",
      "idx:  225  Epoch:  117  loss:  161.68645  loss_sent:  1.3158298  loss_word:  155.10728  Time cost:  6.527334451675415\n",
      "Start and end (230, 235)\n",
      "idx:  230  Epoch:  117  loss:  223.43619  loss_sent:  1.9082493  loss_word:  213.89496  Time cost:  6.619202136993408\n",
      "Start and end (235, 240)\n",
      "idx:  235  Epoch:  117  loss:  309.92004  loss_sent:  1.2808155  loss_word:  303.516  Time cost:  6.62022066116333\n",
      "Start and end (240, 245)\n",
      "idx:  240  Epoch:  117  loss:  215.168  loss_sent:  1.4010822  loss_word:  208.1626  Time cost:  6.57706618309021\n",
      "Start and end (245, 250)\n",
      "idx:  245  Epoch:  117  loss:  232.6424  loss_sent:  1.173836  loss_word:  226.7732  Time cost:  6.650143623352051\n",
      "Start and end (250, 255)\n",
      "idx:  250  Epoch:  117  loss:  171.05128  loss_sent:  1.521656  loss_word:  163.443  Time cost:  6.629842042922974\n",
      "Start and end (255, 260)\n",
      "idx:  255  Epoch:  117  loss:  210.76117  loss_sent:  2.461253  loss_word:  198.4549  Time cost:  6.59478497505188\n",
      "Start and end (260, 265)\n",
      "idx:  260  Epoch:  117  loss:  273.89264  loss_sent:  1.4949881  loss_word:  266.41766  Time cost:  6.7833333015441895\n",
      "Start and end (265, 270)\n",
      "idx:  265  Epoch:  117  loss:  186.84914  loss_sent:  1.1270949  loss_word:  181.21367  Time cost:  6.661106824874878\n",
      "Start and end (270, 275)\n",
      "idx:  270  Epoch:  117  loss:  159.1731  loss_sent:  0.9320899  loss_word:  154.51265  Time cost:  6.6180260181427\n",
      "Start and end (275, 280)\n",
      "idx:  275  Epoch:  117  loss:  166.06818  loss_sent:  1.8846579  loss_word:  156.64488  Time cost:  6.506256580352783\n",
      "Start and end (280, 285)\n",
      "idx:  280  Epoch:  117  loss:  327.09772  loss_sent:  1.1735432  loss_word:  321.22998  Time cost:  6.643873691558838\n",
      "Start and end (285, 290)\n",
      "idx:  285  Epoch:  117  loss:  220.22055  loss_sent:  1.454043  loss_word:  212.95035  Time cost:  6.48525333404541\n",
      "Start and end (290, 295)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  290  Epoch:  117  loss:  141.43349  loss_sent:  1.2407072  loss_word:  135.22993  Time cost:  6.52098274230957\n",
      "Start and end (295, 300)\n",
      "idx:  295  Epoch:  117  loss:  209.18544  loss_sent:  1.1002694  loss_word:  203.68408  Time cost:  6.543124437332153\n",
      "Start and end (300, 305)\n",
      "idx:  300  Epoch:  117  loss:  143.52861  loss_sent:  0.9501236  loss_word:  138.77802  Time cost:  6.546809196472168\n",
      "Start and end (305, 310)\n",
      "idx:  305  Epoch:  117  loss:  183.43944  loss_sent:  1.4701047  loss_word:  176.08891  Time cost:  6.514309883117676\n",
      "Start and end (310, 315)\n",
      "idx:  310  Epoch:  117  loss:  207.7017  loss_sent:  1.2908405  loss_word:  201.24748  Time cost:  6.60737419128418\n",
      "Start and end (315, 320)\n",
      "idx:  315  Epoch:  117  loss:  104.48891  loss_sent:  1.2189769  loss_word:  98.394035  Time cost:  6.553439140319824\n",
      "Start and end (320, 325)\n",
      "idx:  320  Epoch:  117  loss:  220.7632  loss_sent:  1.0452548  loss_word:  215.53693  Time cost:  6.708653450012207\n",
      "Start and end (325, 330)\n",
      "idx:  325  Epoch:  117  loss:  225.72168  loss_sent:  1.001644  loss_word:  220.71344  Time cost:  6.64594292640686\n",
      "Start and end (330, 335)\n",
      "idx:  330  Epoch:  117  loss:  156.87811  loss_sent:  1.0378453  loss_word:  151.68892  Time cost:  6.583454370498657\n",
      "Start and end (335, 340)\n",
      "idx:  335  Epoch:  117  loss:  244.55263  loss_sent:  1.0383296  loss_word:  239.36098  Time cost:  6.572111129760742\n",
      "Start and end (340, 345)\n",
      "idx:  340  Epoch:  117  loss:  301.6994  loss_sent:  1.1195244  loss_word:  296.1018  Time cost:  6.49625825881958\n",
      "Start and end (345, 350)\n",
      "idx:  345  Epoch:  117  loss:  239.02866  loss_sent:  1.9539018  loss_word:  229.25911  Time cost:  6.731361627578735\n",
      "Start and end (350, 355)\n",
      "idx:  350  Epoch:  117  loss:  201.45337  loss_sent:  1.1963238  loss_word:  195.47176  Time cost:  6.654700994491577\n",
      "Start and end (355, 360)\n",
      "idx:  355  Epoch:  117  loss:  240.72519  loss_sent:  1.3986756  loss_word:  233.73181  Time cost:  6.6182541847229\n",
      "Start and end (360, 365)\n",
      "idx:  360  Epoch:  117  loss:  149.6981  loss_sent:  1.2581009  loss_word:  143.4076  Time cost:  6.639062166213989\n",
      "Start and end (365, 370)\n",
      "idx:  365  Epoch:  117  loss:  134.37622  loss_sent:  1.0819919  loss_word:  128.96628  Time cost:  6.5870397090911865\n",
      "Start and end (370, 375)\n",
      "idx:  370  Epoch:  117  loss:  152.10522  loss_sent:  1.6389371  loss_word:  143.91054  Time cost:  6.668228387832642\n",
      "Start and end (375, 380)\n",
      "idx:  375  Epoch:  117  loss:  156.34724  loss_sent:  1.1458284  loss_word:  150.6181  Time cost:  6.651634454727173\n",
      "Start and end (380, 385)\n",
      "idx:  380  Epoch:  117  loss:  209.95833  loss_sent:  1.0744457  loss_word:  204.5861  Time cost:  6.664473295211792\n",
      "Start and end (385, 390)\n",
      "idx:  385  Epoch:  117  loss:  238.81444  loss_sent:  1.4622973  loss_word:  231.50294  Time cost:  6.723708152770996\n",
      "Start and end (390, 395)\n",
      "idx:  390  Epoch:  117  loss:  233.73022  loss_sent:  1.7085422  loss_word:  225.18752  Time cost:  6.736862897872925\n",
      "Start and end (395, 400)\n",
      "idx:  395  Epoch:  117  loss:  248.1521  loss_sent:  2.2222242  loss_word:  237.04099  Time cost:  6.646353006362915\n",
      "Start and end (400, 405)\n",
      "idx:  400  Epoch:  117  loss:  240.28313  loss_sent:  2.1517124  loss_word:  229.52457  Time cost:  6.610479116439819\n",
      "Start and end (405, 410)\n",
      "idx:  405  Epoch:  117  loss:  226.46341  loss_sent:  1.4256439  loss_word:  219.33519  Time cost:  6.599884033203125\n",
      "Start and end (410, 415)\n",
      "idx:  410  Epoch:  117  loss:  212.67673  loss_sent:  1.3273747  loss_word:  206.03983  Time cost:  6.707777261734009\n",
      "Start and end (415, 420)\n",
      "idx:  415  Epoch:  117  loss:  169.83679  loss_sent:  1.1878893  loss_word:  163.89737  Time cost:  6.597674608230591\n",
      "Start and end (420, 425)\n",
      "idx:  420  Epoch:  117  loss:  181.79564  loss_sent:  1.3138714  loss_word:  175.22627  Time cost:  6.632967233657837\n",
      "Start and end (425, 430)\n",
      "idx:  425  Epoch:  117  loss:  288.37527  loss_sent:  1.5209366  loss_word:  280.77054  Time cost:  6.630752086639404\n",
      "Start and end (430, 435)\n",
      "idx:  430  Epoch:  117  loss:  204.25597  loss_sent:  1.7336367  loss_word:  195.58778  Time cost:  6.6432390213012695\n",
      "Start and end (435, 440)\n",
      "idx:  435  Epoch:  117  loss:  237.39929  loss_sent:  1.1520263  loss_word:  231.63916  Time cost:  6.598100662231445\n",
      "Start and end (440, 445)\n",
      "idx:  440  Epoch:  117  loss:  218.2972  loss_sent:  2.7639515  loss_word:  204.47743  Time cost:  6.607986211776733\n",
      "Start and end (445, 450)\n",
      "idx:  445  Epoch:  117  loss:  291.0693  loss_sent:  1.2434553  loss_word:  284.85205  Time cost:  6.686452150344849\n",
      "Start and end (450, 455)\n",
      "idx:  450  Epoch:  117  loss:  206.93532  loss_sent:  0.99314666  loss_word:  201.96957  Time cost:  6.611783266067505\n",
      "Start and end (455, 460)\n",
      "idx:  455  Epoch:  117  loss:  196.70612  loss_sent:  1.3412535  loss_word:  189.99983  Time cost:  6.624817132949829\n",
      "Start and end (460, 465)\n",
      "idx:  460  Epoch:  117  loss:  133.23659  loss_sent:  1.7015791  loss_word:  124.728676  Time cost:  6.461156606674194\n",
      "Start and end (465, 470)\n",
      "idx:  465  Epoch:  117  loss:  135.97313  loss_sent:  1.5840571  loss_word:  128.05286  Time cost:  6.552613019943237\n",
      "Start and end (470, 475)\n",
      "idx:  470  Epoch:  117  loss:  159.88277  loss_sent:  1.5693107  loss_word:  152.03624  Time cost:  6.769945859909058\n",
      "Start and end (475, 480)\n",
      "idx:  475  Epoch:  117  loss:  270.98212  loss_sent:  1.353896  loss_word:  264.21265  Time cost:  6.660336494445801\n",
      "Start and end (480, 485)\n",
      "idx:  480  Epoch:  117  loss:  196.12364  loss_sent:  1.2200809  loss_word:  190.02324  Time cost:  6.819717884063721\n",
      "Start and end (485, 490)\n",
      "idx:  485  Epoch:  117  loss:  226.9903  loss_sent:  1.1314391  loss_word:  221.3331  Time cost:  6.700930595397949\n",
      "Start and end (490, 495)\n",
      "idx:  490  Epoch:  117  loss:  185.71652  loss_sent:  1.4129791  loss_word:  178.65161  Time cost:  6.623173236846924\n",
      "Start and end (495, 500)\n",
      "idx:  495  Epoch:  117  loss:  245.38515  loss_sent:  1.0968357  loss_word:  239.90097  Time cost:  6.819800138473511\n",
      "Start and end (500, 505)\n",
      "idx:  500  Epoch:  117  loss:  256.05826  loss_sent:  1.4775267  loss_word:  248.67065  Time cost:  6.697864294052124\n",
      "Start and end (505, 510)\n",
      "idx:  505  Epoch:  117  loss:  188.62779  loss_sent:  1.6005306  loss_word:  180.62514  Time cost:  6.766076326370239\n",
      "Start and end (510, 515)\n",
      "idx:  510  Epoch:  117  loss:  217.05688  loss_sent:  1.2500114  loss_word:  210.80684  Time cost:  6.688108444213867\n",
      "Start and end (515, 520)\n",
      "idx:  515  Epoch:  117  loss:  260.04593  loss_sent:  1.8803664  loss_word:  250.6441  Time cost:  6.774724245071411\n",
      "Start and end (520, 525)\n",
      "idx:  520  Epoch:  117  loss:  275.17493  loss_sent:  1.351362  loss_word:  268.4181  Time cost:  6.747513055801392\n",
      "Start and end (525, 530)\n",
      "idx:  525  Epoch:  117  loss:  181.33986  loss_sent:  1.0218561  loss_word:  176.23059  Time cost:  6.690742254257202\n",
      "Start and end (530, 535)\n",
      "idx:  530  Epoch:  117  loss:  204.98224  loss_sent:  1.2542121  loss_word:  198.71118  Time cost:  6.647364854812622\n",
      "Start and end (535, 540)\n",
      "idx:  535  Epoch:  117  loss:  293.64716  loss_sent:  0.8171061  loss_word:  289.56165  Time cost:  6.743359088897705\n",
      "Start and end (540, 545)\n",
      "idx:  540  Epoch:  117  loss:  230.66124  loss_sent:  1.0906444  loss_word:  225.20804  Time cost:  6.673040151596069\n",
      "Start and end (545, 550)\n",
      "idx:  545  Epoch:  117  loss:  177.49696  loss_sent:  1.3932362  loss_word:  170.53078  Time cost:  6.62209677696228\n",
      "Start and end (550, 555)\n",
      "idx:  550  Epoch:  117  loss:  231.12033  loss_sent:  1.1654798  loss_word:  225.29294  Time cost:  6.598344564437866\n",
      "Start and end (555, 560)\n",
      "idx:  555  Epoch:  117  loss:  166.31712  loss_sent:  1.0067928  loss_word:  161.28317  Time cost:  6.631421089172363\n",
      "Start and end (560, 565)\n",
      "idx:  560  Epoch:  117  loss:  414.0907  loss_sent:  1.4873967  loss_word:  406.6537  Time cost:  6.67487645149231\n",
      "Start and end (565, 570)\n",
      "idx:  565  Epoch:  117  loss:  120.88557  loss_sent:  0.92861164  loss_word:  116.2425  Time cost:  6.666637897491455\n",
      "Start and end (570, 575)\n",
      "idx:  570  Epoch:  117  loss:  212.89113  loss_sent:  1.3671986  loss_word:  206.05511  Time cost:  6.681718587875366\n",
      "Start and end (575, 580)\n",
      "idx:  575  Epoch:  117  loss:  205.18863  loss_sent:  0.93283165  loss_word:  200.52446  Time cost:  6.54057765007019\n",
      "Start and end (580, 585)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx:  580  Epoch:  117  loss:  284.6357  loss_sent:  1.0177848  loss_word:  279.54678  Time cost:  6.750156879425049\n",
      "Start and end (585, 590)\n",
      "idx:  585  Epoch:  117  loss:  271.42517  loss_sent:  1.4857883  loss_word:  263.99622  Time cost:  6.945807218551636\n",
      "Start and end (590, 595)\n",
      "idx:  590  Epoch:  117  loss:  215.48586  loss_sent:  1.1522871  loss_word:  209.7244  Time cost:  6.81252384185791\n",
      "Start and end (595, 600)\n",
      "idx:  595  Epoch:  117  loss:  214.94214  loss_sent:  1.2504609  loss_word:  208.68983  Time cost:  6.684616327285767\n",
      "Start and end (600, 605)\n",
      "idx:  600  Epoch:  117  loss:  252.13863  loss_sent:  1.2400454  loss_word:  245.9384  Time cost:  6.706771612167358\n",
      "Start and end (605, 610)\n",
      "idx:  605  Epoch:  117  loss:  172.0123  loss_sent:  1.0169923  loss_word:  166.92737  Time cost:  6.979783535003662\n",
      "Start and end (610, 615)\n",
      "idx:  610  Epoch:  117  loss:  285.20636  loss_sent:  1.1862113  loss_word:  279.27527  Time cost:  7.063607692718506\n",
      "Start and end (615, 620)\n",
      "idx:  615  Epoch:  117  loss:  261.16116  loss_sent:  0.9005849  loss_word:  256.65826  Time cost:  7.031722068786621\n",
      "Start and end (620, 625)\n",
      "idx:  620  Epoch:  117  loss:  231.84712  loss_sent:  1.1084387  loss_word:  226.30492  Time cost:  7.390957593917847\n",
      "Start and end (625, 630)\n",
      "idx:  625  Epoch:  117  loss:  228.14642  loss_sent:  1.4851297  loss_word:  220.7208  Time cost:  7.393832206726074\n",
      "Start and end (630, 635)\n",
      "idx:  630  Epoch:  117  loss:  206.1744  loss_sent:  0.92733717  loss_word:  201.5377  Time cost:  6.879482269287109\n",
      "Start and end (635, 640)\n",
      "idx:  635  Epoch:  117  loss:  243.6556  loss_sent:  0.9931906  loss_word:  238.68965  Time cost:  6.709212779998779\n",
      "Start and end (640, 645)\n",
      "idx:  640  Epoch:  117  loss:  160.38365  loss_sent:  1.2374712  loss_word:  154.19629  Time cost:  7.217316627502441\n",
      "Start and end (645, 650)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-0434d5425a35>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    687\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Time cost: \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mstart_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 689\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-1-0434d5425a35>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    562\u001b[0m                                                \u001b[0mtf_num_distribution\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcurrent_num_distribution\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m                                                \u001b[0mtf_captions_matrix\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcurrent_captions_matrix\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 564\u001b[0;31m                                                \u001b[0mtf_captions_masks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcurrent_captions_masks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    565\u001b[0m                                     })\n\u001b[1;32m    566\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1339\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/CPU/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1427\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1431\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#WITH SCOPE FOR SESSION\n",
    "#!/usr/bin/env python\n",
    "# coding=utf-8\n",
    "\n",
    "__author__ = \"Xinpeng.Chen\"\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "\n",
    "import h5py\n",
    "import ipdb\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------------------------------------------\n",
    "# Initialization class\n",
    "#  1. Pooling the visual features into a single dense feature\n",
    "#  2. Then, build sentence LSTM, word LSTM\n",
    "# ------------------------------------------------------------------------------------------------------\n",
    "class RegionPooling_HierarchicalRNN():\n",
    "    def __init__(self, n_words,\n",
    "                       batch_size,\n",
    "                       num_boxes,\n",
    "                       feats_dim,\n",
    "                       project_dim,\n",
    "                       sentRNN_lstm_dim,\n",
    "                       sentRNN_FC_dim,\n",
    "                       wordRNN_lstm_dim,\n",
    "                       S_max,\n",
    "                       N_max,\n",
    "                       word_embed_dim,\n",
    "                       bias_init_vector=None):\n",
    "\n",
    "        self.n_words = n_words\n",
    "        self.batch_size = batch_size\n",
    "        self.num_boxes = num_boxes # 50\n",
    "        self.feats_dim = feats_dim # 4096\n",
    "        self.project_dim = project_dim # 1024\n",
    "        self.S_max = S_max # 6\n",
    "        self.N_max = N_max # 50\n",
    "        self.word_embed_dim = word_embed_dim # 1024\n",
    "\n",
    "        self.sentRNN_lstm_dim = sentRNN_lstm_dim # 512 hidden size\n",
    "        self.sentRNN_FC_dim = sentRNN_FC_dim # 1024 in fully connected layer\n",
    "        self.wordRNN_lstm_dim = wordRNN_lstm_dim # 512 hidden size\n",
    "\t\n",
    "\n",
    "\n",
    "        # word embedding, parameters of embedding\n",
    "        # embedding shape: n_words x wordRNN_lstm_dim\n",
    "        with tf.device('/cpu:0'):\n",
    "            self.Wemb = tf.Variable(tf.random_uniform([n_words, word_embed_dim], -0.1, 0.1), name='Wemb')\n",
    "        print(\"Wemb output>>>>>>>>>>>>\",self.Wemb,tf.shape(self.Wemb))\n",
    "        #self.bemb = tf.Variable(tf.zeros([word_embed_dim]), name='bemb')\n",
    "\n",
    "        # regionPooling_W shape: 4096 x 1024\n",
    "        # regionPooling_b shape: 1024\n",
    "        self.regionPooling_W = tf.Variable(tf.random_uniform([feats_dim, project_dim], -0.1, 0.1), name='regionPooling_W')\n",
    "        self.regionPooling_b = tf.Variable(tf.zeros([project_dim]), name='regionPooling_b')\n",
    "\n",
    "        # sentence LSTM\n",
    "        self.sent_LSTM = tf.nn.rnn_cell.BasicLSTMCell(sentRNN_lstm_dim, state_is_tuple=True)\n",
    "\n",
    "        # logistic classifier\n",
    "        self.logistic_Theta_W = tf.Variable(tf.random_uniform([sentRNN_lstm_dim, 2], -0.1, 0.1), name='logistic_Theta_W')\n",
    "        self.logistic_Theta_b = tf.Variable(tf.zeros(2), name='logistic_Theta_b')\n",
    "\n",
    "        # fc1_W: 512 x 1024, fc1_b: 1024\n",
    "        # fc2_W: 1024 x 1024, fc2_b: 1024\n",
    "        self.fc1_W = tf.Variable(tf.random_uniform([sentRNN_lstm_dim, sentRNN_FC_dim], -0.1, 0.1), name='fc1_W')\n",
    "        self.fc1_b = tf.Variable(tf.zeros(sentRNN_FC_dim), name='fc1_b')\n",
    "        self.fc2_W = tf.Variable(tf.random_uniform([sentRNN_FC_dim, 1024], -0.1, 0.1), name='fc2_W')\n",
    "        self.fc2_b = tf.Variable(tf.zeros(1024), name='fc2_b')\n",
    "        def get_a_cell(lstm_size):\n",
    "          lstm = tf.nn.rnn_cell.BasicLSTMCell(lstm_size)\n",
    "          return lstm\n",
    "        # word LSTM\n",
    "        self.word_LSTM = tf.nn.rnn_cell.BasicLSTMCell(wordRNN_lstm_dim, state_is_tuple=True)\n",
    "        #self.word_LSTM = tf.nn.rnn_cell.MultiRNNCell([self.word_LSTM] * 2, state_is_tuple=True)\n",
    "        self.word_LSTM = tf.nn.rnn_cell.MultiRNNCell([get_a_cell(wordRNN_lstm_dim) for i in range(2)], state_is_tuple=True)\n",
    "        #self.word_LSTM2 = tf.nn.rnn_cell.BasicLSTMCell(wordRNN_lstm_dim, state_is_tuple=True)\n",
    "\n",
    "\n",
    "        self.embed_word_W = tf.Variable(tf.random_uniform([wordRNN_lstm_dim, n_words], -0.1,0.1), name='embed_word_W')\n",
    "        if bias_init_vector is not None:\n",
    "            self.embed_word_b = tf.Variable(bias_init_vector.astype(np.float32), name='embed_word_b')\n",
    "        else:\n",
    "            self.embed_word_b = tf.Variable(tf.zeros([n_words]), name='embed_word_b')\n",
    "\n",
    "    def build_model(self):\n",
    "        # receive the feats in the current image\n",
    "        # it's shape is 10 x 50 x 4096\n",
    "        # tmp_feats: 500 x 4096\n",
    "        feats = tf.placeholder(tf.float32, [self.batch_size, self.num_boxes, self.feats_dim])\n",
    "        tmp_feats = tf.reshape(feats, [-1, self.feats_dim])\n",
    "\n",
    "        # project_vec_all: 500 x 4096 * 4096 x 1024 --> 500 x 1024\n",
    "        # project_vec: 10 x 1024\n",
    "        project_vec_all = tf.matmul(tmp_feats, self.regionPooling_W) + self.regionPooling_b\n",
    "        project_vec_all = tf.reshape(project_vec_all, [self.batch_size, 50, self.project_dim])\n",
    "        project_vec = tf.reduce_max(project_vec_all, reduction_indices=1)\n",
    "\n",
    "        # receive the [continue:0, stop:1] lists\n",
    "        # example: [0, 0, 0, 0, 1, 1], it means this paragraph has five sentences\n",
    "        num_distribution = tf.placeholder(tf.int32, [self.batch_size, self.S_max])\n",
    "\n",
    "        # receive the ground truth words, which has been changed to idx use word2idx function\n",
    "        captions = tf.placeholder(tf.int32, [self.batch_size, self.S_max, self.N_max+1])\n",
    "        #print(\"Captions:>>>>>>>>>>>>>>>>>>>>>>>\",captions)\n",
    "        captions_masks = tf.placeholder(tf.float32, [self.batch_size, self.S_max, self.N_max+1])\n",
    "\n",
    "        # ---------------------------------------------------------------------------------------------------------------------\n",
    "        # The method which initialize the state, is refered from below sites:\n",
    "        # 1. http://stackoverflow.com/questions/38241410/tensorflow-remember-lstm-state-for-next-batch-stateful-lstm/38417699\n",
    "        # 2. https://www.tensorflow.org/api_docs/python/rnn_cell/classes_storing_split_rnncell_state#LSTMStateTuple\n",
    "        # 3. https://medium.com/@erikhallstrm/using-the-tensorflow-lstm-api-3-7-5f2b97ca6b73#.u4w9z6h0h\n",
    "        # ---------------------------------------------------------------------------------------------------------------------\n",
    "        sent_state = self.sent_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state = self.word_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state1 = self.word_LSTM1.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #word_state2 = self.word_LSTM2.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "        #sent_state = tf.zeros([self.batch_size, self.sent_LSTM1.state_size])\n",
    "        #word_state1 = tf.zeros([self.batch_size, self.word_LSTM1.state_size])\n",
    "        #word_state2 = tf.zeros([self.batch_size, self.word_LSTM2.state_size])\n",
    "\n",
    "        probs = []\n",
    "        loss = 0.0\n",
    "        loss_sent = 0.0\n",
    "        loss_word = 0.0\n",
    "        lambda_sent = 5.0\n",
    "        lambda_word = 1.0\n",
    "\n",
    "        print('Start build model:')\n",
    "        #----------------------------------------------------------------------------------------------\n",
    "        # Hierarchical RNN: sentence RNN and words RNN\n",
    "        # The word RNN has the max number, N_max = 50, the number in the papar is 50\n",
    "        #----------------------------------------------------------------------------------------------\n",
    "        for i in range(0, self.S_max):\n",
    "            if i > 0:\n",
    "                tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.variable_scope('sent_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                sent_output, sent_state = self.sent_LSTM(project_vec, sent_state)\n",
    "\n",
    "            with tf.name_scope('fc1'):\n",
    "                hidden1 = tf.nn.relu( tf.matmul(sent_output, self.fc1_W) + self.fc1_b )\n",
    "            with tf.name_scope('fc2'):\n",
    "                sent_topic_vec = tf.nn.relu( tf.matmul(hidden1, self.fc2_W) + self.fc2_b )\n",
    "\n",
    "            # sent_state is a tuple, sent_state = (c, h)\n",
    "            # 'c': shape=(1, 512) dtype=float32, 'h': shape=(1, 512) dtype=float32\n",
    "            # The loss here, I refer from the web which is very helpful for me:\n",
    "            # 1. http://stackoverflow.com/questions/34240703/difference-between-tensorflow-tf-nn-softmax-and-tf-nn-softmax-cross-entropy-with\n",
    "            # 2. http://stackoverflow.com/questions/35277898/tensorflow-for-binary-classification\n",
    "            # 3. http://stackoverflow.com/questions/35226198/is-this-one-hot-encoding-in-tensorflow-fast-or-flawed-for-any-reason\n",
    "            # 4. http://stackoverflow.com/questions/35198528/reshape-y-train-for-binary-text-classification-in-tensorflow\n",
    "            sentRNN_logistic_mu = tf.nn.xw_plus_b( sent_output, self.logistic_Theta_W, self.logistic_Theta_b )\n",
    "            sentRNN_label = tf.stack([ 1 - num_distribution[:, i], num_distribution[:, i] ])\n",
    "            sentRNN_label = tf.transpose(sentRNN_label)\n",
    "            sentRNN_loss = tf.nn.softmax_cross_entropy_with_logits(logits=sentRNN_logistic_mu, labels=sentRNN_label)\n",
    "            sentRNN_loss = tf.reduce_sum(sentRNN_loss)/self.batch_size\n",
    "            loss += sentRNN_loss * lambda_sent\n",
    "            loss_sent += sentRNN_loss\n",
    "\n",
    "            # the begining input of word_LSTM is topic vector, and DON'T compute the loss\n",
    "            # This is follow the paper: Show and Tell\n",
    "            #word_state = self.word_LSTM.zero_state(batch_size=self.batch_size, dtype=tf.float32)\n",
    "            #with tf.variable_scope('word_LSTM'):\n",
    "            #    word_output, word_state = self.word_LSTM(sent_topic_vec)\n",
    "            topic = tf.nn.rnn_cell.LSTMStateTuple(sent_topic_vec[:, 0:512], sent_topic_vec[:, 512:])\n",
    "            word_state = (topic, topic)\n",
    "            for j in range(0, self.N_max):\n",
    "                if j > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                with tf.device('/cpu:0'):\n",
    "                    current_embed = tf.nn.embedding_lookup(self.Wemb, captions[:, i, j])\n",
    "\n",
    "                with tf.variable_scope('word_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                    word_output, word_state = self.word_LSTM(current_embed, word_state)\n",
    "\n",
    "                # How to make one-hot encoder, I refer from this excellent web:\n",
    "                # http://stackoverflow.com/questions/33681517/tensorflow-one-hot-encoder\n",
    "                labels = tf.reshape(captions[:, i, j+1], [-1, 1])\n",
    "                #print(\"Labels and its shape +++++++++++++++\",labels,tf.shape(labels))\n",
    "                indices = tf.reshape(tf.range(0, self.batch_size, 1), [-1, 1])\n",
    "                #print(\"Indices and its shape +++++++++++++++\",indices,tf.shape(indices))\n",
    "                concated = tf.concat([indices, labels],1)\n",
    "                #print(\"Concated+++++++++++++++++++\",concated)\n",
    "                print(\"Success\")\n",
    "                onehot_labels = tf.sparse_to_dense(concated, tf.stack([self.batch_size, self.n_words]), 1.0, 0.0)\n",
    "\n",
    "                # At each timestep the hidden state of the last LSTM layer is used to predict a distribution\n",
    "                # over the words in the vocbulary\n",
    "                logit_words = tf.nn.xw_plus_b(word_output[:], self.embed_word_W, self.embed_word_b)\n",
    "                cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logit_words, labels=onehot_labels)\n",
    "                cross_entropy = cross_entropy * captions_masks[:, i, j]\n",
    "                loss_wordRNN = tf.reduce_sum(cross_entropy) / self.batch_size\n",
    "                loss += loss_wordRNN * lambda_word\n",
    "                loss_word += loss_wordRNN\n",
    "\n",
    "        return feats, num_distribution, captions, captions_masks, loss, loss_sent, loss_word\n",
    "\n",
    "    def generate_model(self):\n",
    "        # feats: 1 x 50 x 4096\n",
    "        feats = tf.placeholder(tf.float32, [1, self.num_boxes, self.feats_dim])\n",
    "        # tmp_feats: 50 x 4096\n",
    "        tmp_feats = tf.reshape(feats, [-1, self.feats_dim])\n",
    "\n",
    "        # project_vec_all: 50 x 4096 * 4096 x 1024 + 1024 --> 50 x 1024\n",
    "        project_vec_all = tf.matmul(tmp_feats, self.regionPooling_W) + self.regionPooling_b\n",
    "        project_vec_all = tf.reshape(project_vec_all, [1, 50, self.project_dim])\n",
    "        project_vec = tf.reduce_max(project_vec_all, reduction_indices=1)\n",
    "\n",
    "        # initialize the sent_LSTM state\n",
    "        sent_state = self.sent_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "\n",
    "        # save the generated paragraph to list, here I named generated_sents\n",
    "        generated_paragraph = []\n",
    "\n",
    "        # pred\n",
    "        pred_re = []\n",
    "\n",
    "        # T_stop: run the sentence RNN forward until the stopping probability p_i (STOP) exceeds a threshold T_stop\n",
    "        T_stop = tf.constant(0.5)\n",
    "\n",
    "        # Start build the generation model\n",
    "        print('Start build the generation model: ')\n",
    "\n",
    "        # sentence RNN\n",
    "        #word_state = self.word_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "        #with tf.variable_scope('word_LSTM'):\n",
    "        #    word_output, word_state = self.word_LSTM(sent_topic_vec, word_state)\n",
    "        for i in range(0, self.S_max):\n",
    "            if i > 0:\n",
    "                tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            # sent_state:\n",
    "            # LSTMStateTuple(c=<tf.Tensor 'sent_LSTM/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "            #                h=<tf.Tensor 'sent_LSTM/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>)\n",
    "            with tf.variable_scope('sent_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                sent_output, sent_state = self.sent_LSTM(project_vec, sent_state)\n",
    "\n",
    "            # self.fc1_W: 512 x 1024, self.fc1_b: 1024\n",
    "            # hidden1: 1 x 1024\n",
    "            # sent_topic_vec: 1 x 1024\n",
    "            with tf.name_scope('fc1'):\n",
    "                hidden1 = tf.nn.relu( tf.matmul(sent_output, self.fc1_W) + self.fc1_b )\n",
    "            with tf.name_scope('fc2'):\n",
    "                sent_topic_vec = tf.nn.relu( tf.matmul(hidden1, self.fc2_W) + self.fc2_b )\n",
    "\n",
    "            sentRNN_logistic_mu = tf.nn.xw_plus_b(sent_output, self.logistic_Theta_W, self.logistic_Theta_b)\n",
    "            pred = tf.nn.softmax(sentRNN_logistic_mu)\n",
    "            pred_re.append(pred)\n",
    "\n",
    "            # save the generated sentence to list, named generated_sent\n",
    "            generated_sent = []\n",
    "\n",
    "            # initialize the word LSTM state\n",
    "            #word_state = self.word_LSTM.zero_state(batch_size=1, dtype=tf.float32)\n",
    "            #with tf.variable_scope('word_LSTM'):\n",
    "            #    word_output, word_state = self.word_LSTM(sent_topic_vec, word_state)\n",
    "            topic = tf.nn.rnn_cell.LSTMStateTuple(sent_topic_vec[:, 0:512], sent_topic_vec[:, 512:])\n",
    "            word_state = (topic, topic)\n",
    "            # word RNN, unrolled to N_max time steps\n",
    "            for j in range(0, self.N_max):\n",
    "                if j > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                if j == 0:\n",
    "                    with tf.device('/cpu:0'):\n",
    "                        # get word embedding of BOS (index = 0)\n",
    "                        current_embed = tf.nn.embedding_lookup(self.Wemb, tf.zeros([1], dtype=tf.int64))\n",
    "\n",
    "                with tf.variable_scope('word_LSTM',reuse=tf.AUTO_REUSE):\n",
    "                    word_output, word_state = self.word_LSTM(current_embed, word_state)\n",
    "\n",
    "                # word_state:\n",
    "                # (\n",
    "                #     LSTMStateTuple(c=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell0/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "                #                    h=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell0/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>),\n",
    "                #     LSTMStateTuple(c=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell1/BasicLSTMCell/add_2:0' shape=(1, 512) dtype=float32>,\n",
    "                #                    h=<tf.Tensor 'word_LSTM_152/MultiRNNCell/Cell1/BasicLSTMCell/mul_2:0' shape=(1, 512) dtype=float32>)\n",
    "                # )\n",
    "                logit_words = tf.nn.xw_plus_b(word_output, self.embed_word_W, self.embed_word_b)\n",
    "                max_prob_index = tf.argmax(logit_words, 1)[0]\n",
    "                generated_sent.append(max_prob_index)\n",
    "\n",
    "                with tf.device('/cpu:0'):\n",
    "                    current_embed = tf.nn.embedding_lookup(self.Wemb, max_prob_index)\n",
    "                    current_embed = tf.expand_dims(current_embed, 0)\n",
    "\n",
    "            generated_paragraph.append(generated_sent)\n",
    "\n",
    "        return feats, generated_paragraph, pred_re, sent_topic_vec\n",
    "\n",
    "\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "# Preparing Functions\n",
    "# -----------------------------------------------------------------------------------------------------\n",
    "def preProBuildWordVocab(sentence_iterator, word_count_threshold=5):\n",
    "    # borrowed this function from NeuralTalk\n",
    "    print('preprocessing word counts and creating vocab based on word count threshold %d' % (word_count_threshold, ))\n",
    "\n",
    "    word_counts = {}\n",
    "    nsents = 0\n",
    "\n",
    "    for sent in sentence_iterator:\n",
    "        nsents += 1\n",
    "        tmp_sent = sent.lower().split(' ')\n",
    "        if '' in tmp_sent:\n",
    "            tmp_sent.remove('')\n",
    "\n",
    "        for w in tmp_sent:\n",
    "           word_counts[w] = word_counts.get(w, 0) + 1\n",
    "\n",
    "    vocab = [w for w in word_counts if word_counts[w] >= word_count_threshold]\n",
    "    print('filtered words from %d to %d' % (len(word_counts), len(vocab)))\n",
    "\n",
    "    ixtoword = {}\n",
    "    ixtoword[0] = '<bos>'\n",
    "    ixtoword[1] = '<eos>'\n",
    "    ixtoword[2] = '<pad>'\n",
    "    ixtoword[3] = '<unk>'\n",
    "\n",
    "    wordtoix = {}\n",
    "    wordtoix['<bos>'] = 0\n",
    "    wordtoix['<eos>'] = 1\n",
    "    wordtoix['<pad>'] = 2\n",
    "    wordtoix['<unk>'] = 3\n",
    "\n",
    "    for idx, w in enumerate(vocab):\n",
    "        wordtoix[w] = idx + 4\n",
    "        ixtoword[idx+4] = w\n",
    "\n",
    "    word_counts['<eos>'] = nsents\n",
    "    word_counts['<bos>'] = nsents\n",
    "    word_counts['<pad>'] = nsents\n",
    "    word_counts['<unk>'] = nsents\n",
    "\n",
    "    bias_init_vector = np.array([1.0 * word_counts[ ixtoword[i] ] for i in ixtoword])\n",
    "    bias_init_vector /= np.sum(bias_init_vector) # normalize to frequencies\n",
    "    bias_init_vector = np.log(bias_init_vector)\n",
    "    bias_init_vector -= np.max(bias_init_vector) # shift to nice numeric range\n",
    "\n",
    "    return wordtoix, ixtoword, bias_init_vector\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Parameters Setting\n",
    "#######################################################################################################\n",
    "batch_size = 5 # Being support batch_size\n",
    "num_boxes = 50 # number of Detected regions in each image\n",
    "feats_dim = 4096 # feature dimensions of each regions\n",
    "project_dim = 1024 # project the features to one vector, which is 1024 dimensions\n",
    "\n",
    "sentRNN_lstm_dim = 512 # the sentence LSTM hidden units\n",
    "sentRNN_FC_dim = 1024 # the fully connected units\n",
    "wordRNN_lstm_dim = 512 # the word LSTM hidden units\n",
    "word_embed_dim = 1024 # the learned embedding vectors for the words\n",
    "\n",
    "S_max = 6\n",
    "N_max = 50\n",
    "T_stop = 0.5\n",
    "\n",
    "n_epochs = 500\n",
    "learning_rate = 0.0001\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Word vocubulary and captions preprocessing stage\n",
    "#######################################################################################################\n",
    "img2paragraph = pickle.load(open('./img2paragraph', 'rb'))\n",
    "all_sentences = []\n",
    "for key, paragraph in img2paragraph.items():\n",
    "    for each_sent in paragraph[1]:\n",
    "        each_sent.replace(',', ' ,')\n",
    "        all_sentences.append(each_sent)\n",
    "word2idx, idx2word, bias_init_vector = preProBuildWordVocab(all_sentences, word_count_threshold=2)\n",
    "np.save('./idx2word_batch', idx2word)\n",
    "\n",
    "img2paragraph_modify = {}\n",
    "for img_name, img_paragraph in img2paragraph.items():\n",
    "    img_paragraph_1 = img_paragraph[1]\n",
    "\n",
    "    # img_paragraph_1 is a list\n",
    "    # it may contain the element: '' or ' ', like this:\n",
    "    # [[\"a man is walking\"], [\"the dog is running\"], [\"\"], [\" \"]]\n",
    "    # so, we should remove them ' ' and '' element\n",
    "    if '' in img_paragraph_1:\n",
    "        img_paragraph_1.remove('')\n",
    "    if ' ' in paragraph[1]:\n",
    "        img_paragraph_1.remove(' ')\n",
    "\n",
    "    # the number sents in each paragraph\n",
    "    # if the sents is bigger than S_max,\n",
    "    # we force the number of sents to be S_max\n",
    "    img_num_sents = len(img_paragraph_1)\n",
    "    if img_num_sents > S_max:\n",
    "        img_num_sents = S_max\n",
    "\n",
    "    # if a paragraph has 4 sentences\n",
    "    # then the img_num_distribution will be like this:\n",
    "    # [0, 0, 0, 1, 1, 1]\n",
    "    img_num_distribution = np.zeros([S_max], dtype=np.int32)\n",
    "    img_num_distribution[img_num_sents-1:] = 1\n",
    "\n",
    "    # we multiply the number 2, because the <pad> is encoded into 2\n",
    "    img_captions_matrix = np.ones([S_max, N_max+1], dtype=np.int32) * 2 # zeros([6, 50])\n",
    "    for idx, img_sent in enumerate(img_paragraph_1):\n",
    "        # the number of sentences is img_num_sents\n",
    "        if idx == img_num_sents:\n",
    "            break\n",
    "\n",
    "        # because we treat the ',' as a word\n",
    "        img_sent = img_sent.replace(',', ' ,')\n",
    "\n",
    "        # Because I have preprocess the paragraph_v1.json file in VScode before,\n",
    "        # and I delete all the 2, 3, 4...bankspaces\n",
    "        # so, actually, the 'elif' code will never run\n",
    "        if img_sent[0] == ' ' and img_sent[1] != ' ':\n",
    "            img_sent = img_sent[1:]\n",
    "        elif img_sent[0] == ' ' and img_sent[1] == ' ' and img_sent[2] != ' ':\n",
    "            img_sent = img_sent[2:]\n",
    "\n",
    "        # Be careful the last part in a sentence, like this:\n",
    "        # '...world.'\n",
    "        # '...world. '\n",
    "        if img_sent[-1] == '.':\n",
    "            img_sent = img_sent[0:-1]\n",
    "        elif img_sent[-1] == ' ' and img_sent[-2] == '.':\n",
    "            img_sent = img_sent[0:-2]\n",
    "\n",
    "        # Last, we add the <bos> and the <eos> in each sentences\n",
    "        img_sent = '<bos> ' + img_sent + ' <eos>'\n",
    "\n",
    "        # translate each word in a sentence into the unique number in word2idx dict\n",
    "        # when we meet the word which is not in the word2idx dict, we use the mark: <unk>\n",
    "        for idy, word in enumerate(img_sent.lower().split(' ')):\n",
    "            # because the biggest number of words in a sentence is N_max, here is 50\n",
    "            if idy == N_max:\n",
    "                break\n",
    "\n",
    "            if word in word2idx:\n",
    "                img_captions_matrix[idx, idy] = word2idx[word]\n",
    "            else:\n",
    "                img_captions_matrix[idx, idy] = word2idx['<unk>']\n",
    "\n",
    "    # Pay attention, the value type 'img_name' here is NUMBER, I change it to STRING type\n",
    "    img2paragraph_modify[str(img_name)] = [img_num_distribution, img_captions_matrix]\n",
    "\n",
    "with open('./img2paragraph_modify_batch', 'wb') as f:\n",
    "    pickle.dump(img2paragraph_modify, f)\n",
    "\n",
    "\n",
    "#######################################################################################################\n",
    "# Train, validation and testing stage\n",
    "#######################################################################################################\n",
    "def train():\n",
    "    ##############################################################################\n",
    "    # some preparing work\n",
    "    ##############################################################################\n",
    "    model_path = './models_batch/'\n",
    "    train_feats_path = './im2p_val_output.h5'\n",
    "    train_output_file = h5py.File(train_feats_path, 'r')\n",
    "    train_feats = train_output_file.get('feats')\n",
    "    train_imgs_full_path_lists = open('./imgs_val_path.txt').read().splitlines()\n",
    "    train_imgs_names = list(map(lambda x: os.path.basename(x).split('.')[0], train_imgs_full_path_lists))\n",
    "    print(\"Train Images Names\",train_imgs_names)\n",
    "\n",
    "    # Model Initialization:\n",
    "    # n_words, batch_size, num_boxes, feats_dim, project_dim, sentRNN_lstm_dim, sentRNN_FC_dim, wordRNN_lstm_dim, S_max, N_max\n",
    "    with tf.variable_scope(tf.get_variable_scope()) as scope:\n",
    "        model = RegionPooling_HierarchicalRNN(n_words = len(word2idx),\n",
    "                                              batch_size = batch_size,\n",
    "                                              num_boxes = num_boxes,\n",
    "                                              feats_dim = feats_dim,\n",
    "                                              project_dim = project_dim,\n",
    "                                              sentRNN_lstm_dim = sentRNN_lstm_dim,\n",
    "                                              sentRNN_FC_dim = sentRNN_FC_dim,\n",
    "                                              wordRNN_lstm_dim = wordRNN_lstm_dim,\n",
    "                                              S_max = S_max,\n",
    "                                              N_max = N_max,\n",
    "                                              word_embed_dim = word_embed_dim,\n",
    "                                              bias_init_vector = bias_init_vector)\n",
    "\n",
    "        tf_feats, tf_num_distribution, tf_captions_matrix, tf_captions_masks, tf_loss, tf_loss_sent, tf_loss_word = model.build_model()\n",
    "    #sess = tf.Session()\n",
    "    with tf.Session() as sess:\n",
    "        saver = tf.train.Saver(max_to_keep=500, write_version=1)\n",
    "        with tf.variable_scope('optimizer',reuse= tf.AUTO_REUSE):\n",
    "            train_op = tf.train.AdamOptimizer(learning_rate).minimize(tf_loss)\n",
    "        tf.global_variables_initializer().run(session=sess)\n",
    "        print(\"Done w session\")\n",
    "\n",
    "        # when you want to train the model from the front model\n",
    "        #new_saver = tf.train.Saver(max_to_keep=500)\n",
    "        #new_saver = tf.train.import_meta_graph('./models_batch/model-92.meta')\n",
    "        #new_saver.restore(sess, tf.train.latest_checkpoint('./models_batch/'))\n",
    "\n",
    "        all_vars = tf.trainable_variables()\n",
    "\n",
    "        # open a loss file to record the loss value\n",
    "        #loss_fd = open('loss_batch.txt', 'a')\n",
    "        img2idx = {}\n",
    "        for idx, img in enumerate(train_imgs_names):\n",
    "            img2idx[img] = idx\n",
    "        print(\"Img2idx\",img2idx)\n",
    "\n",
    "        # plt draw the loss curve\n",
    "        # refer from: http://stackoverflow.com/questions/11874767/real-time-plotting-in-while-loop-with-matplotlib\n",
    "        \n",
    "        loss_to_draw = [315.90247,265.50665,243.4754,229.64827,219.55573,211.94145,204.5995,197.81664,192.10925,186.46211,180.70108,175.55612,170.3046,165.09123,160.21469,160.2537961546135,154.08637935852528,149.32272485827923,144.4875265452814,139.37248678689718,134.57183133995056,129.8052043039155,125.52713392114163,120.7494116071415,115.9398527387333,111.30544829552889,106.93300072288037,102.37911339288473,97.98938349818707,93.8994490746212,89.69688967054606,84.13689,79.04315,75.386566,71.634445,67.22706,63.297745,59.791893,58.797394,54.47476,50.78446,47.608204,44.824448,42.131664,39.22608,36.267944,33.43973,30.94212,29.161013,27.789131,25.733274,23.194603,21.0445,19.291529,18.102518,17.157877,17.008978,14.93577,13.9712105,12.671065,11.739764,10.853795,10.301606,9.915598,9.00906,8.191635,7.7371345,7.485827,7.1387715,7.2393036,7.0062246,6.03907,5.117557,4.7625046,4.913513,4.82957,5.0153527,4.8877826,4.4756083,3.8244674,7.6418276,7.2976747,6.7312045,6.5270514,6.6875095,5.9723153,4.704945,4.7736664,5.8504386,5.804864,4.7677197,3.7789633,3.6227777,3.7564394,4.2505994,4.3439636,3.6212943,3.0515492,3.0061777,3.1802363,3.4463785,3.1947653,2.5623198,3.1410942,3.497574,3.2585106,2.8631153,315.3198,264.98257,243.43536,230.01776,219.96605,211.50975]  #Have to hard code here once it crashes\n",
    "        print(\"Before epoch loop->>>>>>>>>>>>>>>>>>>>>>>>>>>>> the len is\",len(loss_to_draw))\n",
    "        try:\n",
    "            saver.restore(sess, './models_batch/model-112')\n",
    "            print(\"pretrained model loaded successfully\")\n",
    "        except:\n",
    "            print(\"fail to load pretrained model\")\n",
    "            pass\n",
    "        \n",
    "        \n",
    "        for epoch in range(113, n_epochs):\n",
    "            loss_to_draw_epoch = []\n",
    "            # disorganize the order\n",
    "            random.shuffle(train_imgs_names)\n",
    "\n",
    "            for start, end in zip(range(0, len(train_imgs_names), batch_size),\n",
    "                                  range(batch_size, len(train_imgs_names), batch_size)):\n",
    "                loss_fd = open('loss_batch.txt', 'a')\n",
    "\n",
    "                start_time = time.time()\n",
    "                print(\"Start and end\",(start,end))\n",
    "                img_name = train_imgs_names[start:end]\n",
    "                current_feats_index = map(lambda x: img2idx[x], img_name)\n",
    "                current_feats = np.asarray( list(map(lambda x: train_feats[x], current_feats_index) ))\n",
    "\n",
    "                current_num_distribution = np.asarray( list(map(lambda x: img2paragraph_modify[x][0], img_name) ))\n",
    "                current_captions_matrix = np.asarray( list(map(lambda x: img2paragraph_modify[x][1], img_name) ))\n",
    "\n",
    "                current_captions_masks = np.zeros( (current_captions_matrix.shape[0], current_captions_matrix.shape[1], current_captions_matrix.shape[2]) )\n",
    "                # find the non-zero element\n",
    "                nonzeros = np.array( list(map(lambda each_matrix: np.array( list(map(lambda x: (x != 2).sum() + 1, each_matrix ) )), current_captions_matrix ) ))\n",
    "                for i in range(batch_size):\n",
    "                    for ind, row in enumerate(current_captions_masks[i]):\n",
    "                        row[:(nonzeros[i, ind]-1)] = 1\n",
    "\n",
    "                # shape of current_feats: batch_size x 50 x 4096\n",
    "                # shape of current_num_distribution: batch_size x 6\n",
    "                # shape of current_captions_matrix: batch_size x 6 x 50\n",
    "                _, loss_val, loss_sent, loss_word= sess.run(\n",
    "                                    [train_op, tf_loss, tf_loss_sent, tf_loss_word],\n",
    "                                    feed_dict={\n",
    "                                               tf_feats: current_feats,\n",
    "                                               tf_num_distribution: current_num_distribution,\n",
    "                                               tf_captions_matrix: current_captions_matrix,\n",
    "                                               tf_captions_masks: current_captions_masks\n",
    "                                    })\n",
    "\n",
    "                # append loss to list in a epoch\n",
    "                loss_to_draw_epoch.append(loss_val)\n",
    "\n",
    "                # running information\n",
    "                print('idx: ', start, ' Epoch: ', epoch, ' loss: ', loss_val, ' loss_sent: ', loss_sent, ' loss_word: ', loss_word, ' Time cost: ', str((time.time() - start_time)))\n",
    "                loss_fd.write('start: '+ str(start) +' end: ' + str(end) +' epoch ' + str(epoch) + ' loss ' + str(loss_val) + '\\n')\n",
    "                loss_fd.close()\n",
    "            # draw loss curve every epoch\n",
    "            loss_to_draw.append(np.mean(loss_to_draw_epoch)) #This is what weve to write in a temp file once it is about to crash\n",
    "            plt_save_dir = './loss_imgs'\n",
    "            plt_save_img_name = str(epoch) + '.png'\n",
    "            plt.plot(range(len(loss_to_draw)), loss_to_draw, color='g')\n",
    "            plt.grid(True)\n",
    "            plt.savefig(os.path.join(plt_save_dir, plt_save_img_name))\n",
    "            MyFile=open('temporary.txt','a')\n",
    "            if np.mod(epoch, 1) == 0:\n",
    "                #the list loss_to_draw needs to be saved in a temp file\n",
    "                l=loss_to_draw\n",
    "                MyList=map(lambda x:str(x)+',', l)\n",
    "                MyFile.writelines(MyList)\n",
    "                MyFile.write('\\t'+'<><><><><><><><><><><><><>'+'\\n')\n",
    "                MyFile.close()\n",
    "                print(\"Successfully Written to temporary\")\n",
    "                print(\"Epoch \", epoch, \" is done. Saving the model ...\")\n",
    "                saver.save(sess, os.path.join(model_path, 'model'), global_step=epoch)\n",
    "        #loss_fd.close()\n",
    "315.90247,265.50665,243.4754,229.64827,219.55573,211.94145,204.5995,197.81664,192.10925,186.46211,180.70108,175.55612,170.3046,165.09123,160.21469,160.2537961546135,154.08637935852528,149.32272485827923,144.4875265452814,139.37248678689718,134.57183133995056,129.8052043039155,125.52713392114163,120.7494116071415,115.9398527387333,111.30544829552889,106.93300072288037,102.37911339288473,97.98938349818707,93.8994490746212,89.69688967054606,84.13689,79.04315,75.386566,71.634445,67.22706,63.297745,59.791893,58.797394,54.47476,50.78446,47.608204,44.824448,42.131664,39.22608,36.267944,33.43973,30.94212,29.161013,27.789131,25.733274,23.194603,21.0445,19.291529,18.102518,17.157877,17.008978,14.93577,13.9712105,12.671065,11.739764,10.853795,10.301606,9.915598,9.00906,8.191635,7.7371345,7.485827,7.1387715,7.2393036,7.0062246,6.03907,5.117557,4.7625046,4.913513,4.82957,5.0153527,4.8877826,4.4756083,3.8244674,7.6418276,7.2976747,6.7312045,6.5270514,6.6875095,5.9723153,4.704945,4.7736664,5.8504386,5.804864,4.7677197,3.7789633,3.6227777,3.7564394,4.2505994,4.3439636,3.6212943,3.0515492,3.0061777,3.1802363,3.4463785,3.1947653,2.5623198,3.1410942,3.497574,3.2585106,2.8631153\n",
    "\n",
    "def test():\n",
    "    \n",
    "    start_time = time.time()\n",
    "    # change the model path according to your environment\n",
    "    model_path = './model-250'\n",
    "\n",
    "    # It's very important to use Pandas to Series this idx2word dict\n",
    "    # After this operation, we can use list to extract the word at the same time\n",
    "    idx2word = pd.Series(np.load('./data/idx2word_batch.npy').tolist())\n",
    "\n",
    "    test_feats_path = './data/im2p_test_output.h5'\n",
    "    test_output_file = h5py.File(test_feats_path, 'r')\n",
    "    test_feats = test_output_file.get('feats')\n",
    "\n",
    "    test_imgs_full_path_lists = open('./densecap/imgs_test_path.txt').read().splitlines()\n",
    "    test_imgs_names = map(lambda x: os.path.basename(x).split('.')[0], test_imgs_full_path_lists)\n",
    "    \n",
    "    # n_words, batch_size, num_boxes, feats_dim, project_dim, sentRNN_lstm_dim, sentRNN_FC_dim, wordRNN_lstm_dim, S_max, N_max\n",
    "    test_model = RegionPooling_HierarchicalRNN(n_words = len(word2idx),\n",
    "                                               batch_size = batch_size,\n",
    "                                               num_boxes = num_boxes,\n",
    "                                               feats_dim = feats_dim,\n",
    "                                               project_dim = project_dim,\n",
    "                                               sentRNN_lstm_dim = sentRNN_lstm_dim,\n",
    "                                               sentRNN_FC_dim = sentRNN_FC_dim,\n",
    "                                               wordRNN_lstm_dim = wordRNN_lstm_dim,\n",
    "                                               S_max = S_max,\n",
    "                                               N_max = N_max,\n",
    "                                               word_embed_dim = word_embed_dim,\n",
    "                                               bias_init_vector = bias_init_vector)\n",
    "    \n",
    "\n",
    "    tf_feats, tf_generated_paragraph, tf_pred_re, tf_sent_topic_vectors = test_model.generate_model()\n",
    "    sess = tf.InteractiveSession()\n",
    "    #print(\"Before there >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\\n\\n\\n\")\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, model_path)\n",
    "\n",
    "    img2idx = {}\n",
    "    for idx, img in enumerate(test_imgs_names):\n",
    "        img2idx[img] = idx\n",
    "    print(\"IM2idx:::::::::::\",img2idx)\n",
    "\n",
    "    test_fd = open('HRNN_results.txt', 'w')\n",
    "    for idx, img_name in enumerate(test_imgs_names):\n",
    "        print(idx, img_name)\n",
    "        test_fd.write(img_name + '\\n')\n",
    "\n",
    "        each_paragraph = []\n",
    "        current_paragraph = \"\"\n",
    "\n",
    "        current_feats_index = img2idx[img_name]\n",
    "        current_feats = test_feats[current_feats_index]\n",
    "        current_feats = np.reshape(current_feats, [1, 50, 4096])\n",
    "\n",
    "        generated_paragraph_indexes, pred, sent_topic_vectors = sess.run(\n",
    "                                                                         [tf_generated_paragraph, tf_pred_re, tf_sent_topic_vectors],\n",
    "                                                                         feed_dict={\n",
    "                                                                             tf_feats: current_feats\n",
    "                                                                         })\n",
    "\n",
    "        #generated_paragraph = idx2word[generated_paragraph_indexes]\n",
    "        for sent_index in generated_paragraph_indexes:\n",
    "            each_sent = []\n",
    "            for word_index in sent_index:\n",
    "                each_sent.append(idx2word[word_index])\n",
    "            each_paragraph.append(each_sent)\n",
    "\n",
    "        for idx, each_sent in enumerate(each_paragraph):\n",
    "            # if the current sentence is the end sentence of the paragraph\n",
    "            # According to the probability distribution:\n",
    "            # CONTINUE: [1, 0]\n",
    "            # STOP    : [0, 1]\n",
    "            # So, if the first item of pred is less than the T_stop\n",
    "            # the generation process is break\n",
    "            if pred[idx][0][0] <= T_stop:\n",
    "                break\n",
    "            current_sent = ''\n",
    "            for each_word in each_sent:\n",
    "                current_sent += each_word + ' '\n",
    "            current_sent = current_sent.replace('<eos> ', '')\n",
    "            current_sent = current_sent.replace('<pad> ', '')\n",
    "            current_sent = current_sent + '.'\n",
    "            current_sent = current_sent.replace(' .', '.')\n",
    "            current_sent = current_sent.replace(' ,', ',')\n",
    "            current_paragraph +=current_sent\n",
    "            if idx != len(each_paragraph) - 1:\n",
    "                current_paragraph += ' '\n",
    "\n",
    "        test_fd.write(current_paragraph + '\\n')\n",
    "    test_fd.close()\n",
    "    print(\"Time cost: \" + str(time.time()-start_time))\n",
    "\n",
    "train()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}