image_captioning_train_google_colab_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fzVndc0TAcv_"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "vocab_size = 1848\n",
    "max_len = 35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "oL_WagMr-ZWr",
    "outputId": "0fed02e0-b290-4c4f-b1cf-6a1568403077"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "N8E9gDNnBHax"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "encoding_train = pickle.load(open(\"encoded_train_features.pkl\", \"rb\"))\n",
    "encoding_test = pickle.load(open(\"encoded_test_features.pkl\", \"rb\"))\n",
    "embedding_matrix = pickle.load(open(\"embedding_matrix.pkl\", \"rb\"))\n",
    "word_to_idx = pickle.load(open(\"word_to_idx\", \"rb\"))\n",
    "train_descriptions = pickle.load(open(\"train_descriptions.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oxrf9xgSB8Vo"
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Uwjo76JLCExq"
   },
   "outputs": [],
   "source": [
    "# generator remembers the state of the function in the previous call\n",
    "def data_generator(train_descriptions, encoding_train, word_to_idx, max_len, batch_size):\n",
    "    X1, X2, y = [], [], []\n",
    "    n = 0\n",
    "    while True:\n",
    "        for key, desc_list in train_descriptions.items():\n",
    "            n += 1\n",
    "            photo = encoding_train[key]\n",
    "            for desc in desc_list:\n",
    "                seq = [word_to_idx[word] for word in desc.split() if word in word_to_idx]\n",
    "                for i in range(1, len(seq)):\n",
    "                    xi = seq[0:i]\n",
    "                    yi = seq[i]\n",
    "                    xi = pad_sequences([xi], maxlen = max_len, value = 0, padding= 'post')[0] # value=0 denotes the padding word\n",
    "                    yi = to_categorical([yi], num_classes = vocab_size)[0] # one hot encoding of the word corresponding to y\n",
    "                    \n",
    "                    # make a mini batch\n",
    "                    X1.append(photo)\n",
    "                    X2.append(xi)\n",
    "                    y.append(yi)\n",
    "                    \n",
    "                    if n == batch_size:\n",
    "                        yield ([np.array(X1), np.array(X2)], np.array(y))\n",
    "                        \n",
    "                        # prepare for next iteration\n",
    "                        X1, X2, y = [],[],[]\n",
    "                        n = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "v4F46vPLCHCg"
   },
   "outputs": [],
   "source": [
    "from keras.layers import Input, Dense, Dropout, Embedding, LSTM, Add\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 222
    },
    "colab_type": "code",
    "id": "XaHpMq9xCMGX",
    "outputId": "d3a498b2-e602-4f37-b038-e48015d97314"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "input_img_features = Input(shape=(2048,))\n",
    "inp_img1 = Dropout(0.3)(input_img_features)\n",
    "inp_img2 = Dense(256, activation='relu')(inp_img1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 87
    },
    "colab_type": "code",
    "id": "ThYAasYACNxK",
    "outputId": "42e699be-bb91-407b-ee88-b0d280219912"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3239: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "input_captions = Input(shape=(35,))\n",
    "inp_cap1 = Embedding(input_dim = vocab_size, output_dim = 50, mask_zero = True)(input_captions)\n",
    "inp_cap2 = Dropout(0.3)(inp_cap1)\n",
    "inp_cap3 = LSTM(256)(inp_cap2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TLfGlgrtCQB-"
   },
   "outputs": [],
   "source": [
    "decoder1 = Add()([inp_img2, inp_cap3])\n",
    "decoder2 = Dense(256, activation='relu')(decoder1)\n",
    "outputs = Dense(vocab_size, activation = 'softmax')(decoder2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SJV5X2ljCSVC"
   },
   "outputs": [],
   "source": [
    "model = Model(inputs = [input_img_features, input_captions], outputs=outputs) # combined model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 238
    },
    "colab_type": "code",
    "id": "hgbs9HNWCU1y",
    "outputId": "bed4050c-df70-41e6-9d42-366c4608ffed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:203: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.layers[2].set_weights([embedding_matrix])\n",
    "model.layers[2].trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 104
    },
    "colab_type": "code",
    "id": "4AYiynXICdCR",
    "outputId": "a4aa2aa9-5179-4e4a-ce9d-79d6ca0c848f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3576: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss = \"categorical_crossentropy\", optimizer= \"adam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YUmuA6KdCei4"
   },
   "outputs": [],
   "source": [
    "epochs = 20\n",
    "batch_size = 3\n",
    "steps = len(train_descriptions)//batch_size\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "qirSrD59IQw_",
    "outputId": "3099ed5a-f290-4e77-a4a7-e0ccda96d18d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AeWwi88XC1kZ"
   },
   "outputs": [],
   "source": [
    "def train():\n",
    "    for i in range(epochs):\n",
    "        generator = data_generator(train_descriptions, encoding_train, word_to_idx, max_len, batch_size) # generator instance\n",
    "        model.fit_generator(generator, epochs = 1, steps_per_epoch = steps, verbose=1)\n",
    "        model.save('gdrive/My Drive/model_' + str(i) + '.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 776
    },
    "colab_type": "code",
    "id": "f0_GXYFHC4Yd",
    "outputId": "956908e4-16e7-4f5e-ffe2-f87742b8e358"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
      "\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
      "\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 237s 118ms/step - loss: 4.3023\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 235s 117ms/step - loss: 3.5813\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 235s 118ms/step - loss: 3.3214\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 236s 118ms/step - loss: 3.1630\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 237s 119ms/step - loss: 3.0499\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 235s 118ms/step - loss: 2.9649\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 236s 118ms/step - loss: 2.8962\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 234s 117ms/step - loss: 2.8387\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 235s 118ms/step - loss: 2.7950\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 235s 118ms/step - loss: 2.7527\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 236s 118ms/step - loss: 2.7147\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 236s 118ms/step - loss: 2.6887\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 235s 118ms/step - loss: 2.6589\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 235s 118ms/step - loss: 2.6368\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 236s 118ms/step - loss: 2.6147\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 242s 121ms/step - loss: 2.5932\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 239s 120ms/step - loss: 2.5802\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 240s 120ms/step - loss: 2.5636\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 237s 119ms/step - loss: 2.5480\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 238s 119ms/step - loss: 2.5368\n"
     ]
    }
   ],
   "source": [
    "train()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "image captioning train 4.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}