MLP_FarsiDigits.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MLP-FarsiDigits.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "wKmd8ZLJl_Gu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "21dd1379-6fbd-421a-ba82-711097c57ab4"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ypCb1ju9mOdS",
        "outputId": "e238f289-1142-4ba2-ae67-c31373c9129e"
      },
      "source": [
        "!unzip /content/drive/MyDrive/HodaDatasetReader-master.zip "
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/drive/MyDrive/HodaDatasetReader-master.zip\n",
            "728dd5465fcf38c3c82a1d3fda721da30b2e70ed\n",
            "   creating: HodaDatasetReader-master/\n",
            " extracting: HodaDatasetReader-master/.gitignore  \n",
            "   creating: HodaDatasetReader-master/DigitDB/\n",
            "  inflating: HodaDatasetReader-master/DigitDB/RemainingSamples.cdb  \n",
            "  inflating: HodaDatasetReader-master/DigitDB/Test 20000.cdb  \n",
            "  inflating: HodaDatasetReader-master/DigitDB/Train 60000.cdb  \n",
            "  inflating: HodaDatasetReader-master/Farsi_Digits_Sample_1.gif  \n",
            "  inflating: HodaDatasetReader-master/Farsi_Digits_Sample_2.gif  \n",
            "  inflating: HodaDatasetReader-master/Figure_1.png  \n",
            "  inflating: HodaDatasetReader-master/Figure_2.png  \n",
            "  inflating: HodaDatasetReader-master/HodaDatasetReader.py  \n",
            "  inflating: HodaDatasetReader-master/LICENSE  \n",
            "  inflating: HodaDatasetReader-master/README.md  \n",
            "  inflating: HodaDatasetReader-master/dataset_PRL.pdf  \n",
            "  inflating: HodaDatasetReader-master/main.py  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WRukxH9hwklc"
      },
      "source": [
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.layers import Dropout\n",
        "from keras.utils import np_utils\n",
        "from sklearn.metrics import classification_report\n",
        "from HodaDatasetReader.HodaDatasetReader import *\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "import random\n",
        "import numpy"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "46Gcw2gKwkil"
      },
      "source": [
        "# Get the Train and Test data\n",
        "DATA_PATH = '/content/HodaDatasetReader/DigitDB/'\n",
        "\n",
        "TRAIN_PATH = DATA_PATH + 'Train 60000.cdb'\n",
        "\n",
        "TEST_PATH = DATA_PATH + 'Test 20000.cdb'"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W-YFRcUBwkcy",
        "outputId": "e7d57479-0452-459c-936d-7a5877999b72"
      },
      "source": [
        "X_train, Y_train = read_hoda_dataset(dataset_path=TRAIN_PATH,\n",
        "                                    images_height=32,\n",
        "                                    images_width=32,\n",
        "                                    one_hot=False,\n",
        "                                    reshape=True)\n",
        "\n",
        "X_test, Y_test = read_hoda_dataset(dataset_path=TEST_PATH,\n",
        "                                  images_height=32,\n",
        "                                  images_width=32,\n",
        "                                  one_hot=False,\n",
        "                                  reshape=True)\n",
        "\n",
        "print(X_train.shape, Y_train.shape)\n",
        "print(X_test.shape, Y_test.shape)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(60000, 1024) (60000,)\n",
            "(20000, 1024) (20000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163
        },
        "id": "8zY4U57IwkZR",
        "outputId": "57fe6629-dfbb-4a9a-f061-cd43255144f9"
      },
      "source": [
        "# Plot the Train and Test data\n",
        "img1 = plt.subplot(221)\n",
        "train_index = random.randint(0, len(X_train))\n",
        "test_index = random.randint(0, len(X_test))\n",
        "img1.set_title('Train')\n",
        "plt.imshow(X_train[train_index].reshape(32,32))\n",
        "img2 = plt.subplot(222)\n",
        "plt.imshow(X_train[test_index].reshape(32,32))\n",
        "img2.set_title('Test')\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAACSCAYAAAA6uG1VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAALaElEQVR4nO3da4xcdRnH8e+PUlqhKK2SUspFDKghUTFpwLsIEosxKS+IkRisCaYxEQOJL2jkBZpIgsYAGkhMDZdGiA0BDI1RKzZ4S9QU0ai04SJKLLZcBEIVRC4/X8xZXUpnd2Znzpk55//7JJs9c87Mnme6zzx9/v9zWdkmIqLrDpp0ABERTUixi4gipNhFRBFS7CKiCCl2EVGEFLuIKEKK3RhI+qGk9ZOOIyL6K7bYSfrnrK+XJT036/Enh/lZts+2vbmuWCMWYpw5Xv28n0r6TB2xNuHgSQcwKbaXzSxL+ivwGds/2f95kg62/WKTsUWMw6A5XopiO7t+JJ0uabekSyTtBW6QtFzS9yU9LumpavmYWa/53/94kj4t6ZeSvl499y+Szp7YG4rYj6SDJG2U9GdJ/5B0i6QV1balkm6q1j8taYeklZIuB94PXFN1htdM9l0ML8XuwI4CVgDHAxvo/TvdUD0+DngOmOuXfRpwH/AG4GvAdZJUZ8ARQ/g8cA7wQeBo4Cng2mrbeuB1wLHA64HPAs/ZvhT4BXCh7WW2L2w86hGl2B3Yy8Bltp+3/Zztf9i+zfaztvcBl9NLlH4etv1t2y8Bm4FVwMoG4o4YxGeBS23vtv088CXgXEkHAy/QK3In2n7J9m9tPzPBWMem2Dm7eTxu+98zDyQdClwFrAWWV6sPl7SoKmj72zuzYPvZqqlbdoDnRUzC8cD3JL08a91L9P5D/g69rm6LpCOAm+gVxheaD3O80tkd2P63gvkC8BbgNNuvBT5Qrc/QNNrob8DZto+Y9bXU9iO2X7D9ZdsnA+8BPgZ8qnpdq2+RlGI3mMPpzdM9XU3kXjbheCJG8S3gcknHA0g6UtK6avlDkt4maRHwDL1h7UwH+CjwpkkEPA4pdoO5GngN8ATwa+BHkw0nYiTfALYCP5a0j15On1ZtOwq4lV6h2wX8jN7QduZ151ZnGXyz2ZBHp9y8MyJKkM4uIoqQYhcRRUixi4gijFTsJK2VdJ+kByVtHFdQEZOW3O6eBR+gqA5N3w+cBewGdgDn2d45vvAimpfc7qZRrqA4FXjQ9kMAkrYA64C+CXGIlngph42wy7K9+e3P9t12/x8OHepn7eOpJ2wfOWpMHTVUbievp8dceT1KsVtN70zsGbv5/7k6B7SUwzhNZ46wy7Jt2/b7vts+cvQpQ/2sn/jWh0eNp8OGyu3k9fSYK69rvzZW0gZ6dw5hKcN1HxHTKnndPqMUu0foXTA845hq3SvY3gRsAnitVuQM5jEatpuLgc2b28nr9hnlaOwO4CRJJ0g6BPgEvUtQItouud1BC+7sbL8o6UJgG7AIuN72vWOLLGJCktvdNNKcne0fAD8YUyyvsu3v45uQjxhG3bkdzcsVFBFRhBS7iChCa2/LPnuI2+UhbSnvM+rTbzqotHxKZxcRRUixi4gitHYY22VzHYWOGMQgOVTaFEk6u4goQopdRBRhqoexs1vrudryLrTjGbrGqJJDc0tnFxFFSLGLiCKk2EVEEaZ6zq7rBpljaescZEy/0nIrnV1EFCHFLiKK0Jph7P4td78hYBdOQ4kY1LCnm5T8mUhnFxFFSLGLiCK0ZhjbFTnLPZrWxNC1DdNH6ewioggpdhFRhNYOYwe9SUAbTeswIGK2tp0RMW9nJ+l6SY9J+tOsdSsk3Snpger78nrDjBi/5HZZBhnG3gis3W/dRmC77ZOA7dXjiLa5keR2MeYtdrZ/Djy53+p1wOZqeTNwzpjjGottf//9/74i9tfm3I7hLXTObqXtPdXyXmBlvydK2gBsAFjKoQvcXURjBsrt5HX7jHw01rYBz7F9k+01ttcsZsmou4tozFy5nbxun4V2do9KWmV7j6RVwGPjDKprciunVpn63B52WmacR0fbPCW00M5uK7C+Wl4P3DGecCImLrndUYOcevJd4FfAWyTtlnQBcAVwlqQHgA9XjyNaJbldlnmHsbbP67PpzDHHEtGo5HZ/bR6u9pPLxSKiCCl2EVGE1l4bO9s0XiebI7BRl1HyfVo+H5OQzi4iipBiFxFF6MQwdhDTetuZiFFM4xTOtH6+0tlFRBFS7CKiCMUMYyO6bhqHtNMknV1EFCHFLiKK0Llh7CRb+ZxIHNMiQ9pXS2cXEUVIsYuIIqTYRUQROjdnFxGvtJB54mHn+dowF53OLiKKkGIXEUXIMLYBbWjxI7ounV1EFCHFLiKKUOQwNn80OKI8g/zd2GMl3SVpp6R7JV1UrV8h6U5JD1Tfl9cfbsT4JLfLMsgw9kXgC7ZPBt4FfE7SycBGYLvtk4Dt1eOINkluF2SQP5K9B9hTLe+TtAtYDawDTq+ethn4KXBJLVFOkQxbuyO5XZah5uwkvRF4J/AbYGWVLAB7gZV9XrMB2ACwlEMXGmdErYbN7eR1+wx8NFbSMuA24GLbz8zeZtuAD/Q625tsr7G9ZjFLRgo2og4Lye3kdfsM1NlJWkwvGW62fXu1+lFJq2zvkbQKeKyuIOtU118dy4nE7dDl3I5XGuRorIDrgF22r5y1aSuwvlpeD9wx/vAi6pPcLssgnd17gfOBP0qaaYO+CFwB3CLpAuBh4OP1hBhRm+R2QQY5GvtLQH02nznecCKak9wuSy4Xi4gipNhFRBGKvDZ2WIOeSJwjsBHTK51dRBQhxS4iitDpYWz+UHBEzEhnFxFFSLGLiCJ0ehg7rAx1I7ornV1EFCHFLiKKUMwwNkdmI8arbSfRp7OLiCKk2EVEEVLsIqIIxczZ1aVt8xYRpUpnFxFFSLGLiCIUOYzNaSgRc+viZySdXUQUIcUuIoow7zBW0lLg58CS6vm32r5M0gnAFuD1wG+B823/p85gp0GOvnZHcnswXcn5QTq754EzbL8DOAVYK+ldwFeBq2yfCDwFXFBfmBG1SG4XZN5i555/Vg8XV18GzgBurdZvBs6pJcKImiS3yzLQ0VhJi+i18ycC1wJ/Bp62/WL1lN3A6loirFkXjzrF4Lqc2/FKAx2gsP2S7VOAY4BTgbcOugNJGyTdLenuF3h+gWFG1GOhuZ28bp+hjsbafhq4C3g3cISkmc7wGOCRPq/ZZHuN7TWLWTJSsBF1GTa3k9ftM2+xk3SkpCOq5dcAZwG76CXGudXT1gN31BVkRB2S22UZZM5uFbC5mts4CLjF9vcl7QS2SPoK8DvguhrjjKhDcrsg8xY7238A3nmA9Q/Rm+OIaKXkdllku7mdSY8D/wKeaGyn0+MNTNf7Pt72kZMOoguqvH6Y6fsdN2Wa3nffvG602AFIutv2mkZ3OgVKfd8lKfV33Jb3nWtjI6IIKXYRUYRJFLtNE9jnNCj1fZek1N9xK95343N2ERGTkGFsRBSh0WInaa2k+yQ9KGljk/tukqRjJd0laaekeyVdVK1fIelOSQ9U35dPOtYYXfK6HXnd2DC2Okv9fnqX5OwGdgDn2d7ZSAANkrQKWGX7HkmH07urxjnAp4EnbV9RfSiW275kgqHGiJLX7cnrJju7U4EHbT9U3fV1C7Cuwf03xvYe2/dUy/voXW+5mt773Vw9LfdJ64bkdUvyuslitxr426zHRdwnTNIb6V2S9Btgpe091aa9wMoJhRXjk7xuSV7nAEWNJC0DbgMutv3M7G3uzR/kUHi0Tlvzusli9whw7KzHfe+B1wWSFtNLiJtt316tfrSa95iZ/3hsUvHF2CSvW5LXTRa7HcBJkk6QdAjwCWBrg/tvjCTRuy3QLttXztq0ld790SD3SeuK5HVL8rrpu558FLgaWARcb/vyxnbeIEnvA34B/BF4uVr9RXrzG7cAx9G7S8bHbT85kSBjbJLX7cjrXEEREUXIAYqIKEKKXUQUIcUuIoqQYhcRRUixi4gipNhFRBFS7CKiCCl2EVGE/wLkTrqvQwRhqAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y6FekDqkwu2i",
        "outputId": "7d599bd0-d6b4-4e8e-faa3-9c34db6b90f2"
      },
      "source": [
        "# X_train = np.expand_dims(X_train, axis=2)\n",
        "# X_test = np.expand_dims(X_test, axis=2)\n",
        "\n",
        "# Normalize the Train and Test data\n",
        "X_train = X_train.astype('float32') / 255\n",
        "X_test = X_test.astype('float32') / 255\n",
        "\n",
        "# one hot encode outputs\n",
        "Y_train = np_utils.to_categorical(Y_train)\n",
        "Y_test = np_utils.to_categorical(Y_test)\n",
        "num_classes = Y_test.shape[1]\n",
        "print(X_train.shape, X_test.shape, Y_train.shape, Y_test.shape)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(60000, 1024) (20000, 1024) (60000, 10) (20000, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WrSJexeswkJS"
      },
      "source": [
        "def recall_m(y_true, y_pred):\n",
        "    K = tf.keras.backend\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "    recall = true_positives / (possible_positives + K.epsilon())\n",
        "    return recall\n",
        "\n",
        "def precision_m(y_true, y_pred):\n",
        "    K = tf.keras.backend\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\n",
        "    return precision\n",
        "\n",
        "def f1_m(y_true, y_pred):\n",
        "    K = tf.keras.backend\n",
        "    precision = precision_m(y_true, y_pred)\n",
        "    recall = recall_m(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4BCII-W0wuzn"
      },
      "source": [
        "# Build model\n",
        "def build_model():\n",
        "  model = Sequential()\n",
        "  model.add(Dense(100, input_shape=(32 * 32,), activation='relu'))\n",
        "  model.add(Dropout(0.02))\n",
        "  model.add(Dense(400, activation='relu'))\n",
        "  model.add(Dense(num_classes, activation='sigmoid'))\n",
        "  return model"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PzudS-f_wuqZ",
        "outputId": "dab387f9-6e11-4333-d05d-95774f5d4e23"
      },
      "source": [
        "model = build_model()\n",
        "# Compile\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', f1_m, precision_m, recall_m])\n",
        "\n",
        "# Fit the model\n",
        "history = model.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs=10, batch_size=200)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "300/300 [==============================] - 4s 10ms/step - loss: 1.5234 - accuracy: 0.5657 - f1_m: 0.3588 - precision_m: 0.2282 - recall_m: 0.8429 - val_loss: 0.4887 - val_accuracy: 0.8363 - val_f1_m: 0.4723 - val_precision_m: 0.3149 - val_recall_m: 0.9756\n",
            "Epoch 2/10\n",
            "300/300 [==============================] - 3s 9ms/step - loss: 0.2605 - accuracy: 0.9243 - f1_m: 0.4631 - precision_m: 0.3024 - recall_m: 0.9895 - val_loss: 0.3674 - val_accuracy: 0.8850 - val_f1_m: 0.4880 - val_precision_m: 0.3295 - val_recall_m: 0.9828\n",
            "Epoch 3/10\n",
            "300/300 [==============================] - 3s 8ms/step - loss: 0.1927 - accuracy: 0.9446 - f1_m: 0.4728 - precision_m: 0.3103 - recall_m: 0.9930 - val_loss: 0.3209 - val_accuracy: 0.9018 - val_f1_m: 0.4963 - val_precision_m: 0.3377 - val_recall_m: 0.9859\n",
            "Epoch 4/10\n",
            "300/300 [==============================] - 3s 9ms/step - loss: 0.1633 - accuracy: 0.9532 - f1_m: 0.4815 - precision_m: 0.3178 - recall_m: 0.9938 - val_loss: 0.2831 - val_accuracy: 0.9146 - val_f1_m: 0.5020 - val_precision_m: 0.3420 - val_recall_m: 0.9876\n",
            "Epoch 5/10\n",
            "300/300 [==============================] - 3s 9ms/step - loss: 0.1417 - accuracy: 0.9600 - f1_m: 0.4889 - precision_m: 0.3241 - recall_m: 0.9948 - val_loss: 0.2569 - val_accuracy: 0.9254 - val_f1_m: 0.5108 - val_precision_m: 0.3501 - val_recall_m: 0.9887\n",
            "Epoch 6/10\n",
            "300/300 [==============================] - 3s 9ms/step - loss: 0.1269 - accuracy: 0.9645 - f1_m: 0.4979 - precision_m: 0.3320 - recall_m: 0.9956 - val_loss: 0.2344 - val_accuracy: 0.9312 - val_f1_m: 0.5218 - val_precision_m: 0.3603 - val_recall_m: 0.9900\n",
            "Epoch 7/10\n",
            "300/300 [==============================] - 3s 9ms/step - loss: 0.1147 - accuracy: 0.9672 - f1_m: 0.5077 - precision_m: 0.3407 - recall_m: 0.9962 - val_loss: 0.2176 - val_accuracy: 0.9373 - val_f1_m: 0.5285 - val_precision_m: 0.3670 - val_recall_m: 0.9914\n",
            "Epoch 8/10\n",
            "300/300 [==============================] - 3s 9ms/step - loss: 0.1044 - accuracy: 0.9707 - f1_m: 0.5164 - precision_m: 0.3485 - recall_m: 0.9972 - val_loss: 0.1998 - val_accuracy: 0.9427 - val_f1_m: 0.5413 - val_precision_m: 0.3797 - val_recall_m: 0.9918\n",
            "Epoch 9/10\n",
            "300/300 [==============================] - 3s 9ms/step - loss: 0.0909 - accuracy: 0.9753 - f1_m: 0.5281 - precision_m: 0.3592 - recall_m: 0.9973 - val_loss: 0.1848 - val_accuracy: 0.9470 - val_f1_m: 0.5456 - val_precision_m: 0.3838 - val_recall_m: 0.9926\n",
            "Epoch 10/10\n",
            "300/300 [==============================] - 3s 9ms/step - loss: 0.0819 - accuracy: 0.9764 - f1_m: 0.5383 - precision_m: 0.3687 - recall_m: 0.9973 - val_loss: 0.1716 - val_accuracy: 0.9498 - val_f1_m: 0.5484 - val_precision_m: 0.3864 - val_recall_m: 0.9931\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SkiZsKATcdzV",
        "outputId": "6616bcde-46fc-4931-afe8-fddec3d3fae3"
      },
      "source": [
        "# Final evaluation of the model\n",
        "train_eval = model.evaluate(X_train, Y_train, batch_size=200)\n",
        "test_eval = model.evaluate(X_test, Y_test, batch_size=200)\n",
        "print(\"Train eval : \", train_eval)\n",
        "print(\"Test eval : \", test_eval)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "300/300 [==============================] - 1s 3ms/step - loss: 0.0734 - accuracy: 0.9796 - f1_m: 0.5369 - precision_m: 0.3673 - recall_m: 0.9979\n",
            "100/100 [==============================] - 0s 3ms/step - loss: 0.1716 - accuracy: 0.9498 - f1_m: 0.5484 - precision_m: 0.3864 - recall_m: 0.9931\n",
            "Train eval :  [0.07335953414440155, 0.9796333312988281, 0.5368852019309998, 0.36729517579078674, 0.9978994727134705]\n",
            "Test eval :  [0.17162111401557922, 0.9498000144958496, 0.5484216213226318, 0.3863980174064636, 0.993100106716156]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A7Lx92zxks53",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "outputId": "5d7da565-24c0-4b89-8a0c-63c9e4e71f4a"
      },
      "source": [
        "# Plot loss during training\n",
        "plt.subplot(211)\n",
        "plt.title('Loss')\n",
        "plt.plot(history.history['loss'], label='train')\n",
        "plt.legend()\n",
        "\n",
        "# Plot accuracy during training\n",
        "plt.subplot(212)\n",
        "plt.title('Accuracy')\n",
        "plt.plot(history.history['accuracy'], label='train')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXgc9Z3n8fe3b12+JN+XZHPZJmCCcEwM2AlZAiQBdvJMBjKQYzMQCNlkZjOz49mZTSbZmR12MrubsMMRQ0hmNgQ2A+RcciwTG8LhBJvTYAO2MbZ8gCywJdk6ulvf/aNLUkuWbdlqqaTS5/U8/XRV/aq6vyrwp6p/Xf0rc3dERCS6YmEXICIiw0tBLyIScQp6EZGIU9CLiEScgl5EJOIU9CIiEaegFxGJOAW9jGtmtsPMPhB2HSLDSUEvIhJxCnqRfswsbWbfMLM9weMbZpYO2mrM7GdmdsDM3jaz35hZLGj7czPbbWYtZvaKmV0c7l8iUpAIuwCRUegvgeXAUsCBHwN/Bfxn4EtAAzA1WHc54GZ2OvB54Dx332NmtUB8ZMsWGZjO6EWO9IfA19z9LXdvBL4KXBe0ZYGZwHx3z7r7b7wwYFQeSAOLzSzp7jvcfVso1Yv0o6AXOdIs4I2i+TeCZQBfB7YCvzKz7Wa2GsDdtwJ/DPw18JaZ3W9msxAZBRT0IkfaA8wvmp8XLMPdW9z9S+6+ALgC+A/dffHu/n13vyDY1oH/NrJliwxMQS8CSTPLdD+A+4C/MrOpZlYDfBn4HoCZfdjMTjEzAw5S6LLpMrPTzez9wZe27UAb0BXOnyPSl4JeBB6mEMzdjwywAXgBeBF4BvibYN1TgUeAVuAp4HZ3X0uhf/4WYD+wD5gG/MXI/QkiR2e68YiISLTpjF5EJOIU9CIiEaegFxGJOAW9iEjEjbohEGpqary2tjbsMkRExpSNGzfud/epA7WNuqCvra1lw4YNYZchIjKmmNkbR2tT142ISMRFKuiz+S7yXfpdgIhIscgE/c6mw6z8+7X8YtO+sEsRERlVRl0f/cmaPbmMdDLOnY9u4/J3zaAwFImIjBfZbJaGhgba29vDLmVYZTIZ5syZQzKZHPQ2kQn6eMz47EULWP3QizyxtYkLTq0JuyQRGUENDQ1UVVVRW1sb2RM9d6epqYmGhgbq6uoGvV1kum4A/u27ZzOtKs2dj+p+DyLjTXt7O9XV1ZENeQAzo7q6+oQ/tUQq6NOJOJ+5oI7Ht+7nxYaDYZcjIiMsyiHf7WT+xkgFPcDH3zOPqkxCZ/UiIoHIBX1VJsl1y+fz8Ka9vL7/UNjliMg4ceDAAW6//fYT3u7yyy/nwIEDw1BRr8gFPcCnV9SRjMdY89j2sEsRkXHiaEGfy+WOud3DDz/MpEmThqssIKJBP7Uqze+fO4cHNzbwVnO0L7USkdFh9erVbNu2jaVLl3Leeedx4YUXcsUVV7B48WIArrrqKs4991yWLFnCmjVrerarra1l//797Nixg0WLFnH99dezZMkSLrnkEtra2kpSW2Qur+zvhosWcN/vdnLPEztYfdkZYZcjIiPoqz99iZf3NJf0NRfPmsBXPrLkqO233HILmzZt4rnnnmPdunV86EMfYtOmTT2XQd5zzz1MmTKFtrY2zjvvPD760Y9SXV3d5zVee+017rvvPu666y4+9rGP8eCDD3LttdcOufZIntEDzK+u4PJ3zeTe9W/Q3J4NuxwRGWeWLVvW51r3W2+9lbPPPpvly5eza9cuXnvttSO2qaurY+nSpQCce+657NixoyS1RPaMHuDGlQv52Qt7uXf9Tm5atTDsckRkhBzrzHukVFRU9EyvW7eORx55hKeeeory8nJWrVo14LXw6XS6Zzoej5es6yayZ/QAZ86eyIWn1vDtx1+nPZsPuxwRibCqqipaWloGbDt48CCTJ0+mvLycLVu2sH79+hGtLdJBD3DTyoXsb+3goWd2h12KiERYdXU1K1as4Mwzz+TP/uzP+rRdeuml5HI5Fi1axOrVq1m+fPmI1mbuxx/W18wuBb4JxIG73f2Wfu3/E3hfMFsOTHP3SUFbHngxaNvp7lcc673q6+u9lDcecXeuuu0JDrZl+dcvrSIei/4v50TGo82bN7No0aKwyxgRA/2tZrbR3esHWv+4Z/RmFgduAy4DFgPXmNni4nXc/U/cfam7LwX+F/BQUXNbd9vxQn44mBk3rlzIjqbDGsJYRMalwXTdLAO2uvt2d+8E7geuPMb61wD3laK4UrlkyQwW1FRwx6NbGcwnGBGRKBlM0M8GdhXNNwTLjmBm84E64NdFizNmtsHM1pvZVUfZ7oZgnQ2NjY2DLH3w4jHjhosWsGl3M09sbSr564vI6DAeTuRO5m8s9ZexVwMPuHvxJS7zg36jjwPfMLMjrnN09zXuXu/u9VOnDngT8yHrHsL4jke3Dsvri0i4MpkMTU1NkQ777vHoM5nMCW03mOvodwNzi+bnBMsGcjVwc7/CdgfP281sHXAOMOJDS3YPYfx3P9/CCw0HOGvO8I4tISIja86cOTQ0NDAcvQKjSfcdpk7EYIL+aeBUM6ujEPBXUzg778PMzgAmA08VLZsMHHb3DjOrAVYAf39CFZbQx98zj39cu5U7H93G7X94blhliMgwSCaTJ3TXpfHkuF037p4DPg/8EtgM/MDdXzKzr5lZ8VU0VwP3e9/PTYuADWb2PLAWuMXdXy5d+Semewjjn2/apyGMRWTcGNR19COp1NfR99fY0sGK//ZrPvru2fzd7501bO8jIjKShnQdfdT0DmG8W0MYi8i4MO6CHgpDGOe6uvj2E6+HXYqIyLAbl0HfPYTx99fv1BDGIhJ54zLooTCEcUtHju+tfyPsUkREhtW4DfruIYzveXyHhjAWkUgbt0EPcNOqwhDGDz7TEHYpIiLDZlwH/fkLqjl7zkTWPLadfNfousxURKRUxnXQmxk3rVrIG02H+fmmvWGXIyIyLMZ10AP8m8XBEMbrtkV6MCQRGb/GfdDHY8ZnVy7gpT3NPL51f9jliIiU3LgPeoCrzpnN9Alp7lg34oNqiogMOwU9vUMYP7mtied3HQi7HBGRklLQB65ZNo+qTII7H9VZvYhEi4I+UJVJ8onz5/OLl/axvbE17HJEREpGQV/kU++tIxmPseax7WGXIiJSMgr6IlOr0nysfg4PPbObNzWEsYhEhIK+nxsuXEiuq4t7HtcQxiISDQr6fuZVl/Ohs2Zx7293crBNQxiLyNg3qKA3s0vN7BUz22pmqwdo/5SZNZrZc8Hjj4raPmlmrwWPT5ay+OHy2YsW0KohjEUkIo4b9GYWB24DLgMWA9eY2eIBVv0/7r40eNwdbDsF+ArwHmAZ8BUzm1yy6ofJmbMnctFpU/nOExrCWETGvsGc0S8Dtrr7dnfvBO4Hrhzk638Q+H/u/ra7vwP8P+DSkyt1ZN24cgH7Wzt4YKOGMBaRsW0wQT8b2FU03xAs6++jZvaCmT1gZnNPcNtR5/wF1Zw9dxJrHttOLt8VdjkiIietVF/G/hSodfezKJy1/9OJbGxmN5jZBjPb0NjYWKKShsbMuGnlAna+fZifb9oXdjkiIidtMEG/G5hbND8nWNbD3ZvcvSOYvRs4d7DbBtuvcfd6d6+fOnXqYGsfdpcsnsGCqRXc+aiGMBaRsWswQf80cKqZ1ZlZCrga+EnxCmY2s2j2CmBzMP1L4BIzmxx8CXtJsGxMiMWMz15UGML4N69pCGMRGZuOG/TungM+TyGgNwM/cPeXzOxrZnZFsNoXzOwlM3se+ALwqWDbt4H/QuFg8TTwtWDZmNE9hLEGOxORscpGW5dEfX29b9iwIewy+ljz2Db+68Nb+PHNKzh77qSwyxEROYKZbXT3+oHa9MvYQbhm2TwmaAhjERmjFPSDUJVJcl0whPE2DWEsImOMgn6QPvXeOlLxGHdpCGMRGWMU9IM0tSrN72sIYxEZgxT0J0BDGIvIWKSgPwEawlhExiIF/Qm6caWGMBaRsUVBf4KWzOoewvh1DWEsImOCgv4k3LRyIftbOzWEsYiMCQr6k7B8wRQNYSwiY4aC/iQUhjBeqCGMRWRMUNCfpEsWT2fB1AruWKchjEVkdFPQn6RYzLjxooW8vFdDGIvI6KagH4Irz5nF9Alp7linwc5EZPRS0A9BOhHnjy5YwFPbm3hu14GwyxERGZCCfoiueU8whLHO6kVklFLQD1FlOsEnzq/lly9rCGMRGZ0U9CXwqRW1pOIx1jyqIYxFZPRR0JdATWWaj9XP5aFnG9h3UEMYi8joMqigN7NLzewVM9tqZqsHaP8PZvaymb1gZv9qZvOL2vJm9lzw+Ekpix9Nrr9wAfku554nNISxiIwuxw16M4sDtwGXAYuBa8xscb/VngXq3f0s4AHg74va2tx9afC4okR1jzrzqsv58FmzuHf9Gxw8rCGMRWT0GMwZ/TJgq7tvd/dO4H7gyuIV3H2tux8OZtcDc0pb5tjw2ZULONSZ53u/1RDGIjJ6DCboZwO7iuYbgmVH8xng50XzGTPbYGbrzeyqgTYwsxuCdTY0NjYOoqTRacmsiawMhjBe+8pbvNncruERRCR0iVK+mJldC9QDK4sWz3f33Wa2APi1mb3o7n0uOnf3NcAagPr6+jGdjF+4+FQ+ftd6Pv2dpwGYUpFi0cwqzpgxgUUzJ7BoZhWnTKsknYiHXKmIjBeDCfrdwNyi+TnBsj7M7APAXwIr3b2je7m77w6et5vZOuAcILK/Ljp3/mR+95cfYPPeZrbsbWbz3hY272vme+vfoCNXGNI4ETNOmVbJGTOqgvAvPKZWpUOuXkSiaDBB/zRwqpnVUQj4q4GPF69gZucA3wIudfe3ipZPBg67e4eZ1QAr6PtFbSRNLEuyfEE1yxdU9yzL5bvY0XSYzXubex6/ff1tfvTcnp51aipTRcFf+BSwcGolqYSughWRk3fcoHf3nJl9HvglEAfucfeXzOxrwAZ3/wnwdaAS+BczA9gZXGGzCPiWmXVR+D7gFnd/eZj+llEtEY9xyrRKTplWyUfOntWz/J1DnWzeVzjz37K3mc37mvnukzvoDM7+k3HjlGlVLJpZxaKi7p/qSp39i8jg2Gj7srC+vt43bNgQdhmhyuW72L7/UHDm39LzCeCtlp4eMaZVpVk0cwJnzKxicfApYEFNBYm4zv5FxiMz2+ju9QO1lfTLWCmNRDzGadOrOG16FVcu7V3e1NrBln2F4H95bzNb9rbw5Lb9ZPOFg3UqEePUaZU93T+nTa9kalWamso0U8pTxGIW0l8kImFS0I8h1ZVpVpySZsUpNT3LsvkutjW2Bl/+tvDy3mbWvdJ4xI3L4zFjSkWKmso0NZUpplamqakqTBeWBY+qFNUVaeI6KIhEhoJ+jEvGY5wxYwJnzJhQuJ4p0NjSwfbGVva3drK/taPn0djSQWNrJ9sbD7G/taPnSqBiMaPooFB0MKjqOz+tKs2UipS6i0RGOQV9RE2tSh/3ck13p7Uj13swaOmgsee59wDxxs5D7G/ppC2bP+I1zGByeWrATwY1lWmmVhYOBlWZBJWZBBMySdKJGMGX9iIyAhT045iZUZVJUpVJUldTcdz1D3XkaGwp+nTQ2sn+ovn9rZ0833CAxpYODnceeVDolowblekEVZlk8JwI6kj0PCrTfef7r1uZTqh7SWSQFPQyaBXpBBXpBLWDOCgc7syxv6WTxtYO3jnUSWtHjpaOHC3tWVrac7S29063dOTYfaCN1o5gvj1Hvuv4V4NVpOKF0C86GFSlj36wqEwnqUjHqQz+jop0gopUXF1PEnkKehkW5akE86oTzKsuP+Ft3Z32bBct7Vma23OFg0TRAaK5PRssKyzvnj7YlmX3O4cL63Xkjvmpolg6EesT/pXpeO90aoBlPev2HjS6n8uTcV3dJKOOgl5GHTOjLBWnLBVn2oSTf51cvqvogFAI/0Md/Z/zHOrsXda9vKm1k51vHw6WFdYZ7E9OKlK9B4WKdJyKVOKITxFlqTiZZJzyVJyyZGG+LBmnPJWgLBUL2hJ92pJx03cbclIU9BJZiXiMSeUpJpWnhvxaXV1OWzbf5wDRc3DoLD545PscMLoPFHsPtnOoszB/uDNPWzY/6ANHt3jMKE/GyaR6DxADHywK65T1aes+aMQoSyaCA02MTCI46CTipJMxfVEeUQp6kUGIxaznjHxaCV7P3enIddEWhP7hzjztwXNbNh8sz9HW2cXhzhzt2QHWK1r3YFu2z2u1ZfM9w2icCLNCV1Ym2Xsg6Z7PJPsuzyRjpBO908XLC9v1X97blknEyaRipOI6sIwEBb1ICMysJ/wmD9N75INPIW3FB4VsnsOdOdo687Rnu2jP5mnP9U53ZPO0Bweg9mC6PZsP2rp4+1Bnoa3PNl105k/8oAK9B5Z0onBASSeLpruXJ2PHWOd47THSRQel4mXpRIxEbHx0hynoRSIqHitcxlqZHv5/5vkupyMI/7bgwFB4dAUHj6DtiINEno5cV/AoHDR6pnNddGS7eOdQZ+86/dbvHv7jZMUM0ok4qUSs8Ij3PicT1jufiJOKW29bz/Ki7YJlxW3p7vl+belEv3WDddLBJ55SU9CLyJDFY0Z5KkEJvg45Ifkup7PfgaH7gNOz7CgHkN72wgGkM99FZ86D58JBpDPXRWeui4NtWbK5rqCti2zw3JnroiOYLoWz507ixzevKMlrFVPQi8iYFY/1XqEVJncnFxx0ug8CHcGBofig0Nlvum+bU10xPEdKBb2IyBCZGcm4kRylP74bnVWJiEjJKOhFRCJu1N1hyswagTeG8BI1wP4SlTPWaV/0pf3Rl/ZHryjsi/nuPnWghlEX9ENlZhuOdjut8Ub7oi/tj760P3pFfV+o60ZEJOIU9CIiERfFoF8TdgGjiPZFX9offWl/9Ir0vohcH72IiPQVxTN6GcfMbJ2ZvWNmx75hrsg4oqCXyDCzWuBCwIErRvB99QtzGdUiE/RmdqmZvWJmW81sddj1hMnM5prZWjN72cxeMrMvhl3TCPkEsB74LvDJ7oXB/njIzLJm1mlm/1jUdr2ZbTazlmB/vTtY7mZ2StF63zWzvwmmV5lZg5n9uZntA75jZpPN7Gdm1hh8oviZmc0p2n6KmX3HzPYE7T8Klm8ys48UrZc0s/1mds5w7SQzm2RmD5jZluBvP3+43mssMLM/Cf6dbDKz+8wsE3ZNpRaJoDezOHAbcBmwGLjGzBaHW1WocsCX3H0xsBy4eZzsj08A9waPD5rZ9OD/jZ8B1cBDwCPA/QBm9vvAXwfbTaDwKaBpkO81A5gCzAduoPBv6TvB/DygDfjHovX/N1AOLAGmAf8zWP7PwLVF610O7HX3ZwdZx8n4JvALdz8DOBvYPIzvNaqZ2WzgC0C9u58JxIGrw62q9CIR9MAyYKu7b3f3Tgr/kK8MuabQuPted38mmG6h8A95drhVDS8zu4BCyP7A3TcC24CPU/h/Yw6QB74FdLn748FmfwT8vbs/7QVb3X2wv8ruAr7i7h3u3ubuTe7+oLsfDvb53wIrg9pmUjgJudHd33H3rLs/GrzO94DLzaz77rjXUTgoDAszmwhcBHwbwN073f3AcL3fGJEAyoIuuHJgT8j1lFxUgn42sKtovoGIB9tgBf3W5wC/DbeSYfdJ4Ffu3v0z9u8Hy+YCBvwZhXAuNpfCAeFkNLp7e/eMmZWb2bfM7A0zawYeAyYFnyjmAm+7+zv9X8Td9wBPAB81s0kUDgj3nmRNg1EHNFLobnrWzO42s4phfL9Rzd13A/8A7AT2Agfd/VfhVlV6UQl6GYCZVQIPAn/s7s1h1zNczKwM+Biw0sz2Bf3mf0KhW2IBkAGeH2DTXcDCo7zsYQpnd91m9Gvvf13yl4DTgfe4+wQKZ81QOMjsAqYEQT6Qf6LQffP7wFNB+AyXBPBu4A53Pwc4BIzb77TMbDKFT/91wCygwsyuPfZWY09Ugn43hbOmbnOCZeOWmSUphPy97v5Q2PUMs6sodM0sBpYGj0XAb4DfoxBuTcD/Ad5vZr8Mtrsb+FMzO9cKTjGz+UHbc8DHzSxuZpcSdMMcQxWFfvkDZjYF+Ep3g7vvBX4O3B58aZs0s4uKtv0RhfD9IoU+++HUADS4e/cnvAeC9x6vPgC87u6N7p6l8D3Oe0OuqeSiEvRPA6eaWZ2ZpSh8mfKTkGsKjRXudvxtYLO7/4+w6xkBnwS+4+473X1f94PCl6HzgFOAtRTO7BPAKwDu/i8U+tK/D7RQCNwpwWt+EfgIcAD4w6DtWL4BlFEYAXE98It+7dcBWWAL8Bbwx90N7t5G4aBcRyFohk2wX3aZ2enBoouBl4fzPUe5ncDyoOvNKOyPyH05HZlfxprZ5RT+scWBe9z9b0MuKTTBF5O/AV6kt1/6P7n7w+FVFT4zWwX8qbt/OOxa+jOzLwOnufuwdxuY2VIKn2ZSwHbg0wN9fzBemNlXgT+gcLXas8AfuXtHuFWVVmSCXmSsCrp6ngWuc/fHwq5HoicqXTciY5KZXU/hy9qfK+RluOiMXkQk4nRGLyIScaNuMKaamhqvra0NuwwRkTFl48aN+492z9hRF/S1tbVs2LAh7DJERMYUMzvq8B3quhERibhRd0YvIhIF7k4272TzXWTzXXTmuwrzucJ8R/DcvU5nvovKdILzaqcc/8VPkIJeRCLB3enMd9Ge7aIjmy885wrP7bk87dk8HT3TRW3ZPJ1B6PY8FwV0YXm/+aLA7sx3kc0duU02f+JXNC6dO4kf3byi5PtGQS8iw8Ld6cgVgrQtm6ets/Dcns1zuLN3vqNf6Lbn+gZyezbf8zodA4V3rnedoVwtnorHSMaNZCJGMh7rnY8H84nCslQiRkV6gHUSA2/TZz4eI5noO58K3i8ZN6oyydL9ByiioBcZh7q6CiHc1hPCOdo6i+fzPQF9uHs6COa2bJ72zt627vBuG2C66ySCNx4zMokYmWScTDJOOhEjnYyTScbIJOJMrkiRSRTm08FzJhknHaybSfZrSwSv0zMdO+J1U/EYhaFuoklBLxKC7rPdwqP7TDWYznX1nLn2rJPNl3T9zlz/ofmPL2ZQnkqQScYpS8UoS8YLj1Sc6ooUZZMLgdq9vDwVJ5OK91mv+Lk7yMtSRQGdiJGI6xqRUlPQi5yg9myelvYcze1ZWtpztLRnaW4rPBcvby5a3tze294W9AkPVSoRK5yVJrrPToumEzEmlCV7ptOJeNBemE4lYpQXhfCxArnQHv2z3ihT0Mu4kst30dqRo6U9x8G2oqAuCuLmtuyRQV70fLyQjhlUphNMKEtSlUlSlUkwe1IZEzJVVGUSlKUSAwZzd9fDQMHcf/1UPEYsptCVwVHQy5iQzXdxKAjo1o7g0Z6jJXhu7cj2m8/1BHrPuu1ZDnXmj/teZck4E8oSVGWSTMgkmFSeYu6U8iC4E0wIlldlkkXrFdqqMgkqUgmFsIwqCnoZVrl8Fwfasj3h2xvUAwTzUYM6S3v2+F0dZlCZSlCZSVCZLjxXZRLMmpShItU3mIsDuzvAu5cn1UcsEaOgl5PWns2z72A7ew+282Zz8XMb+5o72HewjcaWjuNeeRGPGVXd4ZwuhHNNZYramoqe+e62ykyCqnRvmBfaklRmEpQn4zqTFhmAgl6O4O40t+XY29zGvoPthUfzkc8HDmeP2LYqnWDGxAwzJmY4bdpUZkzMUFOZ7g3rTIKqIJi7gzqd0Jd8IsNJQT/O5LucptYO9haFdvGZ+JvNHew92DZgV0lNZZoZE9PMmVxOfe1kZk4sY/qEDDMnZpg+oRDulWn9LyUy2uhfZQTtb+3gxd0Hee3Nlr7dKgfbebOlg3y/vpRk3JhWVQjsxbMmcPEZ03rOymcEAT6tKkMqob5rkbFIQT/GdYf6poaDvLi78Nh7sL2nvSIV7wnt8xfWMGNimhkTy5hRdCZeXZFS37ZIhCnox5CmINRfDEJ90+6D7CkK9bqaCuprp3DW7ImcOXsii2dOYGL58IydISJjh4J+lOoO9U3BWfqLDUeG+rm1U/jU7Am8a/YklsyewIRhGhBJRMY2Bf0o8Pahzp5Qf6HhAJt2N7P7QFtPe211eU+onxmcrSvURWSwFPQjrDjUu7tg+of6OfMm8Ynz5/OuORNZMmsiE8sU6iJy8hT0w+idINRfPEqozy8O9dkTWTJboS4ipaegL7EDhzv56k9f5ukdb9PwTt9QXzpvEtedP5+zFOoiMoIU9CV2x7pt/Pi53Vx25kyuXV44Uz9z1kRd/SIioVHQl9DBw1m+t/4NPnTWLP7XNeeEXY6ICAD6qWMJ/dNTOzjUmedzqxaGXYqISA8FfYkc7szxnSde5/1nTGPRzAlhlyMi0kNBXyL3/W4X7xzOcvP7dDYvIqOLgr4EOnNd3PXYdpbVTeHc+VPCLkdEpA8FfQn88NkG9jW3c/P7Tgm7FBGRIyjohyjf5dz56HbOnD2Bi06tCbscEZEjKOiH6Oeb9vL6/kN8btUpukuSiIxKQwp6M7vUzF4xs61mtnqA9vlm9q9m9oKZrTOzOUN5v9HG3blt7TYWTK3gg0tmhF2OiMiATjrozSwO3AZcBiwGrjGzxf1W+wfgn939LOBrwN+d7PuNRutebWTz3mZuXLmQuG7cISKj1FDO6JcBW919u7t3AvcDV/ZbZzHw62B67QDtY9rta7cya2KGq5bODrsUEZGjGkrQzwZ2Fc03BMuKPQ/8XjD9b4EqM6sewnuOGr97/W2e3vEO11+0QPdSFZFRbbgT6k+BlWb2LLAS2A3k+69kZjeY2QYz29DY2DjMJZXG7eu2MqUixdXnzQu7FBGRYxpK0O8G5hbNzwmW9XD3Pe7+e+5+DvCXwbID/V/I3de4e72710+dOnUIJY2MTbsPsu6VRj5zQR1lqXjY5YiIHNNQgv5p4FQzqzOzFHA18JPiFcysxsy63+MvgHuG8H6jxh2PbqMyneDa5fPDLkVE5LhOOujdPQd8HvglsBn4gbu/ZGZfM7MrgtVWAa+Y2avAdOBvh1hv6LY3tvLwi3u57vz5unGIiIwJQxqP3t0fBh7ut+zLRdMPAA8M5T1Gm0JnZk8AAAq/SURBVG89up1UPMa/W1EXdikiIoOiy0VOwJ4DbTz0bAN/cN5cplalwy5HRGRQFPQn4K7fbMcdbrhoQdiliIgMmoJ+kJpaO7j/d7u4Yuks5kwuD7scEZFBU9AP0nef3EF7TrcJFJGxR0E/CC3tWf7pyR1csng6p0yrCrscEZEToqAfhHt/u5Pm9hyfW6Ubi4jI2KOgP472bJ67f/M6F55aw9lzJ4VdjojICVPQH8e/bGxgf2sHN6lvXkTGKAX9MeTyXXzr0W2cM28S5y+IxKCbIjIOKeiP4acv7KHhnTbdJlBExjQF/VF0dTm3r93G6dOruPiMaWGXIyJy0hT0R/HI5jd57a1Wblq1kJhuEygiY5iCfgDuzm3rtjF3ShkfPmtm2OWIiAyJgn4AT25r4vldB7hx5UISce0iERnblGIDuH3dVqZWpfnou+eEXYqIyJAp6Pt5btcBntjaxPUX1pFJ6jaBIjL2Kej7uX3tViaWJfn4e3SbQBGJBgV9kVffbOFXL7/JJ99bS2V6SDffEhEZNRT0Re5ct43yVJxPv7c27FJEREpGQR/Y9fZhfvz8Hq5ZNo/JFamwyxERKRkFfeBbj20jZnD9hbpNoIhEi4IeeKulnR9saOCj757DjImZsMsRESkpBT3w7cdfJ5fv4rMrNRSxiETPuA/6g4ez3Lt+J5e/ayZ1NRVhlyMiUnLjPuj/+akdtHboNoEiEl3jOugPd+a454nXef8Z01g8a0LY5YiIDItxHfT3/24X7xzO8jndJlBEImzcBn1nrou7frOdZXVTqK+dEnY5IiLDZtwG/Y+e3c3eg+06mxeRyBuXQZ/vcu54dBtLZk1g5WlTwy5HRGRYjcug/8Wmfby+/xA3v083/RaR6Bt3Qe/u3LZ2KwtqKvjgkhlhlyMiMuyGFPRmdqmZvWJmW81s9QDt88xsrZk9a2YvmNnlQ3m/Ulj3aiMv723mxlULieum3yIyDpx00JtZHLgNuAxYDFxjZov7rfZXwA/c/RzgauD2k32/Urlj7TZmTsxw1dLZYZciIjIihnJGvwzY6u7b3b0TuB+4st86DnT/EmkisGcI7zdkT+94m9/teJsbLlpAKjHueq1EZJwaym2UZgO7iuYbgPf0W+evgV+Z2b8HKoAPDPRCZnYDcAPAvHnzhlDSsd2+ditTKlJcfd7wvYeIhCObzdLQ0EB7e3vYpQyrTCbDnDlzSCaTg95muO+Xdw3wXXf/72Z2PvC/zexMd+8qXsnd1wBrAOrr6304Cnlpz0HWvtLIn15yGmUp3fRbJGoaGhqoqqqitrY2slfTuTtNTU00NDRQV1c36O2G0n+xG5hbND8nWFbsM8APggKfAjJAzRDe86TdsW4blekE151fG8bbi8gwa29vp7q6OrIhD2BmVFdXn/CnlqEE/dPAqWZWZ2YpCl+2/qTfOjuBi4MCF1EI+sYhvOdJ2d7Yyv99cS/XLp/PxLLBf9wRkbElyiHf7WT+xpMOenfPAZ8HfglspnB1zUtm9jUzuyJY7UvA9Wb2PHAf8Cl3H5aumWP51qPbScVjfOaCwX/UERGJiiFdeuLuD7v7ae6+0N3/Nlj2ZXf/STD9sruvcPez3X2pu/+qFEWfiL0H23jo2QY+Vj+XqVXpkX57ERknDhw4wO23n/gV5JdffjkHDhwYhop6Rf4aw7see50uhxsu0k2/RWT4HC3oc7ncMbd7+OGHmTRp0nCVBQz/VTehevtQJ/f9bidXLp3F3CnlYZcjIiPkqz99iZf3NJf0NRfPmsBXPrLkqO2rV69m27ZtLF26lGQySSaTYfLkyWzZsoVXX32Vq666il27dtHe3s4Xv/hFbrjhBgBqa2vZsGEDra2tXHbZZVxwwQU8+eSTzJ49mx//+MeUlZUNufZIn9F/94nXacvmuUk3/RaRYXbLLbewcOFCnnvuOb7+9a/zzDPP8M1vfpNXX30VgHvuuYeNGzeyYcMGbr31Vpqamo54jddee42bb76Zl156iUmTJvHggw+WpLbIntG3tGf57pM7+OCS6Zw6vSrsckRkBB3rzHukLFu2rM+17rfeeis//OEPAdi1axevvfYa1dXVfbapq6tj6dKlAJx77rns2LGjJLVENui//9udNLfrpt8iEo6Kioqe6XXr1vHII4/w1FNPUV5ezqpVqwa8Fj6d7r1gJB6P09bWVpJaItl1057Nc/fjr3PBKTWcPXd4v+QQEQGoqqqipaVlwLaDBw8yefJkysvL2bJlC+vXrx/R2iJ5Rv/AxgYaWzr45h8sDbsUERknqqurWbFiBWeeeSZlZWVMnz69p+3SSy/lzjvvZNGiRZx++uksX758RGuzEH6/dEz19fW+YcOGk94+l+/iff99HdUVaX74ufeOi1/KiQhs3ryZRYsWhV3GiBjobzWzje5eP9D6keu6+dkLe9n1dptuEygiEohU0Hd1Obev28pp0yu5+IxpYZcjIjIqRCroH9n8Jq++2crnVp1CTLcJFBl3RltX9HA4mb8xMkHv7ty2bhtzp5Tx4bNmhl2OiIywTCZDU1NTpMO+ezz6TCZzQttF5qqbHU2H2by3mS9/eDGJeGSOXyIySHPmzKGhoYHGxhEfCX1Edd9h6kREJujraip4/M/fx4SMxpsXGY+SyeQJ3XVpPIlM0ANMqzqxjzMiIuOB+jhERCJOQS8iEnGj7pexZtYIvDGEl6gB9peonLFO+6Iv7Y++tD96RWFfzHf3qQM1jLqgHyoz23C0nwGPN9oXfWl/9KX90Svq+0JdNyIiEaegFxGJuCgG/ZqwCxhFtC/60v7oS/ujV6T3ReT66EVEpK8ontGLiEgRBb2ISMRFJujN7FIze8XMtprZ6rDrCZOZzTWztWb2spm9ZGZfDLumsJlZ3MyeNbOfhV1L2Mxskpk9YGZbzGyzmZ0fdk1hMrM/Cf6dbDKz+8wscmOpRCLozSwO3AZcBiwGrjGzxeFWFaoc8CV3XwwsB24e5/sD4IvA5rCLGCW+CfzC3c8AzmYc7xczmw18Aah39zOBOHB1uFWVXiSCHlgGbHX37e7eCdwPXBlyTaFx973u/kww3ULhH/LscKsKj5nNAT4E3B12LWEzs4nARcC3Ady9090PhFtV6BJAmZklgHJgT8j1lFxUgn42sKtovoFxHGzFzKwWOAf4bbiVhOobwH8EusIuZBSoAxqB7wRdWXebWUXYRYXF3XcD/wDsBPYCB939V+FWVXpRCXoZgJlVAg8Cf+zuzWHXEwYz+zDwlrtvDLuWUSIBvBu4w93PAQ4B4/Y7LTObTOHTfx0wC6gws2vDrar0ohL0u4G5RfNzgmXjlpklKYT8ve7+UNj1hGgFcIWZ7aDQpfd+M/teuCWFqgFocPfuT3gPUAj+8eoDwOvu3ujuWeAh4L0h11RyUQn6p4FTzazOzFIUvkz5Scg1hcbMjEIf7GZ3/x9h1xMmd/8Ld5/j7rUU/r/4tbtH7oxtsNx9H7DLzE4PFl0MvBxiSWHbCSw3s/Lg383FRPDL6UjcYcrdc2b2eeCXFL41v8fdXwq5rDCtAK4DXjSz54Jl/8ndHw6xJhk9/j1wb3BStB34dMj1hMbdf2tmDwDPULha7VkiOByChkAQEYm4qHTdiIjIUSjoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4hT0IiIR9/8B2CcwNJx9PwIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y3tvPPlzxSVD",
        "outputId": "d459bd78-dbe8-4802-b8d8-f63b1f60908e"
      },
      "source": [
        "Y_pred = model.predict(X_test, batch_size=200, verbose=1)\n",
        "print(Y_pred)\n",
        "print(Y_test)\n",
        "Y_pred_bool = np.argmax(Y_pred, axis=1)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100/100 [==============================] - 0s 3ms/step\n",
            "[[9.9962062e-01 5.4249465e-03 7.3274970e-04 ... 3.7899762e-01\n",
            "  4.6551609e-01 2.1617115e-03]\n",
            " [9.9962151e-01 1.1842400e-02 3.5682321e-04 ... 5.2675587e-01\n",
            "  8.0568212e-01 2.5007129e-04]\n",
            " [9.3925786e-01 6.4306855e-01 1.5020370e-04 ... 4.4699740e-01\n",
            "  2.8238881e-01 3.2887757e-03]\n",
            " ...\n",
            " [2.4017751e-02 9.6386975e-01 9.3751937e-02 ... 5.2279234e-04\n",
            "  6.6311359e-03 5.9519613e-01]\n",
            " [1.9000769e-03 1.2021959e-03 1.5613943e-02 ... 5.6133706e-07\n",
            "  7.3301136e-01 9.9985135e-01]\n",
            " [2.3862720e-04 5.9611357e-06 1.7382997e-01 ... 9.5077288e-09\n",
            "  3.8661614e-01 9.9999583e-01]]\n",
            "[[1. 0. 0. ... 0. 0. 0.]\n",
            " [1. 0. 0. ... 0. 0. 0.]\n",
            " [1. 0. 0. ... 0. 0. 0.]\n",
            " ...\n",
            " [0. 0. 0. ... 0. 0. 1.]\n",
            " [0. 0. 0. ... 0. 0. 1.]\n",
            " [0. 0. 0. ... 0. 0. 1.]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f5Ch-xrcskOz",
        "outputId": "c1d1fd52-b1b5-47d1-df73-1b8b66913aa3"
      },
      "source": [
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "\n",
        "y_pred = np.argmax(Y_pred, axis=1)\n",
        "y_true = np.argmax(Y_test, axis=1)\n",
        "\n",
        "print('Confusion Matrix')\n",
        "print(confusion_matrix(y_true, y_pred))\n",
        "print('Classification Report')\n",
        "target_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n",
        "print(classification_report(y_true, y_pred, target_names=target_names))"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Confusion Matrix\n",
            "[[1973    0    0    0    5   17    2    2    1    0]\n",
            " [   8 1961   10    0    2    0   11    4    1    3]\n",
            " [   3   29 1826   69   30    0   27    1    1   14]\n",
            " [   5    1   76 1833   72    1    8    2    0    2]\n",
            " [   7    5   15   83 1841   12   13    2    4   18]\n",
            " [   4    7    0    1   15 1944    5    6   17    1]\n",
            " [   3   14   13    6   17    3 1899    4    5   36]\n",
            " [  15    7   21    5    7   14   17 1910    4    0]\n",
            " [   5    3    2    3   14    7   13    0 1930   23]\n",
            " [   2   32    5    3   12    2   41    0   24 1879]]\n",
            "Classification Report\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.97      0.99      0.98      2000\n",
            "           1       0.95      0.98      0.97      2000\n",
            "           2       0.93      0.91      0.92      2000\n",
            "           3       0.92      0.92      0.92      2000\n",
            "           4       0.91      0.92      0.92      2000\n",
            "           5       0.97      0.97      0.97      2000\n",
            "           6       0.93      0.95      0.94      2000\n",
            "           7       0.99      0.95      0.97      2000\n",
            "           8       0.97      0.96      0.97      2000\n",
            "           9       0.95      0.94      0.95      2000\n",
            "\n",
            "    accuracy                           0.95     20000\n",
            "   macro avg       0.95      0.95      0.95     20000\n",
            "weighted avg       0.95      0.95      0.95     20000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}