keypoint_classification_EN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "igMyGnjE9hEp"
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "RANDOM_SEED = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "t2HDvhIu9hEr"
   },
   "source": [
    "# Specify each path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "9NvZP2Zn9hEy"
   },
   "outputs": [],
   "source": [
    "dataset = 'model/keypoint_classifier/keypoint.csv'\n",
    "model_save_path = 'model/keypoint_classifier/keypoint_classifier.hdf5'\n",
    "tflite_save_path = 'model/keypoint_classifier/keypoint_classifier.tflite'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "s5oMH7x19hEz"
   },
   "source": [
    "# Set number of classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "du4kodXL9hEz"
   },
   "outputs": [],
   "source": [
    "NUM_CLASSES = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XjnL0uso9hEz"
   },
   "source": [
    "# Dataset reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "QT5ZqtEz9hE0"
   },
   "outputs": [],
   "source": [
    "X_dataset = np.loadtxt(dataset, delimiter=',', dtype='float32', usecols=list(range(1, (21 * 2) + 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "QmoKFsp49hE0"
   },
   "outputs": [],
   "source": [
    "y_dataset = np.loadtxt(dataset, delimiter=',', dtype='int32', usecols=(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "xQU7JTZ_9hE0"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_dataset, y_dataset, train_size=0.75, random_state=RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mxK_lETT9hE0"
   },
   "source": [
    "# Model building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "vHBmUf1t9hE1"
   },
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Input((21 * 2, )),\n",
    "    tf.keras.layers.Dropout(0.2),\n",
    "    tf.keras.layers.Dense(20, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.4),\n",
    "    tf.keras.layers.Dense(10, activation='relu'),\n",
    "    tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ypqky9tc9hE1",
    "outputId": "5db082bb-30e3-4110-bf63-a1ee777ecd46"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dropout (Dropout)           (None, 42)                0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 20)                860       \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 20)                0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 10)                210       \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 5)                 55        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,125\n",
      "Trainable params: 1,125\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()  # tf.keras.utils.plot_model(model, show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "MbMjOflQ9hE1"
   },
   "outputs": [],
   "source": [
    "# Model checkpoint callback\n",
    "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "    model_save_path, verbose=1, save_weights_only=False)\n",
    "# Callback for early stopping\n",
    "es_callback = tf.keras.callbacks.EarlyStopping(patience=20, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "c3Dac0M_9hE2"
   },
   "outputs": [],
   "source": [
    "# Model compilation\n",
    "model.compile(\n",
    "    optimizer='adam',\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    metrics=['accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7XI0j1Iu9hE2"
   },
   "source": [
    "# Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WirBl-JE9hE3",
    "outputId": "71b30ca2-8294-4d9d-8aa2-800d90d399de",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "Graph execution error:\n\nDetected at node 'sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits' defined at (most recent call last):\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\runpy.py\", line 194, in _run_module_as_main\n      return _run_code(code, main_globals, None,\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\runpy.py\", line 87, in _run_code\n      exec(code, run_globals)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\traitlets\\config\\application.py\", line 846, in launch_instance\n      app.start()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 712, in start\n      self.io_loop.start()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 199, in start\n      self.asyncio_loop.run_forever()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\asyncio\\base_events.py\", line 570, in run_forever\n      self._run_once()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\asyncio\\base_events.py\", line 1859, in _run_once\n      handle._run()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\asyncio\\events.py\", line 81, in _run\n      self._context.run(self._callback, *self._args)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 504, in dispatch_queue\n      await self.process_one()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 493, in process_one\n      await dispatch(*args)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 400, in dispatch_shell\n      await result\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 724, in execute_request\n      reply_content = await reply_content\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 383, in do_execute\n      res = shell.run_cell(\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 528, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2880, in run_cell\n      result = self._run_cell(\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2935, in _run_cell\n      return runner(coro)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n      coro.send(None)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3134, in run_cell_async\n      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3337, in run_ast_nodes\n      if await self.run_code(code, result, async_=asy):\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3397, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"C:\\Users\\Xia0D\\AppData\\Local\\Temp\\ipykernel_17236\\2889966517.py\", line 1, in <cell line: 1>\n      model.fit(\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 1384, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 1021, in train_function\n      return step_function(self, iterator)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 1010, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 1000, in run_step\n      outputs = model.train_step(data)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 860, in train_step\n      loss = self.compute_loss(x, y, y_pred, sample_weight)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 918, in compute_loss\n      return self.compiled_loss(\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\compile_utils.py\", line 201, in __call__\n      loss_value = loss_obj(y_t, y_p, sample_weight=sw)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\losses.py\", line 141, in __call__\n      losses = call_fn(y_true, y_pred)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\losses.py\", line 245, in call\n      return ag_fn(y_true, y_pred, **self._fn_kwargs)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\losses.py\", line 1862, in sparse_categorical_crossentropy\n      return backend.sparse_categorical_crossentropy(\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\backend.py\", line 5202, in sparse_categorical_crossentropy\n      res = tf.nn.sparse_softmax_cross_entropy_with_logits(\nNode: 'sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits'\nReceived a label value of 5 which is outside the valid range of [0, 5).  Label values: 2 0 0 2 2 2 1 0 2 2 1 2 2 3 1 2 0 1 1 0 2 1 2 5 2 0 3 0 2 1 2 1 0 5 0 0 1 1 2 2 2 0 1 2 1 0 2 0 3 5 2 2 1 0 1 2 2 2 0 0 0 0 1 0 0 2 2 1 2 1 2 5 3 0 5 3 0 2 1 0 0 1 2 2 2 0 2 2 5 3 2 5 0 1 3 1 1 0 1 1 1 2 1 2 1 2 2 1 1 0 1 2 0 5 1 0 0 5 3 1 3 1 2 2 0 2 1 1\n\t [[{{node sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits}}]] [Op:__inference_train_function_715]",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32mc:\\DevSpot\\motion-detection\\motion-detection-step3\\hand-gesture-recognition-mediapipe\\keypoint_classification_EN.ipynb Cell 16'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/DevSpot/motion-detection/motion-detection-step3/hand-gesture-recognition-mediapipe/keypoint_classification_EN.ipynb#ch0000015?line=0'>1</a>\u001b[0m model\u001b[39m.\u001b[39;49mfit(\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/DevSpot/motion-detection/motion-detection-step3/hand-gesture-recognition-mediapipe/keypoint_classification_EN.ipynb#ch0000015?line=1'>2</a>\u001b[0m     X_train,\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/DevSpot/motion-detection/motion-detection-step3/hand-gesture-recognition-mediapipe/keypoint_classification_EN.ipynb#ch0000015?line=2'>3</a>\u001b[0m     y_train,\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/DevSpot/motion-detection/motion-detection-step3/hand-gesture-recognition-mediapipe/keypoint_classification_EN.ipynb#ch0000015?line=3'>4</a>\u001b[0m     epochs\u001b[39m=\u001b[39;49m\u001b[39m1000\u001b[39;49m,\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/DevSpot/motion-detection/motion-detection-step3/hand-gesture-recognition-mediapipe/keypoint_classification_EN.ipynb#ch0000015?line=4'>5</a>\u001b[0m     batch_size\u001b[39m=\u001b[39;49m\u001b[39m128\u001b[39;49m,\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/DevSpot/motion-detection/motion-detection-step3/hand-gesture-recognition-mediapipe/keypoint_classification_EN.ipynb#ch0000015?line=5'>6</a>\u001b[0m     validation_data\u001b[39m=\u001b[39;49m(X_test, y_test),\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/DevSpot/motion-detection/motion-detection-step3/hand-gesture-recognition-mediapipe/keypoint_classification_EN.ipynb#ch0000015?line=6'>7</a>\u001b[0m     callbacks\u001b[39m=\u001b[39;49m[cp_callback, es_callback]\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/DevSpot/motion-detection/motion-detection-step3/hand-gesture-recognition-mediapipe/keypoint_classification_EN.ipynb#ch0000015?line=7'>8</a>\u001b[0m )\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\utils\\traceback_utils.py:67\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/keras/utils/traceback_utils.py?line=64'>65</a>\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:  \u001b[39m# pylint: disable=broad-except\u001b[39;00m\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/keras/utils/traceback_utils.py?line=65'>66</a>\u001b[0m   filtered_tb \u001b[39m=\u001b[39m _process_traceback_frames(e\u001b[39m.\u001b[39m__traceback__)\n\u001b[1;32m---> <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/keras/utils/traceback_utils.py?line=66'>67</a>\u001b[0m   \u001b[39mraise\u001b[39;00m e\u001b[39m.\u001b[39mwith_traceback(filtered_tb) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39m\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/keras/utils/traceback_utils.py?line=67'>68</a>\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/keras/utils/traceback_utils.py?line=68'>69</a>\u001b[0m   \u001b[39mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\execute.py:54\u001b[0m, in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/tensorflow/python/eager/execute.py?line=51'>52</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/tensorflow/python/eager/execute.py?line=52'>53</a>\u001b[0m   ctx\u001b[39m.\u001b[39mensure_initialized()\n\u001b[1;32m---> <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/tensorflow/python/eager/execute.py?line=53'>54</a>\u001b[0m   tensors \u001b[39m=\u001b[39m pywrap_tfe\u001b[39m.\u001b[39mTFE_Py_Execute(ctx\u001b[39m.\u001b[39m_handle, device_name, op_name,\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/tensorflow/python/eager/execute.py?line=54'>55</a>\u001b[0m                                       inputs, attrs, num_outputs)\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/tensorflow/python/eager/execute.py?line=55'>56</a>\u001b[0m \u001b[39mexcept\u001b[39;00m core\u001b[39m.\u001b[39m_NotOkStatusException \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m     <a href='file:///c%3A/Users/Xia0D/AppData/Roaming/Python/Python38/site-packages/tensorflow/python/eager/execute.py?line=56'>57</a>\u001b[0m   \u001b[39mif\u001b[39;00m name \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: Graph execution error:\n\nDetected at node 'sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits' defined at (most recent call last):\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\runpy.py\", line 194, in _run_module_as_main\n      return _run_code(code, main_globals, None,\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\runpy.py\", line 87, in _run_code\n      exec(code, run_globals)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\traitlets\\config\\application.py\", line 846, in launch_instance\n      app.start()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 712, in start\n      self.io_loop.start()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 199, in start\n      self.asyncio_loop.run_forever()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\asyncio\\base_events.py\", line 570, in run_forever\n      self._run_once()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\asyncio\\base_events.py\", line 1859, in _run_once\n      handle._run()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\asyncio\\events.py\", line 81, in _run\n      self._context.run(self._callback, *self._args)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 504, in dispatch_queue\n      await self.process_one()\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 493, in process_one\n      await dispatch(*args)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 400, in dispatch_shell\n      await result\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 724, in execute_request\n      reply_content = await reply_content\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 383, in do_execute\n      res = shell.run_cell(\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 528, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2880, in run_cell\n      result = self._run_cell(\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2935, in _run_cell\n      return runner(coro)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 129, in _pseudo_sync_runner\n      coro.send(None)\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3134, in run_cell_async\n      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3337, in run_ast_nodes\n      if await self.run_code(code, result, async_=asy):\n    File \"c:\\Users\\Xia0D\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3397, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"C:\\Users\\Xia0D\\AppData\\Local\\Temp\\ipykernel_17236\\2889966517.py\", line 1, in <cell line: 1>\n      model.fit(\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\utils\\traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 1384, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 1021, in train_function\n      return step_function(self, iterator)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 1010, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 1000, in run_step\n      outputs = model.train_step(data)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 860, in train_step\n      loss = self.compute_loss(x, y, y_pred, sample_weight)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\training.py\", line 918, in compute_loss\n      return self.compiled_loss(\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\engine\\compile_utils.py\", line 201, in __call__\n      loss_value = loss_obj(y_t, y_p, sample_weight=sw)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\losses.py\", line 141, in __call__\n      losses = call_fn(y_true, y_pred)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\losses.py\", line 245, in call\n      return ag_fn(y_true, y_pred, **self._fn_kwargs)\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\losses.py\", line 1862, in sparse_categorical_crossentropy\n      return backend.sparse_categorical_crossentropy(\n    File \"C:\\Users\\Xia0D\\AppData\\Roaming\\Python\\Python38\\site-packages\\keras\\backend.py\", line 5202, in sparse_categorical_crossentropy\n      res = tf.nn.sparse_softmax_cross_entropy_with_logits(\nNode: 'sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits'\nReceived a label value of 5 which is outside the valid range of [0, 5).  Label values: 2 0 0 2 2 2 1 0 2 2 1 2 2 3 1 2 0 1 1 0 2 1 2 5 2 0 3 0 2 1 2 1 0 5 0 0 1 1 2 2 2 0 1 2 1 0 2 0 3 5 2 2 1 0 1 2 2 2 0 0 0 0 1 0 0 2 2 1 2 1 2 5 3 0 5 3 0 2 1 0 0 1 2 2 2 0 2 2 5 3 2 5 0 1 3 1 1 0 1 1 1 2 1 2 1 2 2 1 1 0 1 2 0 5 1 0 0 5 3 1 3 1 2 2 0 2 1 1\n\t [[{{node sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits}}]] [Op:__inference_train_function_715]"
     ]
    }
   ],
   "source": [
    "model.fit(\n",
    "    X_train,\n",
    "    y_train,\n",
    "    epochs=1000,\n",
    "    batch_size=128,\n",
    "    validation_data=(X_test, y_test),\n",
    "    callbacks=[cp_callback, es_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pxvb2Y299hE3",
    "outputId": "59eb3185-2e37-4b9e-bc9d-ab1b8ac29b7f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10/10 [==============================] - 0s 667us/step - loss: 0.2345 - accuracy: 0.9560\n"
     ]
    }
   ],
   "source": [
    "# Model evaluation\n",
    "val_loss, val_acc = model.evaluate(X_test, y_test, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RBkmDeUW9hE4"
   },
   "outputs": [],
   "source": [
    "# Loading the saved model\n",
    "model = tf.keras.models.load_model(model_save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tFz9Tb0I9hE4",
    "outputId": "1c3b3528-54ae-4ee2-ab04-77429211cbef"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.04985807 0.07093773 0.8639351  0.01526904]\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "# Inference test\n",
    "predict_result = model.predict(np.array([X_test[0]]))\n",
    "print(np.squeeze(predict_result))\n",
    "print(np.argmax(np.squeeze(predict_result)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "S3U4yNWx9hE4"
   },
   "source": [
    "# Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 582
    },
    "id": "AP1V6SCk9hE5",
    "outputId": "08e41a80-7a4a-4619-8125-ecc371368d19"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAFlCAYAAAAjyXUiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkp0lEQVR4nO3deZwU5bX/8c+ZRWRRJLINMwioxD2CEdw3UFmior/LRb1uMSbkRjSai8nVRK8x0VwTcd+HgIIbcDEiUSAq4oLKFkFZFQRlHQEBYUCBmT6/P6YkrcI0zlRPVdd8377qRXctXWfamTlznvNUtbk7IiIiYciLOgAREUkOJRUREQmNkoqIiIRGSUVEREKjpCIiIqFRUhERkdAUZPsE29cu1pzlGujw/XOiDiEnfVq+PuoQcpZ+UGumYtsKC+u1wvh9Wdh8/9DiqYmsJxUREdlNqcqoI6g1JRURkbjwVNQR1JqSiohIXKRyP6moUS8iIqFRpSIiEhOu4S8REQlNAoa/lFREROIiAZWKeioiIhIaVSoiInGh61RERCQ0CRj+UlIREYkLNepFRCQsSZhSrEa9iIiERpWKiEhcaPhLRERCk4DhLyUVEZG40JRiEREJTQIqFTXqRUQkNKpURETiIgGNelUqIiJx4anaL9Uwsz3NbJqZvWdmc83slmD942a2xMxmBUunYL2Z2X1mtsjM3jezozJ9CapURETiIvuVylagm7uXm1khMNnMxgfbfu3uo7+xfy+gY7AcAzwc/LtLqlREROoJr1IePC0MFq/mkD7A8OC4KcA+ZlZU3TmUVEREYsK9staLmfU3sxlpS//0c5hZvpnNAlYDL7v71GDTbcEQ191m1iBYVwwsSzt8ebBulzT8JSISFyFMKXb3UqC0mu2VQCcz2wd4zswOB24AyoA9gmP/G/hDTc6vSkVEJC5Sqdovu8ndNwCTgJ7uvioY4toKPAZ0DXZbAbRNO6wkWLdLSioiInGR/dlfLYIKBTNrCJwBLPiqT2JmBpwLzAkOGQtcGswCOxb43N1XVXcODX+JiNQfRcAwM8unqqgY5e4vmNmrZtYCMGAW8J/B/uOA3sAiYAtweaYTKKmIiMRFlu/95e7vA513sr7bLvZ3YMB3OYeSiohIXCTg3l9KKiIicZGA27QoqYiIxEUCKhXN/hIRkdCoUhERiQsNf4mISGiUVEREJCxVd1DJbYlNKlu3buOyAb9m2/btVFZUcsZpJ3LVTy/h0l9cx+YtXwCwbv0Gjjj0IO67/X8Y+tRoXnxpEgCVlZUs/mQZb744gqZ77xXllxG5ouLW3PvQn2jecl/cnaeHjWbIo0/y0JBBHHBgewD2broXGz/fRI9T+kYbbEyVlLThsaH30rJVc9ydIX99ivsfGBJ1WDmhx5mnctddfyA/L4+hjz3DX+54MOqQJAOrurYle7avXZzdE+yCu/PFF1/SqFFDtldUcOkvruP6a37OkYcfsmOfa397K6eddCx9ep3+tWNfmzyF4SPHMPT+2+s67B06fP+cyM6drmWr5rRs1YI578+ncZNGjH91FFdc8ksWfrB4xz43/fE6Nm0s5547Hokw0iqflq+POoRvad26JUWtWzJz1hyaNGnM1KkT6Nv3J8yfvzDq0L4mkh/UauTl5TF/7pv07H0hy5evYso747j4kitj975VbFthYb3WF68NrfX/hoan/iS0eGoisbO/zIxGjRoCUFFRQUVFBVW3talSvnkz0959j+4nH/etY8e98jq9zzilzmKNs9WfrmXO+/MB2Fy+hYUfLqZ1Uauv7XP2uT15/tlxUYSXE8rKVjNzVtWtlMrLN7NgwULatGkdcVTx17VLZz766GOWLFnK9u3bGTXqec45u0fUYWVXlu/9VRcyDn+Z2cFUfVDLV/fQXwGMdff52QwsDJWVlfT7yS9ZumIlF/6/s/jBYQfv2DbxjXc45odH0qRx468d88WXXzJ5ygx+919X1nW4sVfStg2H/+AQZv7z/R3rjjnuh6xZ/RlLFi+NMLLc0a5dCZ2OPJxp02ZGHUrstSluzbLlK3c8X75iFV27fOsOI8mSgEZ9tZWKmf03MIKqm4xNCxYDnjGz66s5bseHxPx1+DNhxvud5Ofn8+ywB5n43BPMnvchCxd/vGPb+Fdep/fpp37rmNcmT6XzDw6t972Ub2rUuCGlw+7m97/9M+WbNu9Y3+ffevP831Sl7I7GjRsxauRgBl53M5s2lWc+QOqfelCpXAEc5u7b01ea2V3AXGCnTYf0D4mJqqeSbu+9mtD1qB8wecoMOu7fnvUbPmf2vA+49083fWvf8RN3nmzqs4KCAkqH3cNzo19k/Auv7Fifn59Pr7NOp3e3fhFGlxsKCgoYNXIwzzzzHGPGjM98gLByRRltS9rseF5SXMTKlWURRiS7I1NPJQW02cn6omBbbK1bv4GNwV+DX27dyjvTZ9KhXdVnzbw0aTKnHN+VBg32+Noxm8o3M2PmbE476dt9lvps0H1/YNGHixn80PCvrT/p1GP5aOFiVq38NKLIcsfg0jtZsGAR99y7yw/kk2+YPmMWBx7Ygfbt21JYWEi/fn34+wsvRR1WdtXhh3RlS6ZK5Vpgopkt5F+fU7wfcCBwVRbjqrU1n63nd7cOojKVwlNOj24nceoJxwBV1chPL/72X9cTX3+b47seRaOGe9Z1uLHV5ZjO9L3gHObP/ZB/vD4agD//8V5efeVNzjmvF2Oe1V/dmZxwfBcuvrgvs2fPY8b0ql+KN950OxMmvBpxZPFWWVnJNdfeyLgXnyY/L4/Hh41k3rwPow4ru2IwfFVbGacUm1keVR8tmd6on+67eZVOHIa/clFcphTnmjhOKc4V+kGtmVCnFI+/r/ZTinv9MtIpxRlnf7l7CphSB7GIiEiOS+wV9SIiOScGPZHaUlIREYmLBPRUlFREROJClYqIiIQmAZVKYu/9JSIidU+ViohIXGj4S0REQpOA4S8lFRGRuFClIiIioUlAUlGjXkREQqNKRUQkLrL88e51QUlFRCQuEjD8paQiIhIXCUgq6qmIiEhoVKmIiMSFrlMREZHQJGD4S0lFRCQuEjD7Sz0VEZG4SKVqv1TDzPY0s2lm9p6ZzTWzW4L1HcxsqpktMrORZrZHsL5B8HxRsL19pi9BSUVEpP7YCnRz9yOBTkBPMzsW+DNwt7sfCKwHrgj2vwJYH6y/O9ivWkoqIiJxkeVKxauUB08Lg8WBbsDoYP0w4NzgcZ/gOcH27mZm1Z1DSUVEJC48VevFzPqb2Yy0pX/6Kcws38xmAauBl4GPgA3uXhHsshwoDh4XA8sAgu2fA/tW9yWoUS8iEhOeqn2j3t1LgdJqtlcCncxsH+A54OBanzSNkoqISFzU4ZRid99gZpOA44B9zKwgqEZKgBXBbiuAtsByMysAmgKfVfe6Gv4SEaknzKxFUKFgZg2BM4D5wCSgb7DbZcDzweOxwXOC7a+6Vz/vWZWKiEhcZP+K+iJgmJnlU1VUjHL3F8xsHjDCzG4FZgJDgv2HAE+Y2SJgHXBBphMoqYiIxEUIPZXquPv7QOedrF8MdN3J+i+Bf/8u51BSERGJiwTcpkU9FRERCY0qFRGRuEhApaKkIiISFwm4oaSSiohIXKhSERGR0GR59lddUKNeRERCo0pFRCQu9HHCIiISmgQMf2U9qTRsc1K2T5FIWz4aF3UIOanZQX2iDiFnba+syLyTZJWrUS8iIqFJQKWiRr2IiIRGlYqISFyoUS8iIqFJwPCXkoqISFwkoFGvnoqIiIRGlYqISFxo+EtEREKjRr2IiIRGlYqIiIQlCVfUq1EvIiKhUaUiIhIXGv4SEZHQKKmIiEhoNPtLRERCk4BKRY16EREJjSoVEZGY8ARUKkoqIiJxoaQiIiKh0cWPIiIi/6JKRUQkLjT8JSIioVFSERGRsLgrqYiISFgSUKmoUS8iUk+YWVszm2Rm88xsrpldE6z/vZmtMLNZwdI77ZgbzGyRmX1gZj0ynUOViohIXGS/UqkABrr7u2a2F/BPM3s52Ha3uw9K39nMDgUuAA4D2gCvmNn33b1yVydQUhERiYlsX1Hv7quAVcHjTWY2Hyiu5pA+wAh33wosMbNFQFfgnV0doOEvEZG4SHntl91kZu2BzsDUYNVVZva+mQ01s2bBumJgWdphy6k+CSmpiIjERqr2i5n1N7MZaUv/b57GzJoAzwLXuvtG4GHgAKATVZXMnTX9EjT8JSKSIO5eCpTuaruZFVKVUJ5y978Fx3yatn0w8ELwdAXQNu3wkmDdLqlSERGJCU95rZfqmJkBQ4D57n5X2vqitN3OA+YEj8cCF5hZAzPrAHQEplV3DlUqIiJxkf3ZXycAlwCzzWxWsO63wIVm1glw4GPg5wDuPtfMRgHzqJo5NqC6mV+gpCIiEh9Zvkmxu08GbCebxlVzzG3Abbt7Dg1/iYhIaFSpiIjEhD75UUREwpP7n9FVP5PK4NI7+VHv01m9Zi2dOnePOpzY2bptGz/+1c1s215BZWUlZ5x8LAMu64e7c/9jI3jp9Snk5edx/tlncNF5vdlUvoUbbr+PVas/o7Kyksv+/WzO63la1F9GpB5+5C/06tmNNWs+o0uXqtslNWvWlOHDH2C/diUs/WQ5l1wygA0bNkYcaXw1aNCAVyc+S4MGe1BQkM/f/jaOP/yxxpdP5IQkVCr1sqcyfPgofnTWRVGHEVt7FBYyZNDNPFt6B//36F94a/os3pv3IWP+8Rplqz9j7GN3M3bo3fQ89QQARoydwP7tSni29A6G3vl7Bj06nO3bKyL+KqL15BOjOffcy762buDAX/Daa29z5A9O47XX3mbgwCsjii43bN26lTN79OPoLmdydJcenHnmqXTtelTUYWVXCBc/Rq1eJpU3J09l3foNUYcRW2ZGo4Z7AlBRUUlFRSVmxqi/v8R/XtKXvLyqb5t9mzWt2h9jy5YvcXe2fPElTfdqQn5+vfzW2uGtt6axbt3nX1v3o7PO4KmnRgPw1FOjOevsM6IILads3rwFgMLCAgoLCxLxeSNJVy+HvySzysoU51/53yxdUcYFfXrwg0M6smzlp0x47W0mvjWNZk335oYBl9OupIgLz+3J1Tf9hW7n/5zNW75g0I2/2pF45F9atmxBWdkaAMrK1tCyZYuII4q/vLw8pk4ZzwEHtOeRR4YxffrMqEPKKo9BpVFbNf7JN7PLq9m2494zqdTmmp5CIpSfn8foR+/glRGPMGfBRyxcspRt27fTYI9CRj50O317d+d/Bj0MwFsz3uOgA9rx6shHGf3oHfzpgSGUB39hyq7pr+7MUqkUXbr2oMP+XTj66E4cduhBUYeUXfV8+OuWXW1w91J3P9rdj87La1yLU0jU9m7SmC6dDuOt6bNo1WJfup94DADdT+zKh4s/AWDMhEmcftIxmBn7FbemuHVLlixbGWXYsbR69Rpat66qTlq3bsGaNWsjjih3fP75Rl5//W3O7HFq1KFkladqv0St2qQS3AZ5Z8tsoFUdxSh1bN2GjWwsr6owv9y6jSn/fJ8O+xXT7fguTJ9VdUugGe/No11JGwCKWjZn6ruzAVi7fgMfL1tJSVHLaIKPsXEvvsJFF/UF4KKL+vLiCy9nOKJ+a978ezRtujcAe+65J927n8QHHyyKOKosS0Clkqmn0groAaz/xnoD3s5KRHXgySce5JSTj6N58+/x8eIZ3PKHQTz2+Iiow4qNNevWc+OfH6QylcLdOfOU4zjl2B/S+fCDuf5P9zH82Rdp1HBPbhn4cwB+fvG/ceMdD3HeTwcCcO3PLqJZ8Mugvnr88fs46eRj2XffZny48B1uvfVu7rzzYZ544kEuvawfy5au4JJLBkQdZqwVtW7FkCF3k5+fT16eMXr0C4wbNzHqsCQDq25c18yGAI8F94v55ran3f0/Mp2gYI9iDRzXwJaPdnkrHqlGs4P6RB1CztpeWb+ngdfUtq3Ld3YvrRpZc8Yptf592eLl10OLpyaqrVTc/YpqtmVMKCIisvvi0BOpLU0pFhGJiSQkFV1MICIioVGlIiISFx5pOyQUSioiIjGRhOEvJRURkZjwlCoVEREJSRIqFTXqRUQkNKpURERiwtWoFxGRsCRh+EtJRUQkJpLQqFdPRUREQqNKRUQkJpLwuW1KKiIiMZGE4S8lFRGRmFBSERGR0CRh+EuNehERCY0qFRGRmNDwl4iIhEZX1IuISGh0Rb2IiIQmlYBKRY16EREJjZKKiEhMuFutl+qYWVszm2Rm88xsrpldE6z/npm9bGYLg3+bBevNzO4zs0Vm9r6ZHZXpa1BSERGJCU9ZrZcMKoCB7n4ocCwwwMwOBa4HJrp7R2Bi8BygF9AxWPoDD2c6gZKKiEhMuNd+qf71fZW7vxs83gTMB4qBPsCwYLdhwLnB4z7AcK8yBdjHzIqqO4eSiohIPWRm7YHOwFSglbuvCjaVAa2Cx8XAsrTDlgfrdkmzv0REYiKMix/NrD9VQ1VfKXX30m/s0wR4FrjW3Tea/eu87u5mVuMbxiipiIjERBhTioMEUrqr7WZWSFVCecrd/xas/tTMitx9VTC8tTpYvwJom3Z4SbBulzT8JSISE3Uw+8uAIcB8d78rbdNY4LLg8WXA82nrLw1mgR0LfJ42TLZTqlRERGKiDu5SfAJwCTDbzGYF634L3A6MMrMrgE+AfsG2cUBvYBGwBbg80wmUVERE6gl3nwzsqpzpvpP9HRjwXc6hpCIiEhNJuE2LkoqISEzoLsUiIhKaJHzyo5KKiEhMJGH4S1OKRUQkNFmvVHI/70aj0QG9ow4hJ20aeXXUIeSsphc8EHUI9Z56KiIiEpokDH8pqYiIxEQC+vTqqYiISHhUqYiIxISGv0REJDRq1IuISGhSUQcQAiUVEZGY8ARchKFGvYiIhEaViohITKQSMKdYSUVEJCZSCRj+UlIREYmJJPRUlFRERGIiCbO/1KgXEZHQqFIREYkJDX+JiEhokjD8paQiIhITSUgq6qmIiEhoVKmIiMSEeioiIhKaVO7nFCUVEZG40BX1IiISmgTc+kuNehERCY8qFRGRmEjClGIlFRGRmEiZeioiIhKSJPRUlFRERGIiCcNfatSLiEhoVKmIiMSELn4UEZHQJOHiRw1/iYjEhIewZGJmQ81stZnNSVv3ezNbYWazgqV32rYbzGyRmX1gZj0yvb6SiohI/fI40HMn6+92907BMg7AzA4FLgAOC455yMzyq3txJRURkZhIWe2XTNz9DWDdbobUBxjh7lvdfQmwCOha3QFKKiIiMZEKYTGz/mY2I23pv5unv8rM3g+Gx5oF64qBZWn7LA/W7ZKSiohITITRU3H3Unc/Om0p3Y1TPwwcAHQCVgF31vRr0OwvEZGYiGpKsbt/+tVjMxsMvBA8XQG0Tdu1JFi3S/UyqZSUtOGxoffSslVz3J0hf32K+x8YEnVYsTe49E5+1Pt0Vq9ZS6fO3aMOJ3a2bq/gJ4+OZ3tFJRUp5/Qj2nPlGZ0Z8fY8nnprHss+28Skmy6kWeM9AXj89dmMm7UYgMpUiiWrP2fSTRfStFGDKL+M2MnLy2PKO+NYsbKM8877cdThJJKZFbn7quDpecBXM8PGAk+b2V1AG6AjMK2616qXSaWiooLf/OYWZs6aQ5MmjZk6dQKvTHyD+fMXRh1arA0fPoqHHnqMxx67N+pQYmmPgnwG/6wnjRoUsr0yxeWPvMiJBxXTqV0rTjq4LT8tnfC1/X98yhH8+JQjAHh93lKenDxXCWUnrr76ChYsWMReezeJOpSsq4vbtJjZM8CpQHMzWw7cDJxqZp2oGkH7GPg5gLvPNbNRwDygAhjg7pXVvX69TCplZaspK1sNQHn5ZhYsWEibNq2VVDJ4c/JU2rUriTqM2DIzGjUoBKCiMkVFZQrDOLh434zHjn9vCT077Z/tEHNOcXERvXp15/bb7+Oaa3a335y76iKpuPuFO1m9y6Ead78NuG13Xz9jo97MDjaz7mbW5BvrdzbPOee0a1dCpyMPZ9q0mVGHIglQmUrR797n6XbrMxzbsQ1H7Nci4zFfbKvg7Q+Xc/rh7bMfYI65c9DvueGG20ilknD/3szcar9ErdqkYma/BJ4HrgbmmFmftM1/qua4HVPaUqnN4USaBY0bN2LUyMEMvO5mNm0qjzocSYD8vDxGXdOHf9zQjznL1rKobH3GY96Yv5RO7Vpp6Osbevfuzuo1a5k5c3bUodSZMKYURy3T8NfPgB+6e7mZtQdGm1l7d78Xdn2TmmAKWylA4R7FsfwTo6CggFEjB/PMM88xZsz4qMORhNm7YQO67F/EWx8u58DWzardd8J7S+jZqUMdRZY7jj+uC2f96Ex69ujGnns2YO+99+Lxx+7jx5f/MurQpBqZhr/y3L0cwN0/pqq50yuYCRCDQqvmBpfeyYIFi7jn3t2Zwi2S2bryL9n4xVYAvtxewZRFK+nQYp9qj9n05Tb+uaSM0w7drw4izC033nQ7+x/Qhe8fdBwXXzKASa+9lfiEkoRKJVNS+TSYEQBAkGDOApoDR2Qxrqw64fguXHxxX0477XhmTH+JGdNfomfPblGHFXtPPvEgk98Yy0HfP4CPF8/g8h9fEHVIsbJ20xZ+VjqBf79nDBc98HeOPbANJx/SlqffmseZfxrJ6o2b6XfPGG4ZPXnHMa/O+YTjOhbTcI/CCCOXuKiLG0pmm7nvOgwzKwEq3L1sJ9tOcPe3Mp0grsNfcac3rWY2jbw66hByVtMLHog6hJy0bevy0EZt7t3v4lr/6F+z9MlIR5Gq7am4+/JqtmVMKCIiUr/Uy+tURETiKA49kdpSUhERiQklFRERCU0SeqlKKiIiMRHVXYrDpM9TERGR0KhSERGJCfVUREQkNOqpiIhIaFIJSCvqqYiISGhUqYiIxIR6KiIiEprcH/xSUhERiQ1VKiIiEhpd/CgiIpJGlYqISEwkYUqxkoqISEzkfkpRUhERiQ016kVEJDRJGP5So15EREKjSkVEJCZyv05RUhERiQ31VEREJDTqqYiIiKRRpSIiEhO5X6coqYiIxIZ6KiIiEhpPQK2ipCIiEhNJqFTUqBcRqUfMbKiZrTazOWnrvmdmL5vZwuDfZsF6M7P7zGyRmb1vZkdlen0lFRGRmEjhtV52w+NAz2+sux6Y6O4dgYnBc4BeQMdg6Q88nOnFlVRERGLCQ1gynsP9DWDdN1b3AYYFj4cB56atH+5VpgD7mFlRda+vpCIiEhNhVCpm1t/MZqQt/Xfj1K3cfVXwuAxoFTwuBpal7bc8WLdLatSLiMREGI16dy8FSmtxvJtZjaehqVIREZFPvxrWCv5dHaxfAbRN268kWLdLSioiIjHhIfxXQ2OBy4LHlwHPp62/NJgFdizwedow2U5p+EtEJCbq4joVM3sGOBVobmbLgZuB24FRZnYF8AnQL9h9HNAbWARsAS7P9PpZTyq5f31oNCzqAHLUXuffH3UIOatbqyOiDqHeq4sr6t39wl1s6r6TfR0Y8F1eX8NfIiISGg1/iYjERBJu06KkIiISEynP/YaBkoqISEzkfkpRUhERiQ19nLCIiEgaVSoiIjGhD+kSEZHQaPaXiIiEJgk9FSUVEZGYSMLwlxr1IiISGlUqIiIxoZ6KiIiExnVFvYiIhCUJjXr1VEREJDSqVEREYkI9FRERCU0SphQrqYiIxEQSeipKKiIiMZGE2V9q1IuISGhUqYiIxIQa9SIiEho16kVEJDRq1IuISGjUqBcREUmjSkVEJCY0/CUiIqFRo15EREKTUk9FRETkX1SpiIjERO7XKUoqIiKxoUa9iIiERklFRERCo4sfRURE0qhSERGJiboY/jKzj4FNQCVQ4e5Hm9n3gJFAe+BjoJ+7r6/J66tSERGJCQ/hv910mrt3cvejg+fXAxPdvSMwMXheI/U2qfQ481TmznmDBfMm85tfD4g6nJxQUtKGl1/6P957bxKzZr3K1VddEXVIOUPfb7uv8d6NuemR3zFk0mD++mophxx1CPsf0oF7xtzNoy8/zB+G/p5GTRpFHWZWuHutlxrqAwwLHg8Dzq3pC1m2G0MFexTHrvOUl5fH/Llv0rP3hSxfvoop74zj4kuuZP78hVGHtoNFHcBOtG7dkqLWLZk5aw5NmjRm6tQJ9O37k1i9b7H7ZiM3vt8AurU6IuoQAPj1XQOZPW0uE0ZMoKCwgAYNG3D70/9L6a2DmT1lNj3OP5PWbVszbNDwqEMF4KVlE0L7cT2q6MRafwu/u2pytfGY2RJgPVU/Lo+6e6mZbXD3fYLtBqz/6vl3VS8rla5dOvPRRx+zZMlStm/fzqhRz3PO2T2iDiv2yspWM3PWHADKyzezYMFC2rRpHXFU8afvt93XaK9GHHHMEUwYMQGAiu0VbN64mZIOxcyeMhuAd994lxN7nRBlmLFmZv3NbEba0v8bu5zo7kcBvYABZnZy+kavqjRqnNwyJhUz62pmXYLHh5rZf5lZ75qeMA7aFLdm2fKVO54vX7FKvxy/o3btSuh05OFMmzYz6lBiT99vu69129ZsWPc51901kIfGP8Cv/nItezZswMcffsLxPY4D4OSzTqZFmxYRR5odYQx/uXupux+dtpR+4xwrgn9XA88BXYFPzawIIPh3dU2/hmqTipndDNwHPGxm/ws8ADQGrjez31Vz3I5MmUptrmlsElONGzdi1MjBDLzuZjZtKo86HEmQ/IJ8Oh5+IC8Mf4Ere13Fl1u+5PwB53PXdXdx9qVn8eCL99OwcUMqtldEHWpWpPBaL9Uxs8ZmttdXj4EzgTnAWOCyYLfLgOdr+jVkmlLcF+gENADKgBJ332hmg4CpwG07OyjIjKUQz57KyhVltC1ps+N5SXERK1eWRRhR7igoKGDUyME888xzjBkzPupwcoK+33bf2lVrWbNqLQtmfQDAm+Pe5Pwrz2fYoOHccFHV37HFHYrp2r1rlGFmTR3c+r4V8FxV24QC4Gl3n2Bm04FRZnYF8AnQr6YnyDT8VeHule6+BfjI3TcCuPsXQKqmJ43a9BmzOPDADrRv35bCwkL69evD3194KeqwcsLg0jtZsGAR99xbmnlnAfT99l2sX7OeNavWULJ/CQCdT+jM0oVL2WffpgCYGf/xywt58ckXowwzZ7n7Ync/MlgOc/fbgvWfuXt3d+/o7qe7+7qaniNTpbLNzBoFSeWHX600s6bkcFKprKzkmmtvZNyLT5Ofl8fjw0Yyb96HUYcVeycc34WLL+7L7NnzmDG96pfijTfdzoQJr0YcWbzp++27efCmh7j+/t9QUFhI2dJVDBp4F6f/W3fOuexsACaPf4t/jExmUk7C56lUO6XYzBq4+9adrG8OFLn77EwniOPwVy6I45TiXKBvtpqLy5TiXBPmlOLDWh1T62/huZ9OjfTXR7WVys4SSrB+LbA2KxGJiNRTSahUdO8vEZGYSMJn1NfLix9FRCQ7VKmIiMSEhr9ERCQ0SRj+UlIREYkJVSoiIhKaJFQqatSLiEhoVKmIiMSEe87eqGQHJRURkZioi8+ozzYlFRGRmMj2J/HWBfVUREQkNKpURERiQsNfIiISmiQMfympiIjEhC5+FBGR0OjiRxERkTSqVEREYkI9FRERCY1mf4mISGiSUKmopyIiIqFRpSIiEhOaUiwiIqFJwvCXkoqISEyoUS8iIqFJQqWiRr2IiIRGlYqISEyoUS8iIqFJwr2/lFRERGJClYqIiIRGjXoREZE0qlRERGIiCT0VVSoiIjHh7rVeMjGznmb2gZktMrPrw/4aVKmIiMREtnsqZpYPPAicASwHppvZWHefF9Y5VKmIiNQfXYFF7r7Y3bcBI4A+YZ5ASUVEJCY8hCWDYmBZ2vPlwbrQZH34q2LbCsv2OWrKzPq7e2nUceQavW81p/euZurL+xbG70sz6w/0T1tVWpfvXX2vVPpn3kV2Qu9bzem9qxm9b7vJ3Uvd/ei0JT2hrADapj0vCdaFpr4nFRGR+mQ60NHMOpjZHsAFwNgwT6DZXyIi9YS7V5jZVcA/gHxgqLvPDfMc9T2pJH6MNkv0vtWc3rua0fsWEncfB4zL1utbEu41IyIi8aCeioiIhKbeJpVs36ogicxsqJmtNrM5UceSS8ysrZlNMrN5ZjbXzK6JOqZcYGZ7mtk0M3sveN9uiTomyaxeDn8Ftyr4kLRbFQAXhnmrgiQys5OBcmC4ux8edTy5wsyKgCJ3f9fM9gL+CZyr77fqmZkBjd293MwKgcnANe4+JeLQpBr1tVLJ+q0Kksjd3wDWRR1HrnH3Ve7+bvB4EzCfkK9iTiKvUh48LQyW+vdXcI6pr0kl67cqENkZM2sPdAamRhxKTjCzfDObBawGXnZ3vW8xV1+TikidM7MmwLPAte6+Mep4coG7V7p7J6qu/O5qZhp2jbn6mlSyfqsCkXRBT+BZ4Cl3/1vU8eQad98ATAJ6RhyKZFBfk0rWb1Ug8pWg4TwEmO/ud0UdT64wsxZmtk/wuCFVE2sWRBqUZFQvk4q7VwBf3apgPjAq7FsVJJGZPQO8AxxkZsvN7IqoY8oRJwCXAN3MbFaw9I46qBxQBEwys/ep+kPwZXd/IeKYJIN6OaVYRESyo15WKiIikh1KKiIiEholFRERCY2SioiIhEZJRUREQqOkIiIioVFSERGR0CipiIhIaP4/dAs0qKk+EFYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 504x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.93      0.96       406\n",
      "           1       0.92      0.96      0.94       382\n",
      "           2       0.96      0.98      0.97       324\n",
      "           3       0.91      0.97      0.94        71\n",
      "\n",
      "    accuracy                           0.96      1183\n",
      "   macro avg       0.95      0.96      0.95      1183\n",
      "weighted avg       0.96      0.96      0.96      1183\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "\n",
    "def print_confusion_matrix(y_true, y_pred, report=True):\n",
    "    labels = sorted(list(set(y_true)))\n",
    "    cmx_data = confusion_matrix(y_true, y_pred, labels=labels)\n",
    "    \n",
    "    df_cmx = pd.DataFrame(cmx_data, index=labels, columns=labels)\n",
    " \n",
    "    fig, ax = plt.subplots(figsize=(7, 6))\n",
    "    sns.heatmap(df_cmx, annot=True, fmt='g' ,square=False)\n",
    "    ax.set_ylim(len(set(y_true)), 0)\n",
    "    plt.show()\n",
    "    \n",
    "    if report:\n",
    "        print('Classification Report')\n",
    "        print(classification_report(y_test, y_pred))\n",
    "\n",
    "Y_pred = model.predict(X_test)\n",
    "y_pred = np.argmax(Y_pred, axis=1)\n",
    "\n",
    "print_confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FNP6aqzc9hE5"
   },
   "source": [
    "# Convert to model for Tensorflow-Lite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ODjnYyld9hE6"
   },
   "outputs": [],
   "source": [
    "# Save as a model dedicated to inference\n",
    "model.save(model_save_path, include_optimizer=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zRfuK8Y59hE6",
    "outputId": "a4ca585c-b5d5-4244-8291-8674063209bb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: C:\\Users\\Xia0D\\AppData\\Local\\Temp\\tmpu51ewteh\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Buffer deduplication procedure will be skipped when flatbuffer library is not properly loaded\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6416"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Transform model (quantization)\n",
    "\n",
    "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n",
    "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
    "tflite_quantized_model = converter.convert()\n",
    "\n",
    "open(tflite_save_path, 'wb').write(tflite_quantized_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CHBPBXdx9hE6"
   },
   "source": [
    "# Inference test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mGAzLocO9hE7"
   },
   "outputs": [],
   "source": [
    "interpreter = tf.lite.Interpreter(model_path=tflite_save_path)\n",
    "interpreter.allocate_tensors()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oQuDK8YS9hE7"
   },
   "outputs": [],
   "source": [
    "# Get I / O tensor\n",
    "input_details = interpreter.get_input_details()\n",
    "output_details = interpreter.get_output_details()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2_ixAf_l9hE7"
   },
   "outputs": [],
   "source": [
    "interpreter.set_tensor(input_details[0]['index'], np.array([X_test[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "s4FoAnuc9hE7",
    "outputId": "91f18257-8d8b-4ef3-c558-e9b5f94fabbf",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 0 ns\n",
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Inference implementation\n",
    "interpreter.invoke()\n",
    "tflite_results = interpreter.get_tensor(output_details[0]['index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vONjp19J9hE8",
    "outputId": "77205e24-fd00-42c4-f7b6-e06e527c2cba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.04985808 0.07093775 0.8639351  0.01526904]\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "print(np.squeeze(tflite_results))\n",
    "print(np.argmax(np.squeeze(tflite_results)))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "keypoint_classification_EN.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "interpreter": {
   "hash": "df4d5c61567f020e3a860d559b13db8f0e5f5cd9ae100d8cced79bf21b6d70d8"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}