howto/4_evaluation/evaluation_webcam_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation with video feed\n",
    "\n",
    "This notebook runs the hand detection and gesture classification models in the same manner at the gesture control pipeline. It uses the webcam of a laptop and disclays the live video feed adding boxes around hands and outputting the detected gestures. It can be used to manually test the gesture recognition pipeline on the laptop under changing light conditions or other factors as wished.\n",
    "\n",
    "All needed adjustments are marked with \"Todo\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import cv2\n",
    "import tensorflow as tf\n",
    "import pathlib\n",
    "import time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ToDo: Only necessary if you changed the selected gestures in the script data_preprocessing_lared.ipynb.\n",
    "# adjust the names and quantity of the gestures here too.\n",
    "INDEX_TO_LABEL = {\n",
    "    0 :\"crawl\",\n",
    "    1 :\"fist\",\n",
    "    2 :\"five\",\n",
    "    3 :\"palm\",\n",
    "    4 :\"peace\",\n",
    "}\n",
    "\n",
    "IMAGE_HEIGHT = 300\n",
    "IMAGE_WIDTH = 300\n",
    "\n",
    "# Todo: change thresholds to and see how the models react.\n",
    "# Lower the threshold to see objects like ears to be detected as hands.\n",
    "THRESHOLD_HAND_DETECTION = 0.8\n",
    "THRESHOLD_GESTURE_CLASSIFICATION = 0.95\n",
    "\n",
    "font = cv2.FONT_HERSHEY_SIMPLEX\n",
    "color = (0,205,205)\n",
    "fontScale = 0.35\n",
    "thickness = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Helper methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_detected_hand_boxes(img):\n",
    "    \"\"\" Takes in image of arbitrary size and returns bounding boxes along with confidence scores after detecting\n",
    "    hands in the image.\n",
    "    \"\"\"\n",
    "    input_tensor = tf.convert_to_tensor(img)\n",
    "    input_tensor = input_tensor[tf.newaxis,...]\n",
    "    \n",
    "    hand_detections = model_detect(input_tensor)\n",
    "    \n",
    "    boxes, hand_scores = filter_detected_boxes(hand_detections, threshold=THRESHOLD_HAND_DETECTION)\n",
    "    \n",
    "    return boxes, hand_scores\n",
    "\n",
    "\n",
    "def filter_detected_boxes(hand_detections, threshold):\n",
    "    \"\"\" Filter the detected hands according to defined threshold.\n",
    "    \n",
    "    Only bounding boxes that have been detected above the threshold will be returned.\n",
    "    Returns: \n",
    "        boxes: Bounding boxes of detected hands\n",
    "        hand_scores: Probability that the detected object is a hand\n",
    "    \"\"\"\n",
    "    boxes_raw = hand_detections['detection_boxes'].numpy()[0]\n",
    "    scores = hand_detections['detection_scores'].numpy()[0]\n",
    "    \n",
    "    boxes = []\n",
    "    hand_scores = []\n",
    "    for i, score in enumerate(scores):\n",
    "        if score > threshold:\n",
    "            box = boxes_raw[i]\n",
    "            \n",
    "            ymin = int(float(box[0])*IMAGE_HEIGHT)\n",
    "            xmin = int(float(box[1])*IMAGE_WIDTH)\n",
    "            ymax = int(float(box[2])*IMAGE_HEIGHT)\n",
    "            xmax = int(float(box[3])*IMAGE_WIDTH)\n",
    "\n",
    "            score = round(score,2)\n",
    "            hand_scores.append(score)\n",
    "            \n",
    "            boxes.append((xmin, ymin, xmax, ymax))\n",
    "            \n",
    "    return boxes, hand_scores\n",
    "\n",
    "\n",
    "def add_boxes_to_img(img, boxes, hand_scores, gesture_names, gesture_scores):\n",
    "    \"\"\" Draw bounding boxes on an image\n",
    "    \n",
    "    The calculated probabilities for a hand detection or a gesture classification will be added to the image.\n",
    "    The gesture names will be drawn on the image.    \n",
    "    \n",
    "    Input:\n",
    "        img: Image that contains the bounding boxes\n",
    "        boxes: The bounding boxes to draw\n",
    "        hand_scores: The probability of a detected hand\n",
    "        gesture_names: The names of classified gestures\n",
    "        gesture_scores: The probability of a classified gesture\n",
    "    Returns: \n",
    "        img: The input image with drawn bounding boxes and scores\n",
    "    \"\"\"\n",
    "    \n",
    "    for box, hand_score, gesture_name, gesture_score in zip(boxes, hand_scores, gesture_names, gesture_scores):\n",
    "        (xmin, ymin, xmax, ymax) = box\n",
    "\n",
    "        img = cv2.rectangle(img, (xmin, ymin), (xmax, ymax), color, 1)\n",
    "        img = cv2.putText(img, 'hand ' + str(hand_score), (xmin+2, ymin+10), font, fontScale, color, thickness, cv2.LINE_AA)\n",
    "        \n",
    "        if gesture_name:\n",
    "            img = cv2.putText(img, gesture_name +\" \"+ gesture_score, (xmin+2, ymax-5), font, fontScale, color, thickness, cv2.LINE_AA)\n",
    "        \n",
    "    return img\n",
    "\n",
    "\n",
    "def crop_rect(img, xmin, xmax, ymin, ymax):\n",
    "    \"\"\" Crop the the rectangular image to a square\n",
    "    \n",
    "    Input:\n",
    "        img: Image that contains the bounding box\n",
    "        xmin, xmax, ymin, ymax: Coordinates of a hand bounding box\n",
    "    Returns: \n",
    "        img: Input image cropped to a square\n",
    "    \"\"\"\n",
    "    x, y = xmin, ymin\n",
    "    w = xmax - xmin # width\n",
    "    h = ymax - ymin # height\n",
    "    \n",
    "    # crop a square form\n",
    "    if w > h:\n",
    "        y = y - int((w-h)/2)\n",
    "        h = w    \n",
    "        #make sure y is within picture\n",
    "        y = max(y,0)\n",
    "        y = min(y, IMAGE_HEIGHT-h)\n",
    "\n",
    "    elif h > w:\n",
    "        x = x - int((h-w)/2)\n",
    "        w = h\n",
    "        #make sure x is within picture\n",
    "        x = max(x,0)\n",
    "        x = min(x,IMAGE_WIDTH-w)\n",
    "        \n",
    "    img = img[y:y+h, x:x+w]\n",
    "    img = cv2.resize(img, (64,64))\n",
    "        \n",
    "    return img\n",
    "\n",
    "\n",
    "def get_prediction_gesture(img):\n",
    "    \"\"\" Predict the gesture on the input image\n",
    "    \n",
    "    Input:\n",
    "        img: Image that contains the gesture\n",
    "    Returns: \n",
    "        gesture_name: The name of the predicted gesture\n",
    "        confidence: The probability for that gesture\n",
    "    \"\"\"\n",
    "\n",
    "    img =  cv2.resize(img, (64,64))\n",
    "    img = img/255\n",
    "    img = img.astype('float32')\n",
    "    input_tensor = tf.convert_to_tensor(img)\n",
    "    input_tensor = input_tensor[tf.newaxis,...]\n",
    "    \n",
    "    predictions = model_classify(input_tensor)\n",
    "    \n",
    "    predicted_index = np.argmax(predictions, axis=1)[0]\n",
    "    confidence = round(np.max(predictions[0]), 2)\n",
    "    \n",
    "    if confidence > THRESHOLD_GESTURE_CLASSIFICATION:\n",
    "        gesture_name = INDEX_TO_LABEL[predicted_index]\n",
    "    else:\n",
    "        gesture_name = \"\"\n",
    "    \n",
    "    return gesture_name, str(confidence)\n",
    "\n",
    "\n",
    "def classify_gestures_for_boxes(boxes):\n",
    "    \"\"\" Classify the gestures for all bounding boxes \n",
    "    \n",
    "    Input:\n",
    "        boxes: Bounding boxes containing hand detections\n",
    "    Returns: \n",
    "        gesture_names: The names of the classified gestures\n",
    "        gesture_scores: The probability of the classifed gestures\n",
    "    \"\"\"\n",
    "    gesture_names = []\n",
    "    gesture_scores = []\n",
    "    \n",
    "    for box in boxes:\n",
    "        (xmin, ymin, xmax, ymax) = box\n",
    "        cropped_image = crop_rect(img, xmin, xmax, ymin, ymax)\n",
    "        gesture_name, gesture_score = get_prediction_gesture(cropped_image)\n",
    "        gesture_names.append(gesture_name)\n",
    "        gesture_scores.append(gesture_score)\n",
    "        \n",
    "    return gesture_names, gesture_scores\n",
    "\n",
    "\n",
    "def filter_detected_boxes(hand_detections, threshold):\n",
    "    \"\"\" Filter the detected hands according to defined threshold.\n",
    "    \n",
    "    Only bounding boxes that have been detected above the threshold will be returned.\n",
    "    Returns: \n",
    "        boxes: Bounding boxes of detected hands\n",
    "        hand_scores: Probability that the detected object is a hand\n",
    "    \"\"\"\n",
    "    boxes_raw = hand_detections['detection_boxes'].numpy()[0]\n",
    "    scores = hand_detections['detection_scores'].numpy()[0]\n",
    "    \n",
    "    boxes = []\n",
    "    hand_scores = []\n",
    "    for i, score in enumerate(scores):\n",
    "        if score > threshold:\n",
    "            box = boxes_raw[i]\n",
    "            \n",
    "            ymin = int(float(box[0])*IMAGE_HEIGHT)\n",
    "            xmin = int(float(box[1])*IMAGE_WIDTH)\n",
    "            ymax = int(float(box[2])*IMAGE_HEIGHT)\n",
    "            xmax = int(float(box[3])*IMAGE_WIDTH)\n",
    "\n",
    "            score = round(score,2)\n",
    "            hand_scores.append(score)\n",
    "            \n",
    "            boxes.append((xmin, ymin, xmax, ymax))\n",
    "            \n",
    "    return boxes, hand_scores\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Load models\n",
    "\n",
    "The detection and classification models are needed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "hand detection model loaded\n",
      "gesture classification model loaded\n"
     ]
    }
   ],
   "source": [
    "# hand detection model\n",
    "model_dir = pathlib.Path(\"../2_detection/model_ssd_mobilenetV2\")\n",
    "model_detect = tf.saved_model.load(str(model_dir))\n",
    "model_detect = model_detect.signatures['serving_default']\n",
    "print(\"hand detection model loaded\")\n",
    "\n",
    "# gesture classification model\n",
    "model_dir = pathlib.Path(\"../3_classification/model_mobilenet\")\n",
    "model_classify = tf.saved_model.load(str(model_dir))\n",
    "print(\"gesture classification model loaded\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start webcam displaying the live video feed\n",
    "\n",
    "This script has only been tested on Windows but should \n",
    "run also on other operation systems.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28.6fps            \r"
     ]
    }
   ],
   "source": [
    "video = cv2.VideoCapture(0)\n",
    "\n",
    "_, frame = video.read()\n",
    "\n",
    "prev_time = time.time()\n",
    "\n",
    "while True:\n",
    "    # load current video image\n",
    "    _, frame = video.read()\n",
    "        \n",
    "    img = frame\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "    img = img[60:420, 80:560]    # crop black stride\n",
    "    img = cv2.resize(img, (IMAGE_HEIGHT,IMAGE_WIDTH))\n",
    "    img = cv2.flip(img, 1)\n",
    "    \n",
    "    # Create input Tensor\n",
    "    input_tensor = tf.convert_to_tensor(img)\n",
    "    input_tensor = input_tensor[tf.newaxis,...]\n",
    "    \n",
    "    # Detect hands\n",
    "    hand_detections = model_detect(input_tensor)\n",
    "    boxes, hand_scores = filter_detected_boxes(hand_detections, threshold=THRESHOLD_HAND_DETECTION)\n",
    "\n",
    "    # Classify gestures\n",
    "    gesture_names, gesture_scores = classify_gestures_for_boxes(boxes)\n",
    "\n",
    "    # Draw boxes and labels on image\n",
    "    img = add_boxes_to_img(img, boxes, hand_scores, gesture_names, gesture_scores)\n",
    "\n",
    "    # Print current frames per second\n",
    "    fps = round(1 / (time.time() - prev_time),1)\n",
    "    print(str(fps)+\"fps\", \"\\r\", end=\"\")\n",
    "    prev_time = time.time()\n",
    "    \n",
    "    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n",
    "    img = cv2.resize(img, (600,600)) # display image a bit bigger for convenience\n",
    "    \n",
    "    cv2.imshow('webcam', img)\n",
    "    if cv2.waitKey(1) == 27: \n",
    "        break  # esc to quit\n",
    "\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}