DDW_CV_Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "38398a87-263a-4fcf-893c-c69dcf14d045",
   "metadata": {},
   "source": [
    "Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b2fa4d0-07ca-4c0a-b061-3425c67ba31b",
   "metadata": {},
   "outputs": [],
   "source": [
    "###importing all the packages required for different operations\n",
    "\n",
    "###import to get/calculate differenet distances between eyes, lips\n",
    "from scipy.spatial import distance as eu_dist\n",
    "###import pretrained face landmark detections model\n",
    "import dlib\n",
    "###import basic image processing functions\n",
    "from imutils import face_utils\n",
    "from imutils.video import VideoStream\n",
    "from imutils.video import FPS\n",
    "import imutils\n",
    "###for text to speech as alarm\n",
    "import pyttsx3\n",
    "###for handling different warning alarms\n",
    "from threading import Thread\n",
    "\n",
    "###other basic usage imports\n",
    "import numpy as np\n",
    "import argparse\n",
    "import time\n",
    "import cv2\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3271cda7-5426-408a-8b7e-0a6d61673358",
   "metadata": {},
   "source": [
    "Warning Alarm Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3524c25-a0f6-4616-96f0-c372c91d8d71",
   "metadata": {},
   "outputs": [],
   "source": [
    "###function to raise alarm for eyes closed and yawning as warning by converting text to speech\n",
    "\n",
    "###initializing the text to speech library engine\n",
    "txt_spch_engine = pyttsx3.init()\n",
    "\n",
    "###alarm function to run in thread based enclosure\n",
    "def warn_alarm(msg):\n",
    "    ###globaling declaring to store the states\n",
    "    global eye_close_warn_alarm\n",
    "    global yawn_warn_alarm\n",
    "    global bool_alarm_prsnt   ###to say if 1 alarm is already available for threading purpose\n",
    "\n",
    "    ###using loop for eyes closed because alarm should be continuous to wake up person as person is sleeping\n",
    "    while eye_close_warn_alarm:\n",
    "        print(\"Eyes Closed Warning Alarm...\")\n",
    "        ###text converted to speech and played as warning alarm\n",
    "        txt_spch_engine.say(msg)\n",
    "        txt_spch_engine.runAndWait()\n",
    "\n",
    "    ###not alarming continuously as person is yawning and not sleeping, he is awake \n",
    "    if yawn_warn_alarm:\n",
    "        print(\"Yawning Warning Alarm...\")\n",
    "        bool_alarm_prsnt = True\n",
    "        ###text converted to speech and played as warning alarm\n",
    "        txt_spch_engine.say(msg)\n",
    "        txt_spch_engine.runAndWait()\n",
    "        bool_alarm_prsnt = False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2d83b27-d224-476b-93bc-a8ed01438b80",
   "metadata": {},
   "source": [
    "Eye Aspect Ration(EAR) Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f95fd6d-7406-41b7-b5e4-9589588dd480",
   "metadata": {},
   "outputs": [],
   "source": [
    "###function to calculate eye aspect ratio to identify whether person is sleeping or not based on eyes are closed or not \n",
    "###i.e., eye coordinate's eucledian distance using formulae \n",
    "###((sum of vertical dist)/(2*horizontal dist)) of eyes or from \"EAR.jpg\", (((||p2-p6||)+(||p3-p5||))/(2*(||p1-p4||))), \n",
    "###where, p1...p6 are 2d landmark locations of eye\n",
    "def Single_E_A_R(side_eye):\n",
    "    vert_dist1 = eu_dist.euclidean(side_eye[1], side_eye[5])   ###||p2-p6||\n",
    "    vert_dist2 = eu_dist.euclidean(side_eye[2], side_eye[4])   ###||p3-p5||\n",
    "\n",
    "    hor_dist = eu_dist.euclidean(side_eye[0], side_eye[3])   ###||p1-p4||\n",
    "\n",
    "    sngl_ear = (vert_dist1 + vert_dist2) / (2.0 * hor_dist)   ###(((||p2-p6||)+(||p3-p5||))/(2*(||p1-p4||)))\n",
    "\n",
    "    return sngl_ear\n",
    "\n",
    "###calculating EAR for both left side and right side eye\n",
    "def Both_Eyes_E_A_R(shape):\n",
    "    ###get the coordinates, as in x,y of left and right eye from the DLIB using face_utils\n",
    "    ###check coordinates of each part of face in image \"Face_Landmarks.jpg\", \n",
    "    ###where left eye = 36,41 and right eye = 42,47\n",
    "    (left_eye_Start, left_eye_End) = face_utils.FACIAL_LANDMARKS_IDXS[\"left_eye\"]   ###for left eye(\"left_eye\" key in dict of library)\n",
    "    (right_eye_Start, right_eye_End) = face_utils.FACIAL_LANDMARKS_IDXS[\"right_eye\"]   ###for right eye(\"right_eye\" key in dict of library)\n",
    "\n",
    "    ###get the shapes of both eyes based on coordinates received from prev step\n",
    "    leftEye_All = shape[left_eye_Start:left_eye_End]\n",
    "    rightEye_All = shape[right_eye_Start:right_eye_End]\n",
    "\n",
    "    ###calculate the EAR of both eyes from above function\n",
    "    left_E_A_R = Single_E_A_R(leftEye_All)\n",
    "    right_E_A_R = Single_E_A_R(rightEye_All)\n",
    "\n",
    "    ###getting the average EAR of both eyes\n",
    "    E_A_R = (left_E_A_R + right_E_A_R) / 2.0\n",
    "    \n",
    "    return (E_A_R, leftEye_All, rightEye_All)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc032b30-dc94-4fa5-9615-ce1820dabcce",
   "metadata": {},
   "source": [
    "Lips Distance Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e348bd58-d4fe-48ca-afc0-ea5ecf38f40a",
   "metadata": {},
   "outputs": [],
   "source": [
    "###calculating the distance between upper and lower lip to see if the person is yawning or not\n",
    "def Up_Low_Lip_Distance(shape):\n",
    "    ###getting upper and upper-inner lip cordinates manually by checking the image \"Face_Landmarks.jpg\" as there is no direct key in dictionary to get individually required mouth coordinates\n",
    "    up_lip = shape[50:53] \n",
    "    up_in_lip = shape[61:64]\n",
    "    up_lip = np.concatenate((up_lip, up_in_lip))   ###then merge upper lip as 1\n",
    "\n",
    "    ###getting lower and lower-inner lip cordinates manually by checking the image \"Face_Landmarks.jpg\" as there is no direct key in dictionary to get individually required mouth coordinates\n",
    "    low_lip = shape[56:59]\n",
    "    low_in_lip = shape[65:68]\n",
    "    low_lip = np.concatenate((low_lip, low_in_lip))   ###then merge lower lip as 1\n",
    "\n",
    "    ###getting avg/mean of upper and lower lip\n",
    "    uplip_mean = np.mean(up_lip, axis=0)\n",
    "    lowlip_mean = np.mean(low_lip, axis=0)\n",
    "\n",
    "    ###calculating the distance between upper and lower lip\n",
    "    lip_dist = abs(uplip_mean[1] - lowlip_mean[1])\n",
    "    \n",
    "    return lip_dist"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f162ece1-e073-4210-8d07-97f541f2890c",
   "metadata": {},
   "source": [
    "Initializations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f4cb8e4-385d-4dfa-a93b-6fcf1dc145f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "Eye_E_A_R_Threshold = 0.30   ###mention the threshold value when the eyes to be considered as closed between 0-1 based on EAR(depends on size of eye and vary from person to person)\n",
    "Eye_E_A_R_Consec_Frames = 20   ###number of frames till the eyes are closed after which the Warning Alarm should be raised \n",
    "Yawn_Threshold = 70   ###mention the threshold value for yawning as in the distance between upper and lower lip to be considered as yawning\n",
    "eye_close_warn_alarm = False   ###boolean for warning alarm on closed eyes\n",
    "yawn_warn_alarm = False   ###boolean for warning alarm on yawning\n",
    "bool_alarm_prsnt = False   ###boolean for checking if 1 alarm is already available\n",
    "Counter = 0   ###frame counter for warning alarming"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "024e0102-1427-4d14-8db7-5fde91b67ad2",
   "metadata": {},
   "source": [
    "Face Landmark Detector and Predictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "567dc430-9ccc-493b-bdc1-71646844cd8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Loading Face Landmark Detector and Predictor...\")\n",
    "\n",
    "###dlib's face detector using HOG + Linear SVM(slower but great accuracy)\n",
    "#face_detector = dlib.get_frontal_face_detector()\n",
    "###opencv's face detector using HAAR Cascades(faster but less accuracy)\n",
    "face_detector = cv2.CascadeClassifier(\"G:\\\\Projects\\\\ADAS\\\\DDW_CV\\\\DLIB_Models\\\\haarcascade_frontalface_default.xml\")   ###for opencv's detector use a predefined xml\n",
    "###load dlib's face landmark predictor by using predtrained model \n",
    "face_predictor = dlib.shape_predictor('G:\\\\Projects\\\\ADAS\\\\DDW_CV\\\\DLIB_Models\\\\shape_predictor_68_face_landmarks.dat')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "899ec27a-954c-45f9-9f7f-1cee22579a4e",
   "metadata": {},
   "source": [
    "Video Stream for DDW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3ac5549-59c4-4d69-8f41-73acc83dc271",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Starting the Video Stream...\")\n",
    "\n",
    "###faster than cv2.VideoCapture and from android app webcam else give src=0 for local cam\n",
    "vid_strm = VideoStream(\"https://192.168.0.105:8080/video\").start()\n",
    "time.sleep(1.0)   ###cooldown time, not cumpolsary\n",
    "###save the video stream used for predictions\n",
    "vid_strm_save = cv2.VideoWriter(\"G:\\\\Projects\\\\ADAS\\\\DDW_CV\\\\Outputs\\\\DDW_CV.mp4\",cv2.VideoWriter_fourcc(*'MP4V'),10,(1920,1080))\n",
    "\n",
    "###initialize the FPS counter\n",
    "fps = FPS().start()\n",
    "\n",
    "###loop through the frames from video stream\n",
    "while True:\n",
    "    ###get the frame from video stream\n",
    "    vs_img = vid_strm.read()\n",
    "\n",
    "    ###if no frame then exit the process\n",
    "    if vs_img is None:\n",
    "        break\n",
    "    \n",
    "    #frame = imutils.resize(frame, width=450)   ###resize for faster streaming\n",
    "    gray_img = cv2.cvtColor(vs_img, cv2.COLOR_BGR2GRAY)   ###convert to grayscale for faster detection\n",
    "\n",
    "    ###detect faces on image using the detector defined\n",
    "    det_rects = face_detector.detectMultiScale(gray_img, scaleFactor=1.1, \n",
    "\t\tminNeighbors=5, minSize=(30, 30),\n",
    "\t\tflags=cv2.CASCADE_SCALE_IMAGE)\n",
    "\n",
    "    ###loop through the face detections\n",
    "    for (x, y, w, h) in det_rects:\n",
    "        ###form a dlib rectangle or box object on face in image from HAAR Cascade bounding box\n",
    "        det_rect = dlib.rectangle(int(x), int(y), int(x + w),int(y + h))\n",
    "\n",
    "        ###predict the detected face on img to get landmarks\n",
    "        det_shape = face_predictor(gray_img, det_rect)\n",
    "        ###convert facial landmark, i.e., x,y coordinate to numpy array\n",
    "        det_shape = face_utils.shape_to_np(det_shape)\n",
    "\n",
    "        ###get the EAR of both the eyes and each eye's coordinates\n",
    "        E_A_R, leftEye_EAR, rightEye_EAR = Both_Eyes_E_A_R(det_shape)\n",
    "\n",
    "        ###get the distance between upper and lower lips\n",
    "        uplow_lip_dist = Up_Low_Lip_Distance(det_shape)\n",
    "\n",
    "        ###to find the convexity for correction of the both eyes and draw on image for representation\n",
    "        leftEye_ConvexHull = cv2.convexHull(leftEye_EAR)\n",
    "        rightEye_ConvexHull = cv2.convexHull(rightEye_EAR)\n",
    "        cv2.drawContours(vs_img, [leftEye_ConvexHull], -1, (0, 255, 0), 1)\n",
    "        cv2.drawContours(vs_img, [rightEye_ConvexHull], -1, (0, 255, 0), 1)\n",
    "\n",
    "        ###get the mouthcoordinates and draw them on the image for representation\n",
    "        uplow_lip_outline = det_shape[48:60]\n",
    "        cv2.drawContours(vs_img, [uplow_lip_outline], -1, (0, 255, 0), 1)\n",
    "\n",
    "        ###check if the EAR is less than threshold to raise the alarm\n",
    "        if E_A_R < Eye_E_A_R_Threshold :\n",
    "            Counter += 1   ###increment the frame counter\n",
    "\n",
    "            ###if the frame counter is greater than defined sleeping frames threshold, warn with alarm\n",
    "            if Counter >= Eye_E_A_R_Consec_Frames :\n",
    "                ###check if bool alarm is not set already as in no current current alarm \n",
    "                if eye_close_warn_alarm == False :\n",
    "                    eye_close_warn_alarm = True   ###set sleeping warning bool as true\n",
    "                    ###threading to handle multiple alarms at a time\n",
    "                    thrd = Thread(target=warn_alarm, args=(\"Sleeping, Wake Up Now\",))\n",
    "                    thrd.deamon = True\n",
    "                    thrd.start()\n",
    "\n",
    "                ###put the drowsy info on image\n",
    "                cv2.putText(vs_img, \"DROWSY!!!\", (10, 30),\n",
    "                            cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "\n",
    "        else :   ###reinitialize the frame counter and alarm bool\n",
    "            Counter = 0\n",
    "            bool_alarm_prsnt = False\n",
    "\n",
    "        ###check if lip distance is greater than the yawn threshold to raise alarm\n",
    "        if (uplow_lip_dist > Yawn_Threshold) :\n",
    "            ###check if the alarm bool is not set already as in no current current alarm\n",
    "            if yawn_warn_alarm == False and bool_alarm_prsnt == False :\n",
    "                yawn_warn_alarm = True   ###set yawning bool as true\n",
    "                ###threading to handle multiple alarms at a time\n",
    "                thrd = Thread(target=warn_alarm, args=(\"Yawning, Get Freshed Up\",))\n",
    "                thrd.deamon = True\n",
    "                thrd.start()\n",
    "\n",
    "            ###draw the yawning info on image    \n",
    "            cv2.putText(vs_img, \"YAWNING!!!\", (10, 50),\n",
    "                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "            \n",
    "        else:   ###reinitialize the alarm bool\n",
    "            yawn_warn_alarm = False\n",
    "\n",
    "        ###put the EAR and Lip Distance on image\n",
    "        cv2.putText(vs_img, \"E_A_R : {:.2f}\".format(E_A_R), (300, 30),\n",
    "                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "        cv2.putText(vs_img, \"LIPDIST : {:.2f}\".format(uplow_lip_dist), (300, 60),\n",
    "                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)\n",
    "\n",
    "    ###show the frames and save the video\n",
    "    cv2.imshow(\"DDW_Frame\", vs_img)\n",
    "    vid_strm_save.write(vs_img)\n",
    "    \n",
    "    ###if the 'q' key was pressed, break from the loop\n",
    "    if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "        break    \n",
    "    ###update the FPS counter\n",
    "    fps.update()\n",
    "\n",
    "###stop the timer and display FPS information\n",
    "fps.stop()\n",
    "\n",
    "print(\"Elapsed time: {:.2f}\".format(fps.elapsed()))\n",
    "print(\"Approx. FPS: {:.2f}\".format(fps.fps()))\n",
    "\n",
    "###clean the objects\n",
    "cv2.destroyAllWindows()\n",
    "vid_strm_save.release()\n",
    "vid_strm.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91188375-82df-40fc-b933-f6ba2b240978",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}