feat: Holistic with iris

homuler · Jan 30, 2021 · 0bf2bdd · 0bf2bdd
1 parent 95827dc
commit 0bf2bdd
Show file tree

Hide file tree

Showing 9 changed files with 710 additions and 5 deletions.
diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab b/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab
@@ -45,6 +45,7 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   config: {fileID: 4900000, guid: 329a3f95713299bb5b816d6489f5eec8, type: 3}
+  detectIris: 1
 --- !u!114 &-5097301488528543352
 MonoBehaviour:
   m_ObjectHideFlags: 0
@@ -65,5 +66,7 @@ MonoBehaviour:
     type: 3}
   faceLandmarksPrefab: {fileID: 2250723881323860951, guid: 570a388dca04100b8b491a9bc4ef1d51,
     type: 3}
+  irisLandmarksPrefab: {fileID: 6305487298461950687, guid: eec693dd0163888b7a498aff810792a3,
+    type: 3}
   handLandmarksPrefab: {fileID: 3100741085041498678, guid: 4eb3a6d7cfea73afdb656b7166ae1a8d,
     type: 3}
diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_android.txt b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_android.txt
@@ -17,13 +17,17 @@
 # CHANGES:
 #   - Add ImageTransformationCalculator and rotate the input
 #   - Remove AnnotationOverlayCalculator
+#   - Detect iris landmarks
 #   - Add PacketPresenceCalculator
 
 # Tracks and renders pose + hands + face landmarks.
 
 # GPU buffer. (GpuBuffer)
 input_stream: "input_video"
 
+# (boolean)
+input_side_packet: "enable_iris_detection"
+
 output_stream: "pose_landmarks"
 output_stream: "pose_roi"
 output_stream: "pose_detection"
@@ -100,11 +104,44 @@ node {
   output_stream: "POSE_LANDMARKS:pose_landmarks"
   output_stream: "POSE_ROI:pose_roi"
   output_stream: "POSE_DETECTION:pose_detection"
-  output_stream: "FACE_LANDMARKS:face_landmarks"
+  output_stream: "FACE_LANDMARKS:face_landmarks_orig"
   output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
   output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
 }
 
+node {
+  calculator: "GateCalculator"
+  input_stream: "face_landmarks_orig"
+  input_side_packet: "ALLOW:enable_iris_detection"
+  output_stream: "gated_face_landmarks"
+}
+
+# Detects iris landmarks
+node {
+  calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksGpu"
+  input_stream: "IMAGE:transformed_input_video"
+  input_stream: "FACE_LANDMARKS:gated_face_landmarks"
+  output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks"
+  output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks"
+  output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks"
+}
+
+node {
+  calculator: "ConcatenateNormalizedLandmarkListCalculator"
+  input_stream: "refined_face_landmarks"
+  input_stream: "left_iris_landmarks"
+  input_stream: "right_iris_landmarks"
+  output_stream: "face_landmarks_with_iris"
+}
+
+node {
+  calculator: "SwitchMuxCalculator"
+  input_side_packet: "ENABLE:enable_iris_detection"
+  input_stream: "C0__FUNC_INPUT:face_landmarks_orig"
+  input_stream: "C1__FUNC_INPUT:face_landmarks_with_iris"
+  output_stream: "FUNC_INPUT:face_landmarks"
+}
+
 node {
   calculator: "PacketPresenceCalculator"
   input_stream: "PACKET:pose_landmarks"

diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt
@@ -17,13 +17,17 @@
 # CHANGES:
 #   - Add ImageTransformationCalculator and rotate the input
 #   - Remove AnnotationOverlayCalculator
+#   - Detect iris landmarks
 #   - Add PacketPresenceCalculator
 
 # Tracks and renders pose + hands + face landmarks.
 
 # CPU image. (ImageFrame)
 input_stream: "input_video"
 
+# (boolean)
+input_side_packet: "enable_iris_detection"
+
 output_stream: "pose_landmarks"
 output_stream: "pose_roi"
 output_stream: "pose_detection"
@@ -100,11 +104,44 @@ node {
   output_stream: "POSE_LANDMARKS:pose_landmarks"
   output_stream: "POSE_ROI:pose_roi"
   output_stream: "POSE_DETECTION:pose_detection"
-  output_stream: "FACE_LANDMARKS:face_landmarks"
+  output_stream: "FACE_LANDMARKS:face_landmarks_orig"
   output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
   output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
 }
 
+node {
+  calculator: "GateCalculator"
+  input_stream: "face_landmarks_orig"
+  input_side_packet: "ALLOW:enable_iris_detection"
+  output_stream: "gated_face_landmarks"
+}
+
+# Detects iris landmarks
+node {
+  calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksCpu"
+  input_stream: "IMAGE:transformed_input_video"
+  input_stream: "FACE_LANDMARKS:gated_face_landmarks"
+  output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks"
+  output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks"
+  output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks"
+}
+
+node {
+  calculator: "ConcatenateNormalizedLandmarkListCalculator"
+  input_stream: "refined_face_landmarks"
+  input_stream: "left_iris_landmarks"
+  input_stream: "right_iris_landmarks"
+  output_stream: "face_landmarks_with_iris"
+}
+
+node {
+  calculator: "SwitchMuxCalculator"
+  input_side_packet: "ENABLE:enable_iris_detection"
+  input_stream: "C0__FUNC_INPUT:face_landmarks_orig"
+  input_stream: "C1__FUNC_INPUT:face_landmarks_with_iris"
+  output_stream: "FUNC_INPUT:face_landmarks"
+}
+
 node {
   calculator: "PacketPresenceCalculator"
   input_stream: "PACKET:pose_landmarks"

diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt
@@ -18,13 +18,17 @@
 #   - `input_video` is ImageFrame (ImageFrameToGpuBufferCalculator converts it into GpuBuffer)
 #   - Add ImageTransformationCalculator and rotate the input
 #   - Remove AnnotationOverlayCalculator
+#   - Detect iris landmarks
 #   - Add PacketPresenceCalculator
 
 # Tracks and renders pose + hands + face landmarks.
 
 # ImageFrame
 input_stream: "input_video"
 
+# (boolean)
+input_side_packet: "enable_iris_detection"
+
 output_stream: "pose_landmarks"
 output_stream: "pose_roi"
 output_stream: "pose_detection"
@@ -52,7 +56,7 @@ output_stream: "right_hand_landmarks_presence"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:pose_landmarks_presence"
+  input_stream: "FINISHED:face_landmarks_presence"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true
@@ -107,11 +111,44 @@ node {
   output_stream: "POSE_LANDMARKS:pose_landmarks"
   output_stream: "POSE_ROI:pose_roi"
   output_stream: "POSE_DETECTION:pose_detection"
-  output_stream: "FACE_LANDMARKS:face_landmarks"
+  output_stream: "FACE_LANDMARKS:face_landmarks_orig"
   output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
   output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
 }
 
+node {
+  calculator: "GateCalculator"
+  input_stream: "face_landmarks_orig"
+  input_side_packet: "ALLOW:enable_iris_detection"
+  output_stream: "gated_face_landmarks"
+}
+
+# Detects iris landmarks
+node {
+  calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksGpu"
+  input_stream: "IMAGE:transformed_input_video"
+  input_stream: "FACE_LANDMARKS:gated_face_landmarks"
+  output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks"
+  output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks"
+  output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks"
+}
+
+node {
+  calculator: "ConcatenateNormalizedLandmarkListCalculator"
+  input_stream: "refined_face_landmarks"
+  input_stream: "left_iris_landmarks"
+  input_stream: "right_iris_landmarks"
+  output_stream: "face_landmarks_with_iris"
+}
+
+node {
+  calculator: "SwitchMuxCalculator"
+  input_side_packet: "ENABLE:enable_iris_detection"
+  input_stream: "C0__FUNC_INPUT:face_landmarks_orig"
+  input_stream: "C1__FUNC_INPUT:face_landmarks_with_iris"
+  output_stream: "FUNC_INPUT:face_landmarks"
+}
+
 node {
   calculator: "PacketPresenceCalculator"
   input_stream: "PACKET:pose_landmarks"

diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs
@@ -1,25 +1,36 @@
 using Mediapipe;
 using UnityEngine;
+using System.Collections.Generic;
 
 public class HolisticAnnotationController : AnnotationController {
   [SerializeField] GameObject poseLandmarksPrefab = null;
   [SerializeField] GameObject poseRoiPrefab = null;
   [SerializeField] GameObject poseDetectionPrefab = null;
   [SerializeField] GameObject faceLandmarksPrefab = null;
+  [SerializeField] GameObject irisLandmarksPrefab = null;
   [SerializeField] GameObject handLandmarksPrefab = null;
 
   private GameObject poseLandmarksAnnotation;
   private GameObject poseRoiAnnotation;
   private GameObject poseDetectionAnnotation;
   private GameObject faceLandmarksAnnotation;
+  private GameObject leftIrisLandmarksAnnotation;
+  private GameObject rightIrisLandmarksAnnotation;
   private GameObject leftHandLandmarksAnnotation;
   private GameObject rightHandLandmarksAnnotation;
 
+  enum Side {
+    Left = 1,
+    Right = 2,
+  }
+
   void Awake() {
     poseLandmarksAnnotation = Instantiate(poseLandmarksPrefab);
     poseRoiAnnotation = Instantiate(poseRoiPrefab);
     poseDetectionAnnotation = Instantiate(poseDetectionPrefab);
     faceLandmarksAnnotation = Instantiate(faceLandmarksPrefab);
+    leftIrisLandmarksAnnotation = Instantiate(irisLandmarksPrefab);
+    rightIrisLandmarksAnnotation = Instantiate(irisLandmarksPrefab);
     leftHandLandmarksAnnotation = Instantiate(handLandmarksPrefab);
     rightHandLandmarksAnnotation = Instantiate(handLandmarksPrefab);
   }
@@ -29,15 +40,23 @@ public class HolisticAnnotationController : AnnotationController {
     Destroy(poseRoiAnnotation);
     Destroy(poseDetectionAnnotation);
     Destroy(faceLandmarksAnnotation);
+    Destroy(leftIrisLandmarksAnnotation);
+    Destroy(rightIrisLandmarksAnnotation);
     Destroy(leftHandLandmarksAnnotation);
     Destroy(rightHandLandmarksAnnotation);
   }
 
+  void ClearIrisAnnotations() {
+    leftIrisLandmarksAnnotation.GetComponent<IrisAnnotationController>().Clear();
+    rightIrisLandmarksAnnotation.GetComponent<IrisAnnotationController>().Clear();
+  }
+
   public override void Clear() {
     poseLandmarksAnnotation.GetComponent<FullBodyPoseLandmarkListAnnotationController>().Clear();
     poseRoiAnnotation.GetComponent<RectAnnotationController>().Clear();
     poseDetectionAnnotation.GetComponent<DetectionAnnotationController>().Clear();
     faceLandmarksAnnotation.GetComponent<FaceLandmarkListAnnotationController>().Clear();
+    ClearIrisAnnotations();
     leftHandLandmarksAnnotation.GetComponent<HandLandmarkListAnnotationController>().Clear();
     rightHandLandmarksAnnotation.GetComponent<HandLandmarkListAnnotationController>().Clear();
   }
@@ -51,5 +70,26 @@ public class HolisticAnnotationController : AnnotationController {
     faceLandmarksAnnotation.GetComponent<FaceLandmarkListAnnotationController>().Draw(screenTransform, faceLandmarks, isFlipped);
     leftHandLandmarksAnnotation.GetComponent<HandLandmarkListAnnotationController>().Draw(screenTransform, leftHandLandmarks, isFlipped);
     rightHandLandmarksAnnotation.GetComponent<HandLandmarkListAnnotationController>().Draw(screenTransform, rightHandLandmarks, isFlipped);
+
+    if (faceLandmarks.Landmark.Count == 0) {
+      ClearIrisAnnotations();
+    } else if (faceLandmarks.Landmark.Count > 468) {
+      var leftIrisLandmarks = GetIrisLandmarks(faceLandmarks, Side.Left);
+      leftIrisLandmarksAnnotation.GetComponent<IrisAnnotationController>().Draw(screenTransform, leftIrisLandmarks, isFlipped);
+
+      var rightIrisLandmarks = GetIrisLandmarks(faceLandmarks, Side.Right);
+      rightIrisLandmarksAnnotation.GetComponent<IrisAnnotationController>().Draw(screenTransform, rightIrisLandmarks, isFlipped);
+    }
+  }
+
+  private IList<NormalizedLandmark> GetIrisLandmarks(NormalizedLandmarkList landmarkList, Side side) {
+    var irisLandmarks = new List<NormalizedLandmark>(5);
+    var offset = 468 + (side == Side.Left ? 0 : 5);
+
+    for (var i = 0; i < 5; i++) {
+      irisLandmarks.Add(landmarkList.Landmark[offset + i]);
+    }
+
+    return irisLandmarks;
   }
 }
diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs
@@ -1,6 +1,9 @@
 using Mediapipe;
+using UnityEngine;
 
 public class HolisticGraph : DemoGraph {
+  [SerializeField] bool detectIris = true;
+
   private const string poseLandmarksStream = "pose_landmarks";
   private OutputStreamPoller<NormalizedLandmarkList> poseLandmarksStreamPoller;
   private NormalizedLandmarkListPacket poseLandmarksPacket;
@@ -89,6 +92,8 @@ public class HolisticGraph : DemoGraph {
     rightHandLandmarksPresencePacket = new BoolPacket();
 
     sidePacket = new SidePacket();
+    var irisFlag = new BoolPacket(detectIris);
+    sidePacket.Emplace("enable_iris_detection", irisFlag);
 
     return graph.StartRun(sidePacket);
   }