diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab b/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab index 9d999dc1f..b374d7339 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Objects/HolisticDesktopGPU.prefab @@ -45,6 +45,7 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: config: {fileID: 4900000, guid: 329a3f95713299bb5b816d6489f5eec8, type: 3} + detectIris: 1 --- !u!114 &-5097301488528543352 MonoBehaviour: m_ObjectHideFlags: 0 @@ -65,5 +66,7 @@ MonoBehaviour: type: 3} faceLandmarksPrefab: {fileID: 2250723881323860951, guid: 570a388dca04100b8b491a9bc4ef1d51, type: 3} + irisLandmarksPrefab: {fileID: 6305487298461950687, guid: eec693dd0163888b7a498aff810792a3, + type: 3} handLandmarksPrefab: {fileID: 3100741085041498678, guid: 4eb3a6d7cfea73afdb656b7166ae1a8d, type: 3} diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_android.txt b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_android.txt index 3b559070d..ea27c344b 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_android.txt +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_android.txt @@ -17,6 +17,7 @@ # CHANGES: # - Add ImageTransformationCalculator and rotate the input # - Remove AnnotationOverlayCalculator +# - Detect iris landmarks # - Add PacketPresenceCalculator # Tracks and renders pose + hands + face landmarks. @@ -24,6 +25,9 @@ # GPU buffer. (GpuBuffer) input_stream: "input_video" +# (boolean) +input_side_packet: "enable_iris_detection" + output_stream: "pose_landmarks" output_stream: "pose_roi" output_stream: "pose_detection" @@ -100,11 +104,44 @@ node { output_stream: "POSE_LANDMARKS:pose_landmarks" output_stream: "POSE_ROI:pose_roi" output_stream: "POSE_DETECTION:pose_detection" - output_stream: "FACE_LANDMARKS:face_landmarks" + output_stream: "FACE_LANDMARKS:face_landmarks_orig" output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks" output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks" } +node { + calculator: "GateCalculator" + input_stream: "face_landmarks_orig" + input_side_packet: "ALLOW:enable_iris_detection" + output_stream: "gated_face_landmarks" +} + +# Detects iris landmarks +node { + calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksGpu" + input_stream: "IMAGE:transformed_input_video" + input_stream: "FACE_LANDMARKS:gated_face_landmarks" + output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks" + output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" + output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks" +} + +node { + calculator: "ConcatenateNormalizedLandmarkListCalculator" + input_stream: "refined_face_landmarks" + input_stream: "left_iris_landmarks" + input_stream: "right_iris_landmarks" + output_stream: "face_landmarks_with_iris" +} + +node { + calculator: "SwitchMuxCalculator" + input_side_packet: "ENABLE:enable_iris_detection" + input_stream: "C0__FUNC_INPUT:face_landmarks_orig" + input_stream: "C1__FUNC_INPUT:face_landmarks_with_iris" + output_stream: "FUNC_INPUT:face_landmarks" +} + node { calculator: "PacketPresenceCalculator" input_stream: "PACKET:pose_landmarks" diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt index 409150d57..8c7ca0dd7 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_cpu.txt @@ -17,6 +17,7 @@ # CHANGES: # - Add ImageTransformationCalculator and rotate the input # - Remove AnnotationOverlayCalculator +# - Detect iris landmarks # - Add PacketPresenceCalculator # Tracks and renders pose + hands + face landmarks. @@ -24,6 +25,9 @@ # CPU image. (ImageFrame) input_stream: "input_video" +# (boolean) +input_side_packet: "enable_iris_detection" + output_stream: "pose_landmarks" output_stream: "pose_roi" output_stream: "pose_detection" @@ -100,11 +104,44 @@ node { output_stream: "POSE_LANDMARKS:pose_landmarks" output_stream: "POSE_ROI:pose_roi" output_stream: "POSE_DETECTION:pose_detection" - output_stream: "FACE_LANDMARKS:face_landmarks" + output_stream: "FACE_LANDMARKS:face_landmarks_orig" output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks" output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks" } +node { + calculator: "GateCalculator" + input_stream: "face_landmarks_orig" + input_side_packet: "ALLOW:enable_iris_detection" + output_stream: "gated_face_landmarks" +} + +# Detects iris landmarks +node { + calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksCpu" + input_stream: "IMAGE:transformed_input_video" + input_stream: "FACE_LANDMARKS:gated_face_landmarks" + output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks" + output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" + output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks" +} + +node { + calculator: "ConcatenateNormalizedLandmarkListCalculator" + input_stream: "refined_face_landmarks" + input_stream: "left_iris_landmarks" + input_stream: "right_iris_landmarks" + output_stream: "face_landmarks_with_iris" +} + +node { + calculator: "SwitchMuxCalculator" + input_side_packet: "ENABLE:enable_iris_detection" + input_stream: "C0__FUNC_INPUT:face_landmarks_orig" + input_stream: "C1__FUNC_INPUT:face_landmarks_with_iris" + output_stream: "FUNC_INPUT:face_landmarks" +} + node { calculator: "PacketPresenceCalculator" input_stream: "PACKET:pose_landmarks" diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt index a42c451f1..2f417511b 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Resources/holistic_desktop_gpu.txt @@ -18,6 +18,7 @@ # - `input_video` is ImageFrame (ImageFrameToGpuBufferCalculator converts it into GpuBuffer) # - Add ImageTransformationCalculator and rotate the input # - Remove AnnotationOverlayCalculator +# - Detect iris landmarks # - Add PacketPresenceCalculator # Tracks and renders pose + hands + face landmarks. @@ -25,6 +26,9 @@ # ImageFrame input_stream: "input_video" +# (boolean) +input_side_packet: "enable_iris_detection" + output_stream: "pose_landmarks" output_stream: "pose_roi" output_stream: "pose_detection" @@ -52,7 +56,7 @@ output_stream: "right_hand_landmarks_presence" node { calculator: "FlowLimiterCalculator" input_stream: "input_video" - input_stream: "FINISHED:pose_landmarks_presence" + input_stream: "FINISHED:face_landmarks_presence" input_stream_info: { tag_index: "FINISHED" back_edge: true @@ -107,11 +111,44 @@ node { output_stream: "POSE_LANDMARKS:pose_landmarks" output_stream: "POSE_ROI:pose_roi" output_stream: "POSE_DETECTION:pose_detection" - output_stream: "FACE_LANDMARKS:face_landmarks" + output_stream: "FACE_LANDMARKS:face_landmarks_orig" output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks" output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks" } +node { + calculator: "GateCalculator" + input_stream: "face_landmarks_orig" + input_side_packet: "ALLOW:enable_iris_detection" + output_stream: "gated_face_landmarks" +} + +# Detects iris landmarks +node { + calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksGpu" + input_stream: "IMAGE:transformed_input_video" + input_stream: "FACE_LANDMARKS:gated_face_landmarks" + output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks" + output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" + output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks" +} + +node { + calculator: "ConcatenateNormalizedLandmarkListCalculator" + input_stream: "refined_face_landmarks" + input_stream: "left_iris_landmarks" + input_stream: "right_iris_landmarks" + output_stream: "face_landmarks_with_iris" +} + +node { + calculator: "SwitchMuxCalculator" + input_side_packet: "ENABLE:enable_iris_detection" + input_stream: "C0__FUNC_INPUT:face_landmarks_orig" + input_stream: "C1__FUNC_INPUT:face_landmarks_with_iris" + output_stream: "FUNC_INPUT:face_landmarks" +} + node { calculator: "PacketPresenceCalculator" input_stream: "PACKET:pose_landmarks" diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs index b15dd7d74..797a70e14 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticAnnotationController.cs @@ -1,25 +1,36 @@ using Mediapipe; using UnityEngine; +using System.Collections.Generic; public class HolisticAnnotationController : AnnotationController { [SerializeField] GameObject poseLandmarksPrefab = null; [SerializeField] GameObject poseRoiPrefab = null; [SerializeField] GameObject poseDetectionPrefab = null; [SerializeField] GameObject faceLandmarksPrefab = null; + [SerializeField] GameObject irisLandmarksPrefab = null; [SerializeField] GameObject handLandmarksPrefab = null; private GameObject poseLandmarksAnnotation; private GameObject poseRoiAnnotation; private GameObject poseDetectionAnnotation; private GameObject faceLandmarksAnnotation; + private GameObject leftIrisLandmarksAnnotation; + private GameObject rightIrisLandmarksAnnotation; private GameObject leftHandLandmarksAnnotation; private GameObject rightHandLandmarksAnnotation; + enum Side { + Left = 1, + Right = 2, + } + void Awake() { poseLandmarksAnnotation = Instantiate(poseLandmarksPrefab); poseRoiAnnotation = Instantiate(poseRoiPrefab); poseDetectionAnnotation = Instantiate(poseDetectionPrefab); faceLandmarksAnnotation = Instantiate(faceLandmarksPrefab); + leftIrisLandmarksAnnotation = Instantiate(irisLandmarksPrefab); + rightIrisLandmarksAnnotation = Instantiate(irisLandmarksPrefab); leftHandLandmarksAnnotation = Instantiate(handLandmarksPrefab); rightHandLandmarksAnnotation = Instantiate(handLandmarksPrefab); } @@ -29,15 +40,23 @@ void OnDestroy() { Destroy(poseRoiAnnotation); Destroy(poseDetectionAnnotation); Destroy(faceLandmarksAnnotation); + Destroy(leftIrisLandmarksAnnotation); + Destroy(rightIrisLandmarksAnnotation); Destroy(leftHandLandmarksAnnotation); Destroy(rightHandLandmarksAnnotation); } + void ClearIrisAnnotations() { + leftIrisLandmarksAnnotation.GetComponent().Clear(); + rightIrisLandmarksAnnotation.GetComponent().Clear(); + } + public override void Clear() { poseLandmarksAnnotation.GetComponent().Clear(); poseRoiAnnotation.GetComponent().Clear(); poseDetectionAnnotation.GetComponent().Clear(); faceLandmarksAnnotation.GetComponent().Clear(); + ClearIrisAnnotations(); leftHandLandmarksAnnotation.GetComponent().Clear(); rightHandLandmarksAnnotation.GetComponent().Clear(); } @@ -51,5 +70,26 @@ public void Draw(Transform screenTransform, NormalizedLandmarkList poseLandmarks faceLandmarksAnnotation.GetComponent().Draw(screenTransform, faceLandmarks, isFlipped); leftHandLandmarksAnnotation.GetComponent().Draw(screenTransform, leftHandLandmarks, isFlipped); rightHandLandmarksAnnotation.GetComponent().Draw(screenTransform, rightHandLandmarks, isFlipped); + + if (faceLandmarks.Landmark.Count == 0) { + ClearIrisAnnotations(); + } else if (faceLandmarks.Landmark.Count > 468) { + var leftIrisLandmarks = GetIrisLandmarks(faceLandmarks, Side.Left); + leftIrisLandmarksAnnotation.GetComponent().Draw(screenTransform, leftIrisLandmarks, isFlipped); + + var rightIrisLandmarks = GetIrisLandmarks(faceLandmarks, Side.Right); + rightIrisLandmarksAnnotation.GetComponent().Draw(screenTransform, rightIrisLandmarks, isFlipped); + } + } + + private IList GetIrisLandmarks(NormalizedLandmarkList landmarkList, Side side) { + var irisLandmarks = new List(5); + var offset = 468 + (side == Side.Left ? 0 : 5); + + for (var i = 0; i < 5; i++) { + irisLandmarks.Add(landmarkList.Landmark[offset + i]); + } + + return irisLandmarks; } } diff --git a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs index 5dba6c6b1..229be7b5c 100644 --- a/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs +++ b/Assets/MediaPipe/Examples/Graphs/Holistic/Scripts/HolisticGraph.cs @@ -1,6 +1,9 @@ using Mediapipe; +using UnityEngine; public class HolisticGraph : DemoGraph { + [SerializeField] bool detectIris = true; + private const string poseLandmarksStream = "pose_landmarks"; private OutputStreamPoller poseLandmarksStreamPoller; private NormalizedLandmarkListPacket poseLandmarksPacket; @@ -89,6 +92,8 @@ public override Status StartRun() { rightHandLandmarksPresencePacket = new BoolPacket(); sidePacket = new SidePacket(); + var irisFlag = new BoolPacket(detectIris); + sidePacket.Emplace("enable_iris_detection", irisFlag); return graph.StartRun(sidePacket); } diff --git a/C/third_party/mediapipe_extension.diff b/C/third_party/mediapipe_extension.diff index 6c02f060d..461792c8f 100644 --- a/C/third_party/mediapipe_extension.diff +++ b/C/third_party/mediapipe_extension.diff @@ -148,3 +148,548 @@ index 8806267..c0e66bd 100644 helper_.BindFramebuffer(dst); glActiveTexture(GL_TEXTURE1); +diff --git a/mediapipe/graphs/iris_tracking/BUILD b/mediapipe/graphs/iris_tracking/BUILD +index 86e667b..e0062a1 100644 +--- a/mediapipe/graphs/iris_tracking/BUILD ++++ b/mediapipe/graphs/iris_tracking/BUILD +@@ -30,10 +30,9 @@ cc_library( + "//mediapipe/calculators/image:image_file_properties_calculator", + "//mediapipe/calculators/image:opencv_encoded_image_to_image_frame_calculator", + "//mediapipe/calculators/image:opencv_image_encoder_calculator", +- "//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator", + "//mediapipe/graphs/iris_tracking/subgraphs:iris_and_depth_renderer_cpu", + "//mediapipe/modules/face_landmark:face_landmark_front_cpu", +- "//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_cpu", ++ "//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_from_face_landmarks_cpu", + ], + ) + +@@ -43,10 +42,9 @@ cc_library( + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/core:split_vector_calculator", +- "//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator", + "//mediapipe/graphs/iris_tracking/subgraphs:iris_renderer_cpu", + "//mediapipe/modules/face_landmark:face_landmark_front_cpu", +- "//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_cpu", ++ "//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_from_face_landmarks_cpu", + ], + ) + +@@ -58,10 +56,9 @@ cc_library( + "//mediapipe/calculators/core:split_vector_calculator", + "//mediapipe/calculators/video:opencv_video_decoder_calculator", + "//mediapipe/calculators/video:opencv_video_encoder_calculator", +- "//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator", + "//mediapipe/graphs/iris_tracking/subgraphs:iris_renderer_cpu", + "//mediapipe/modules/face_landmark:face_landmark_front_cpu", +- "//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_cpu", ++ "//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_from_face_landmarks_cpu", + ], + ) + +@@ -71,10 +68,9 @@ cc_library( + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/core:split_vector_calculator", +- "//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator", + "//mediapipe/graphs/iris_tracking/subgraphs:iris_and_depth_renderer_gpu", + "//mediapipe/modules/face_landmark:face_landmark_front_gpu", +- "//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_gpu", ++ "//mediapipe/modules/iris_landmark:iris_landmark_left_and_right_from_face_landmarks_gpu", + ], + ) + +diff --git a/mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt b/mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt +index 3597e7f..8d67b5d 100644 +--- a/mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt ++++ b/mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt +@@ -82,40 +82,12 @@ node { + } + } + +-# Gets two landmarks which define left eye boundary. +-node { +- calculator: "SplitNormalizedLandmarkListCalculator" +- input_stream: "face_landmarks" +- output_stream: "left_eye_boundary_landmarks" +- node_options: { +- [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { +- ranges: { begin: 33 end: 34 } +- ranges: { begin: 133 end: 134 } +- combine_outputs: true +- } +- } +-} +- +-# Gets two landmarks which define right eye boundary. +-node { +- calculator: "SplitNormalizedLandmarkListCalculator" +- input_stream: "face_landmarks" +- output_stream: "right_eye_boundary_landmarks" +- node_options: { +- [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { +- ranges: { begin: 362 end: 363 } +- ranges: { begin: 263 end: 264 } +- combine_outputs: true +- } +- } +-} +- + # Detects iris landmarks, eye contour landmarks, and corresponding rect (ROI). + node { +- calculator: "IrisLandmarkLeftAndRightCpu" ++ calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksCpu" + input_stream: "IMAGE:input_image" +- input_stream: "LEFT_EYE_BOUNDARY_LANDMARKS:left_eye_boundary_landmarks" +- input_stream: "RIGHT_EYE_BOUNDARY_LANDMARKS:right_eye_boundary_landmarks" ++ input_stream: "FACE_LANDMARKS:face_landmarks" ++ output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks" + output_stream: "LEFT_EYE_CONTOUR_LANDMARKS:left_eye_contour_landmarks" + output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" + output_stream: "LEFT_EYE_ROI:left_eye_rect_from_landmarks" +@@ -124,20 +96,6 @@ node { + output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks" + } + +-node { +- calculator: "ConcatenateNormalizedLandmarkListCalculator" +- input_stream: "left_eye_contour_landmarks" +- input_stream: "right_eye_contour_landmarks" +- output_stream: "refined_eye_landmarks" +-} +- +-node { +- calculator: "UpdateFaceLandmarksCalculator" +- input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks" +- input_stream: "FACE_LANDMARKS:face_landmarks" +- output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks" +-} +- + # Renders annotations and overlays them on top of the input images. + node { + calculator: "IrisAndDepthRendererCpu" +diff --git a/mediapipe/graphs/iris_tracking/iris_tracking_cpu.pbtxt b/mediapipe/graphs/iris_tracking/iris_tracking_cpu.pbtxt +index c0a3857..cec2ff3 100644 +--- a/mediapipe/graphs/iris_tracking/iris_tracking_cpu.pbtxt ++++ b/mediapipe/graphs/iris_tracking/iris_tracking_cpu.pbtxt +@@ -61,40 +61,12 @@ node { + } + } + +-# Gets two landmarks which define left eye boundary. +-node { +- calculator: "SplitNormalizedLandmarkListCalculator" +- input_stream: "face_landmarks" +- output_stream: "left_eye_boundary_landmarks" +- node_options: { +- [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { +- ranges: { begin: 33 end: 34 } +- ranges: { begin: 133 end: 134 } +- combine_outputs: true +- } +- } +-} +- +-# Gets two landmarks which define right eye boundary. +-node { +- calculator: "SplitNormalizedLandmarkListCalculator" +- input_stream: "face_landmarks" +- output_stream: "right_eye_boundary_landmarks" +- node_options: { +- [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { +- ranges: { begin: 362 end: 363 } +- ranges: { begin: 263 end: 264 } +- combine_outputs: true +- } +- } +-} +- + # Detects iris landmarks, eye contour landmarks, and corresponding rect (ROI). + node { +- calculator: "IrisLandmarkLeftAndRightCpu" ++ calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksCpu" + input_stream: "IMAGE:input_video" +- input_stream: "LEFT_EYE_BOUNDARY_LANDMARKS:left_eye_boundary_landmarks" +- input_stream: "RIGHT_EYE_BOUNDARY_LANDMARKS:right_eye_boundary_landmarks" ++ input_stream: "FACE_LANDMARKS:face_landmarks" ++ output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks" + output_stream: "LEFT_EYE_CONTOUR_LANDMARKS:left_eye_contour_landmarks" + output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" + output_stream: "LEFT_EYE_ROI:left_eye_rect_from_landmarks" +@@ -103,20 +75,6 @@ node { + output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks" + } + +-node { +- calculator: "ConcatenateNormalizedLandmarkListCalculator" +- input_stream: "left_eye_contour_landmarks" +- input_stream: "right_eye_contour_landmarks" +- output_stream: "refined_eye_landmarks" +-} +- +-node { +- calculator: "UpdateFaceLandmarksCalculator" +- input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks" +- input_stream: "FACE_LANDMARKS:face_landmarks" +- output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks" +-} +- + # Renders annotations and overlays them on top of the input images. + node { + calculator: "IrisRendererCpu" +diff --git a/mediapipe/graphs/iris_tracking/iris_tracking_cpu_video_input.pbtxt b/mediapipe/graphs/iris_tracking/iris_tracking_cpu_video_input.pbtxt +index 82229bd..bbbcea6 100644 +--- a/mediapipe/graphs/iris_tracking/iris_tracking_cpu_video_input.pbtxt ++++ b/mediapipe/graphs/iris_tracking/iris_tracking_cpu_video_input.pbtxt +@@ -64,40 +64,12 @@ node { + } + } + +-# Gets two landmarks which define left eye boundary. +-node { +- calculator: "SplitNormalizedLandmarkListCalculator" +- input_stream: "face_landmarks" +- output_stream: "left_eye_boundary_landmarks" +- node_options: { +- [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { +- ranges: { begin: 33 end: 34 } +- ranges: { begin: 133 end: 134 } +- combine_outputs: true +- } +- } +-} +- +-# Gets two landmarks which define right eye boundary. +-node { +- calculator: "SplitNormalizedLandmarkListCalculator" +- input_stream: "face_landmarks" +- output_stream: "right_eye_boundary_landmarks" +- node_options: { +- [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { +- ranges: { begin: 362 end: 363 } +- ranges: { begin: 263 end: 264 } +- combine_outputs: true +- } +- } +-} +- + # Detects iris landmarks, eye contour landmarks, and corresponding rect (ROI). + node { +- calculator: "IrisLandmarkLeftAndRightCpu" ++ calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksCpu" + input_stream: "IMAGE:input_video" +- input_stream: "LEFT_EYE_BOUNDARY_LANDMARKS:left_eye_boundary_landmarks" +- input_stream: "RIGHT_EYE_BOUNDARY_LANDMARKS:right_eye_boundary_landmarks" ++ input_stream: "FACE_LANDMARKS:face_landmarks" ++ output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks" + output_stream: "LEFT_EYE_CONTOUR_LANDMARKS:left_eye_contour_landmarks" + output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" + output_stream: "LEFT_EYE_ROI:left_eye_rect_from_landmarks" +@@ -106,20 +78,6 @@ node { + output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks" + } + +-node { +- calculator: "ConcatenateNormalizedLandmarkListCalculator" +- input_stream: "left_eye_contour_landmarks" +- input_stream: "right_eye_contour_landmarks" +- output_stream: "refined_eye_landmarks" +-} +- +-node { +- calculator: "UpdateFaceLandmarksCalculator" +- input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks" +- input_stream: "FACE_LANDMARKS:face_landmarks" +- output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks" +-} +- + # Renders annotations and overlays them on top of the input images. + node { + calculator: "IrisRendererCpu" +diff --git a/mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt b/mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt +index 505a951..944fb45 100644 +--- a/mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt ++++ b/mediapipe/graphs/iris_tracking/iris_tracking_gpu.pbtxt +@@ -81,40 +81,12 @@ node { + } + } + +-# Gets two landmarks which define left eye boundary. +-node { +- calculator: "SplitNormalizedLandmarkListCalculator" +- input_stream: "face_landmarks" +- output_stream: "left_eye_boundary_landmarks" +- node_options: { +- [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { +- ranges: { begin: 33 end: 34 } +- ranges: { begin: 133 end: 134 } +- combine_outputs: true +- } +- } +-} +- +-# Gets two landmarks which define right eye boundary. +-node { +- calculator: "SplitNormalizedLandmarkListCalculator" +- input_stream: "face_landmarks" +- output_stream: "right_eye_boundary_landmarks" +- node_options: { +- [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { +- ranges: { begin: 362 end: 363 } +- ranges: { begin: 263 end: 264 } +- combine_outputs: true +- } +- } +-} +- + # Detects iris landmarks, eye contour landmarks, and corresponding rect (ROI). + node { +- calculator: "IrisLandmarkLeftAndRightGpu" ++ calculator: "IrisLandmarkLeftAndRightFromFaceLandmarksGpu" + input_stream: "IMAGE:throttled_input_video" +- input_stream: "LEFT_EYE_BOUNDARY_LANDMARKS:left_eye_boundary_landmarks" +- input_stream: "RIGHT_EYE_BOUNDARY_LANDMARKS:right_eye_boundary_landmarks" ++ input_stream: "FACE_LANDMARKS:face_landmarks" ++ output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks" + output_stream: "LEFT_EYE_CONTOUR_LANDMARKS:left_eye_contour_landmarks" + output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" + output_stream: "LEFT_EYE_ROI:left_eye_rect_from_landmarks" +@@ -123,20 +95,6 @@ node { + output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks" + } + +-node { +- calculator: "ConcatenateNormalizedLandmarkListCalculator" +- input_stream: "left_eye_contour_landmarks" +- input_stream: "right_eye_contour_landmarks" +- output_stream: "refined_eye_landmarks" +-} +- +-node { +- calculator: "UpdateFaceLandmarksCalculator" +- input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks" +- input_stream: "FACE_LANDMARKS:face_landmarks" +- output_stream: "UPDATED_FACE_LANDMARKS:updated_face_landmarks" +-} +- + # Renders annotations and overlays them on top of the input images. + node { + calculator: "IrisAndDepthRendererGpu" +diff --git a/mediapipe/modules/iris_landmark/BUILD b/mediapipe/modules/iris_landmark/BUILD +index e16a79b..6bf377a 100644 +--- a/mediapipe/modules/iris_landmark/BUILD ++++ b/mediapipe/modules/iris_landmark/BUILD +@@ -72,6 +72,18 @@ mediapipe_simple_subgraph( + ], + ) + ++mediapipe_simple_subgraph( ++ name = "iris_landmark_left_and_right_from_face_landmarks_gpu", ++ graph = "iris_landmark_left_and_right_from_face_landmarks_gpu.pbtxt", ++ register_as = "IrisLandmarkLeftAndRightFromFaceLandmarksGpu", ++ deps = [ ++ ":iris_landmark_left_and_right_gpu", ++ "//mediapipe/calculators/core:concatenate_normalized_landmark_list_calculator", ++ "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", ++ "//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator", ++ ], ++) ++ + mediapipe_simple_subgraph( + name = "iris_landmark_left_and_right_cpu", + graph = "iris_landmark_left_and_right_cpu.pbtxt", +@@ -85,6 +97,18 @@ mediapipe_simple_subgraph( + ], + ) + ++mediapipe_simple_subgraph( ++ name = "iris_landmark_left_and_right_from_face_landmarks_cpu", ++ graph = "iris_landmark_left_and_right_from_face_landmarks_cpu.pbtxt", ++ register_as = "IrisLandmarkLeftAndRightFromFaceLandmarksCpu", ++ deps = [ ++ ":iris_landmark_left_and_right_cpu", ++ "//mediapipe/calculators/core:concatenate_normalized_landmark_list_calculator", ++ "//mediapipe/calculators/core:split_normalized_landmark_list_calculator", ++ "//mediapipe/graphs/iris_tracking/calculators:update_face_landmarks_calculator", ++ ], ++) ++ + exports_files( + srcs = [ + "iris_landmark.tflite", +diff --git a/mediapipe/modules/iris_landmark/iris_landmark_left_and_right_from_face_landmarks_cpu.pbtxt b/mediapipe/modules/iris_landmark/iris_landmark_left_and_right_from_face_landmarks_cpu.pbtxt +new file mode 100644 +index 0000000..e7772d6 +--- /dev/null ++++ b/mediapipe/modules/iris_landmark/iris_landmark_left_and_right_from_face_landmarks_cpu.pbtxt +@@ -0,0 +1,82 @@ ++# Detects iris landmarks, eye contour landmarks, and corresponding rect (ROI) from face landmarks. ++ ++type: "IrisLandmarkLeftAndRightFromFaceLandmarksCpu" ++ ++# CPU image. (ImageFrame) ++input_stream: "IMAGE:image" ++# Face landmarks. (NormalizedLandmarkList) ++input_stream: "FACE_LANDMARKS:face_landmarks" ++ ++# Refined face landmarks. (NormalizedLandmarkList) ++output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks" ++ ++# 71 normalized eye contour landmarks. (NormalizedLandmarkList) ++output_stream: "LEFT_EYE_CONTOUR_LANDMARKS:left_eye_contour_landmarks" ++# 5 normalized iris landmarks. (NormalizedLandmarkList) ++output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" ++# Region of interest used to do calculations for the left eye. (NormalizedRect) ++output_stream: "LEFT_EYE_ROI:left_eye_rect_from_landmarks" ++ ++# 71 normalized eye contour landmarks. (NormalizedLandmarkList) ++output_stream: "RIGHT_EYE_CONTOUR_LANDMARKS:right_eye_contour_landmarks" ++# 5 normalized iris landmarks. (NormalizedLandmarkList) ++output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks" ++# Region of interest used to do calculations for the right eye. (NormalizedRect) ++output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks" ++ ++ ++# Gets two landmarks which define left eye boundary. ++node { ++ calculator: "SplitNormalizedLandmarkListCalculator" ++ input_stream: "face_landmarks" ++ output_stream: "left_eye_boundary_landmarks" ++ node_options: { ++ [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { ++ ranges: { begin: 33 end: 34 } ++ ranges: { begin: 133 end: 134 } ++ combine_outputs: true ++ } ++ } ++} ++ ++# Gets two landmarks which define right eye boundary. ++node { ++ calculator: "SplitNormalizedLandmarkListCalculator" ++ input_stream: "face_landmarks" ++ output_stream: "right_eye_boundary_landmarks" ++ node_options: { ++ [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { ++ ranges: { begin: 362 end: 363 } ++ ranges: { begin: 263 end: 264 } ++ combine_outputs: true ++ } ++ } ++} ++ ++# Detects iris landmarks, eye contour landmarks, and corresponding rect (ROI). ++node { ++ calculator: "IrisLandmarkLeftAndRightCpu" ++ input_stream: "IMAGE:image" ++ input_stream: "LEFT_EYE_BOUNDARY_LANDMARKS:left_eye_boundary_landmarks" ++ input_stream: "RIGHT_EYE_BOUNDARY_LANDMARKS:right_eye_boundary_landmarks" ++ output_stream: "LEFT_EYE_CONTOUR_LANDMARKS:left_eye_contour_landmarks" ++ output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" ++ output_stream: "LEFT_EYE_ROI:left_eye_rect_from_landmarks" ++ output_stream: "RIGHT_EYE_CONTOUR_LANDMARKS:right_eye_contour_landmarks" ++ output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks" ++ output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks" ++} ++ ++node { ++ calculator: "ConcatenateNormalizedLandmarkListCalculator" ++ input_stream: "left_eye_contour_landmarks" ++ input_stream: "right_eye_contour_landmarks" ++ output_stream: "refined_eye_landmarks" ++} ++ ++node { ++ calculator: "UpdateFaceLandmarksCalculator" ++ input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks" ++ input_stream: "FACE_LANDMARKS:face_landmarks" ++ output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks" ++} +diff --git a/mediapipe/modules/iris_landmark/iris_landmark_left_and_right_from_face_landmarks_gpu.pbtxt b/mediapipe/modules/iris_landmark/iris_landmark_left_and_right_from_face_landmarks_gpu.pbtxt +new file mode 100644 +index 0000000..f0a4ce9 +--- /dev/null ++++ b/mediapipe/modules/iris_landmark/iris_landmark_left_and_right_from_face_landmarks_gpu.pbtxt +@@ -0,0 +1,81 @@ ++# Detects iris landmarks, eye contour landmarks, and corresponding rect (ROI) from face landmarks. ++ ++type: "IrisLandmarkLeftAndRightFromFaceLandmarksGpu" ++ ++# GPU image. (GpuBuffer) ++input_stream: "IMAGE:image" ++# Face landmarks. (NormalizedLandmarkList) ++input_stream: "FACE_LANDMARKS:face_landmarks" ++ ++# Refined face landmarks. (NormalizedLandmarkList) ++output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks" ++ ++# 71 normalized eye contour landmarks. (NormalizedLandmarkList) ++output_stream: "LEFT_EYE_CONTOUR_LANDMARKS:left_eye_contour_landmarks" ++# 5 normalized iris landmarks. (NormalizedLandmarkList) ++output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" ++# Region of interest used to do calculations for the left eye. (NormalizedRect) ++output_stream: "LEFT_EYE_ROI:left_eye_rect_from_landmarks" ++ ++# 71 normalized eye contour landmarks. (NormalizedLandmarkList) ++output_stream: "RIGHT_EYE_CONTOUR_LANDMARKS:right_eye_contour_landmarks" ++# 5 normalized iris landmarks. (NormalizedLandmarkList) ++output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks" ++# Region of interest used to do calculations for the right eye. (NormalizedRect) ++output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks" ++ ++# Gets two landmarks which define left eye boundary. ++node { ++ calculator: "SplitNormalizedLandmarkListCalculator" ++ input_stream: "face_landmarks" ++ output_stream: "left_eye_boundary_landmarks" ++ node_options: { ++ [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { ++ ranges: { begin: 33 end: 34 } ++ ranges: { begin: 133 end: 134 } ++ combine_outputs: true ++ } ++ } ++} ++ ++# Gets two landmarks which define right eye boundary. ++node { ++ calculator: "SplitNormalizedLandmarkListCalculator" ++ input_stream: "face_landmarks" ++ output_stream: "right_eye_boundary_landmarks" ++ node_options: { ++ [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] { ++ ranges: { begin: 362 end: 363 } ++ ranges: { begin: 263 end: 264 } ++ combine_outputs: true ++ } ++ } ++} ++ ++# Detects iris landmarks, eye contour landmarks, and corresponding rect (ROI). ++node { ++ calculator: "IrisLandmarkLeftAndRightGpu" ++ input_stream: "IMAGE:image" ++ input_stream: "LEFT_EYE_BOUNDARY_LANDMARKS:left_eye_boundary_landmarks" ++ input_stream: "RIGHT_EYE_BOUNDARY_LANDMARKS:right_eye_boundary_landmarks" ++ output_stream: "LEFT_EYE_CONTOUR_LANDMARKS:left_eye_contour_landmarks" ++ output_stream: "LEFT_EYE_IRIS_LANDMARKS:left_iris_landmarks" ++ output_stream: "LEFT_EYE_ROI:left_eye_rect_from_landmarks" ++ output_stream: "RIGHT_EYE_CONTOUR_LANDMARKS:right_eye_contour_landmarks" ++ output_stream: "RIGHT_EYE_IRIS_LANDMARKS:right_iris_landmarks" ++ output_stream: "RIGHT_EYE_ROI:right_eye_rect_from_landmarks" ++} ++ ++node { ++ calculator: "ConcatenateNormalizedLandmarkListCalculator" ++ input_stream: "left_eye_contour_landmarks" ++ input_stream: "right_eye_contour_landmarks" ++ output_stream: "refined_eye_landmarks" ++} ++ ++node { ++ calculator: "UpdateFaceLandmarksCalculator" ++ input_stream: "NEW_EYE_LANDMARKS:refined_eye_landmarks" ++ input_stream: "FACE_LANDMARKS:face_landmarks" ++ output_stream: "UPDATED_FACE_LANDMARKS:refined_face_landmarks" ++} diff --git a/ProjectSettings/ProjectSettings.asset b/ProjectSettings/ProjectSettings.asset index d688b6b1b..091632960 100644 --- a/ProjectSettings/ProjectSettings.asset +++ b/ProjectSettings/ProjectSettings.asset @@ -657,6 +657,7 @@ PlayerSettings: additionalIl2CppArgs: scriptingRuntimeVersion: 1 gcIncremental: 0 + assemblyVersionValidation: 1 gcWBarrierValidation: 0 apiCompatibilityLevelPerPlatform: {} m_RenderingPath: 1 diff --git a/README.md b/README.md index ba66122c2..e2ec9fbac 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ When building, select **only** `iOS` scene in `Scenes In Build` and set Bundle I - Iris Tracking (on CPU/GPU) - Hand Tracking (on CPU/GPU) - Pose Tracking (on CPU/GPU) -- Holistic (on CPU/GPU) +- Holistic (with Iris) (on CPU/GPU) - Hair Segmentation (on GPU) - Object Detection (on CPU/GPU) - Official Demo (on CPU/GPU) (default is Hand Tracking)